From fd8ae36bbe7c1480c25aefe487d45d5374e27c57 Mon Sep 17 00:00:00 2001 From: Kanghwan <861393+karljang@users.noreply.github.com> Date: Sun, 24 May 2026 20:56:47 -0700 Subject: [PATCH 001/308] [None][feat] Add SkipSoftmax sparse attention support for visual generation (#12947) Signed-off-by: Kanghwan Jang <861393+karljang@users.noreply.github.com> --- docs/source/features/sparse-attention.md | 77 ++ .../visual_gen/attention_backend/trtllm.py | 7 + tensorrt_llm/_torch/visual_gen/config.py | 49 ++ .../_torch/visual_gen/modules/attention.py | 21 +- .../_torch/visual_gen/pipeline_loader.py | 8 + .../_torch/visual_gen/sparse_attention.py | 17 + tensorrt_llm/visual_gen/__init__.py | 4 + tensorrt_llm/visual_gen/args.py | 24 + tensorrt_llm/visual_gen/sparse_attention.py | 381 +++++++++ .../visual_gen/test_skip_softmax_config.py | 755 ++++++++++++++++++ 10 files changed, 1342 insertions(+), 1 deletion(-) create mode 100644 tensorrt_llm/_torch/visual_gen/sparse_attention.py create mode 100644 tensorrt_llm/visual_gen/sparse_attention.py create mode 100644 tests/unittest/_torch/visual_gen/test_skip_softmax_config.py diff --git a/docs/source/features/sparse-attention.md b/docs/source/features/sparse-attention.md index a864c3407b52..9f2163eaf24a 100644 --- a/docs/source/features/sparse-attention.md +++ b/docs/source/features/sparse-attention.md @@ -350,6 +350,83 @@ Skip Softmax Attention is supported only with the **trtllm** attention backend, - **Hopper decode**: [XQA](https://github.com/NVIDIA/TensorRT-LLM/tree/main/cpp/kernels/xqa) - **Blackwell**: [trtllm-gen](https://github.com/NVIDIA/TensorRT-LLM/tree/main/cpp/tensorrt_llm/kernels/trtllmGenKernels) +#### Configuring Skip Softmax Attention + +The single tunable is the per-block **threshold** — the larger it is, the more KV +blocks are skipped (higher sparsity, lower fidelity). You have three ways to +set it, listed in priority order; the first match wins. + +**1. Raw threshold (LLM API or YAML).** The simplest path: pass a fixed value +that the kernel uses directly. Resolution-dependent; calibrate per workload. + +```python +# LLM API +from tensorrt_llm.llmapi import SkipSoftmaxAttentionConfig +sparse_attention_config = SkipSoftmaxAttentionConfig(threshold_scale_factor=1000.0) +# Or per-phase: +sparse_attention_config = SkipSoftmaxAttentionConfig( + threshold_scale_factor={"prefill": 1000.0, "decode": 500.0} +) +``` + +```yaml +# YAML (e.g., extra_llm_api_options for trtllm-serve / trtllm-bench) +attention: + backend: TRTLLM + sparse_attention_config: + algorithm: skip_softmax + threshold_scale_factor: 1000.0 +``` + +**2. Target sparsity (visual-generation models).** Visual-gen pipelines also +accept a semantic `target_sparsity` (∈ [0, 1]). The threshold is computed from +a calibration formula `threshold = exp(log_a + b · target_sparsity)` whose +coefficients come from ModelOpt — either a sparse YAML +(`sparse_config_path`) or the checkpoint's `config.json`. You do *not* set +the formula coefficients yourself; they are an internal ModelOpt-produced +artifact. + +```yaml +# Point at a ModelOpt-produced sparse YAML and just set the sparsity target: +attention: + backend: TRTLLM + sparse_config_path: /path/to/sparse.yaml + sparse_attention_config: + algorithm: skip_softmax + target_sparsity: 0.70 +``` + +`sparse_config_path` is a path to the ModelOpt-produced sparse-attention +artifact, not a second TRT-LLM runtime configuration file. For +`trtllm-serve`/`trtllm-bench`, put this path inside the normal `--config` +YAML. If a consolidated `sparse.yaml` is already present at the checkpoint +root, omit `sparse_config_path` and let the loader auto-detect it. + +**3. Auto-detect from checkpoint.** ModelOpt-calibrated checkpoints ship the +calibration formula in their `config.json` under +`sparse_attention_config.threshold_scale_factor.prefill = {a, b}`, or in a +consolidated `sparse.yaml` at the checkpoint root. The pipeline loader picks +either up automatically — no `sparse_config_path` needed for the common case. +Auto-detection supplies calibration only; the user must still set +`target_sparsity` or an explicit `threshold_scale_factor` to enable skip-softmax. + +```yaml +# With a ModelOpt-calibrated checkpoint, this is enough: +attention: + backend: TRTLLM + sparse_attention_config: + algorithm: skip_softmax + target_sparsity: 0.70 # formula auto-resolved from checkpoint +``` + +**Per-layer overrides** are part of the ModelOpt calibration output (the +`disabled_layers` list in `sparse.yaml`), not a user-facing knob. To disable +skip-softmax on specific layers, edit the ModelOpt sparse YAML — the pipeline +loader will pick up the overrides automatically. + +For algorithm details, see the [paper](https://arxiv.org/pdf/2512.12087) and +the [tech blog](../blogs/tech_blog/blog16_Accelerating_Long_Context_Inference_with_Skip_Softmax_Attention.md). + ### Summary diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/trtllm.py b/tensorrt_llm/_torch/visual_gen/attention_backend/trtllm.py index a064bb34b439..53294b12edad 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/trtllm.py @@ -23,6 +23,7 @@ import torch +from tensorrt_llm.llmapi.llm_args import SkipSoftmaxAttentionConfig from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.visual_gen.args import QuantAttentionConfig @@ -202,6 +203,7 @@ def __init__( max_seq_len: int = 4096, quant_attention_config: Optional[QuantAttentionConfig] = None, attention_metadata_state: Optional[dict] = None, + sparse_attention_config: Optional[SkipSoftmaxAttentionConfig] = None, ): num_kv_heads = num_kv_heads or num_heads @@ -213,6 +215,11 @@ def __init__( quant_config=quant_config, dtype=dtype, ) + # Plain attribute (no construct-time caching). The kernel re-reads + # self.sparse_attention_config per forward call, so callers like + # apply_skip_softmax_overrides() can swap or clear it post-construction + # and the next forward picks up the change. + self.sparse_attention_config = sparse_attention_config # TRTLLM expects flat [B*S, H*D] format self._preferred_layout = AttentionTensorLayout.NHD diff --git a/tensorrt_llm/_torch/visual_gen/config.py b/tensorrt_llm/_torch/visual_gen/config.py index 11f83aacd01f..3f7009bef569 100644 --- a/tensorrt_llm/_torch/visual_gen/config.py +++ b/tensorrt_llm/_torch/visual_gen/config.py @@ -40,6 +40,16 @@ TorchCompileConfig, VisualGenArgs, ) +from tensorrt_llm.visual_gen.sparse_attention import SkipSoftmaxConfig +from tensorrt_llm.visual_gen.sparse_attention import ( + auto_detect_sparse_attention_config as _auto_detect_sparse_attention_config, +) +from tensorrt_llm.visual_gen.sparse_attention import ( + auto_detect_sparse_yaml as _auto_detect_sparse_yaml, +) +from tensorrt_llm.visual_gen.sparse_attention import ( + load_sparse_config_from_yaml as _load_sparse_config_from_yaml, +) # ============================================================================= # Utilities @@ -466,6 +476,45 @@ def from_pretrained( "safetensors with embedded config metadata." ) + # Load sparse attention calibration. ModelOpt artifacts carry the + # calibration formula and disabled-layer/component map; user config + # contributes only public knobs such as target_sparsity. + yaml_sparse = None + yaml_path = attention_cfg.sparse_config_path + if yaml_path is not None: + yaml_sparse = _load_sparse_config_from_yaml(yaml_path) + if yaml_sparse is not None: + logger.info(f"Loaded sparse config from {yaml_path}") + + if yaml_sparse is None: + yaml_sparse = _auto_detect_sparse_yaml(str(checkpoint_path)) + if yaml_sparse is not None: + logger.info("Auto-detected sparse config YAML from checkpoint") + + if yaml_sparse is None: + ckpt_dict = vars(pretrained_config) if pretrained_config else {} + yaml_sparse = _auto_detect_sparse_attention_config(ckpt_dict) + if yaml_sparse is not None: + formula = yaml_sparse._formula + if formula is not None: + logger.info( + "Auto-detected sparse config from config.json " + f"(formula: log_a={formula.log_a:.2f}, b={formula.b:.2f})" + ) + else: + logger.info("Auto-detected sparse config from config.json") + + if yaml_sparse is not None: + user_cfg = attention_cfg.sparse_attention_config + if user_cfg is not None and isinstance(user_cfg, SkipSoftmaxConfig): + attention_cfg = attention_cfg.model_copy( + update={"sparse_attention_config": yaml_sparse._with_public_overrides(user_cfg)} + ) + else: + attention_cfg = attention_cfg.model_copy( + update={"sparse_attention_config": yaml_sparse} + ) + # Resolve quant_config. A user dict containing ``quant_algo`` # is parsed via ``load_diffusion_quant_config``; a user dict # without ``quant_algo`` (including ``{}``) falls through to the diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index dc01f8926403..e6f9f66a0a8c 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -4,11 +4,13 @@ import torch import torch.nn as nn +from tensorrt_llm.llmapi.llm_args import SkipSoftmaxAttentionConfig + from ...modules.linear import Linear, WeightMode, WeightsLoadingConfig from ...modules.rms_norm import RMSNorm from ..attention_backend.interface import AttentionTensorLayout from ..attention_backend.utils import create_attention -from ..config import DiffusionModelConfig +from ..config import DiffusionModelConfig, SkipSoftmaxConfig class QKVMode(str, Enum): @@ -158,6 +160,22 @@ def __init__( backend_num_heads = self.num_attention_heads backend_num_kv_heads = self.num_key_value_heads + # Resolve sparse attention config for TRTLLM backend + sparse_attention_config = None + ss_cfg = config.attention.sparse_attention_config + if isinstance(ss_cfg, SkipSoftmaxConfig) and backend_name == "TRTLLM": + # Cache the resolved scalar on a private attr (idempotent across + # all Attention modules); does NOT mutate the source-of-truth + # `threshold_scale_factor` / `target_sparsity` fields. Subsequent + # callers — including `apply_skip_softmax_overrides` — read the + # cached value via `resolve_threshold(module_name)`. + threshold = ss_cfg.get_or_resolve_threshold() + + if threshold is not None and threshold > 0: + sparse_attention_config = SkipSoftmaxAttentionConfig( + threshold_scale_factor={"prefill": threshold, "decode": 0} + ) + # Create compute backend self.attn = create_attention( backend=backend_name, @@ -169,6 +187,7 @@ def __init__( dtype=self.dtype, attention_config=config.attention, attention_metadata_state=attention_metadata_state, + sparse_attention_config=sparse_attention_config, ) # Wrap with parallelism strategy (orthogonal to backend choice) diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py index e7c5a40bd9f9..cb015195d357 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py @@ -26,6 +26,7 @@ from tensorrt_llm.llmapi.utils import download_hf_model from tensorrt_llm.logger import logger from tensorrt_llm.visual_gen.args import VisualGenArgs +from tensorrt_llm.visual_gen.sparse_attention import SkipSoftmaxConfig, apply_skip_softmax_overrides from .config import DiffusionModelConfig from .mapping import VisualGenMapping @@ -274,6 +275,13 @@ def load( if hasattr(pipeline, "post_load_weights"): pipeline.post_load_weights() + sparse_cfg = config.attention.sparse_attention_config + if isinstance(sparse_cfg, SkipSoftmaxConfig) and ( + sparse_cfg._layer_overrides or sparse_cfg._component_configs + ): + n = apply_skip_softmax_overrides(pipeline, sparse_cfg) + logger.info(f"Applied skip_softmax sparse config to {n} backends") + if config.torch_compile.enable: torch._dynamo.config.cache_size_limit = 128 pipeline.torch_compile() diff --git a/tensorrt_llm/_torch/visual_gen/sparse_attention.py b/tensorrt_llm/_torch/visual_gen/sparse_attention.py new file mode 100644 index 000000000000..416faa91b2e0 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/sparse_attention.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compatibility wrapper for visual-gen sparse attention helpers.""" + +from tensorrt_llm.visual_gen.sparse_attention import * # noqa: F403 diff --git a/tensorrt_llm/visual_gen/__init__.py b/tensorrt_llm/visual_gen/__init__.py index 2bf306a6ea50..0975fa2ee2ca 100644 --- a/tensorrt_llm/visual_gen/__init__.py +++ b/tensorrt_llm/visual_gen/__init__.py @@ -32,6 +32,8 @@ CudaGraphConfig, ParallelConfig, QuantAttentionConfig, + SkipSoftmaxConfig, + SparseAttentionConfig, TeaCacheConfig, TorchCompileConfig, VisualGenArgs, @@ -56,6 +58,8 @@ "ParallelConfig", "AttentionConfig", "QuantAttentionConfig", + "SparseAttentionConfig", + "SkipSoftmaxConfig", "CacheConfig", "TeaCacheConfig", "CacheDiTConfig", diff --git a/tensorrt_llm/visual_gen/args.py b/tensorrt_llm/visual_gen/args.py index b2d23dbcac60..8f66a5ad31ab 100644 --- a/tensorrt_llm/visual_gen/args.py +++ b/tensorrt_llm/visual_gen/args.py @@ -30,6 +30,8 @@ from tensorrt_llm.llmapi.utils import StrictBaseModel, set_api_status from tensorrt_llm.models.modeling_utils import QuantConfig +from .sparse_attention import SkipSoftmaxConfig + # ============================================================================= # Type aliases # ============================================================================= @@ -90,6 +92,13 @@ class QuantAttentionConfig(StrictBaseModel): ) +# Discriminated union of sparse attention configs. +SparseAttentionConfig = Annotated[ + Union[SkipSoftmaxConfig], + Field(discriminator="algorithm"), +] + + class AttentionConfig(StrictBaseModel): """Configuration for Attention layers.""" @@ -107,6 +116,19 @@ class AttentionConfig(StrictBaseModel): "attention; leave as None to disable." ), ) + sparse_attention_config: Optional[SparseAttentionConfig] = Field( + None, + status="prototype", + description="Sparse attention configuration. Currently supports: skip_softmax.", + ) + sparse_config_path: Optional[str] = Field( + None, + status="prototype", + description=( + "Path to a ModelOpt sparse attention YAML config file. " + "Overrides auto-detection from the checkpoint directory." + ), + ) @model_validator(mode="after") def _validate_quant_attention_config(self) -> "AttentionConfig": @@ -552,6 +574,8 @@ def from_yaml(cls, yaml_path: Union[str, Path], **overrides: Any) -> "VisualGenA __all__ = [ "QuantAttentionConfig", + "SparseAttentionConfig", + "SkipSoftmaxConfig", "AttentionConfig", "ParallelConfig", "BaseCacheConfig", diff --git a/tensorrt_llm/visual_gen/sparse_attention.py b/tensorrt_llm/visual_gen/sparse_attention.py new file mode 100644 index 000000000000..41409c6ac6d6 --- /dev/null +++ b/tensorrt_llm/visual_gen/sparse_attention.py @@ -0,0 +1,381 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Skip-softmax sparse attention helpers for visual generation.""" + +import fnmatch +import math +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Optional, Union + +import yaml +from pydantic import Field as PydanticField +from pydantic import PrivateAttr, model_validator + +from tensorrt_llm.llmapi.llm_args import SkipSoftmaxAttentionConfig +from tensorrt_llm.llmapi.utils import StrictBaseModel + +if TYPE_CHECKING: + import torch + + +class SkipSoftmaxFormula(StrictBaseModel): + """Exponential calibration formula: threshold = exp(log_a + b * sparsity). + + Equivalent to: threshold = a * exp(b * sparsity) where a = exp(log_a). + Stored in log-space (log_a) to match ModelOpt diffusion format and + avoid precision loss. Accepts either 'log_a' (diffusion format) or 'a' + (LLM format) at construction; 'a' is normalized to log_a = log(a). + """ + + log_a: float = PydanticField(description="Log of coefficient a (log-space)") + b: float = PydanticField(description="Coefficient b") + + @model_validator(mode="before") + @classmethod + def _accept_linear_a(cls, values): + """Normalize LLM-format 'a' to diffusion-format 'log_a'.""" + if not isinstance(values, dict) or "a" not in values: + return values + if "log_a" in values: + raise ValueError( + "SkipSoftmaxFormula: specify either 'log_a' (diffusion format) " + "or 'a' (LLM format), not both." + ) + a = values["a"] + if a <= 0: + raise ValueError( + f"SkipSoftmaxFormula: 'a' must be positive (got {a}). " + "Use 'log_a' directly if you need log(a) of a non-positive value." + ) + values = {**values} + values["log_a"] = math.log(a) + values.pop("a") + return values + + +class SkipSoftmaxConfig(SkipSoftmaxAttentionConfig): + """SkipSoftmax sparse attention configuration for visual generation. + + Extends the shared :class:`SkipSoftmaxAttentionConfig` from + ``tensorrt_llm.llmapi.llm_args`` (used by the LLM backend) without adding + any user-facing fields. The user-facing surface is exactly: + + - ``threshold_scale_factor`` — raw value, resolution-dependent. + - ``target_sparsity`` — semantic target; needs calibration to resolve. + + Calibration state (formula coefficients and per-layer overrides) is loaded + from ModelOpt-produced artifacts (``sparse.yaml`` or checkpoint + ``config.json``) and stored in private attributes — it is *not* settable + via the user-facing constructor or YAML config. Users wanting custom + calibration should author a ModelOpt sparse YAML and point + :attr:`AttentionConfig.sparse_config_path` at it. + + To attach calibration in code (loaders / tests), use + :meth:`SkipSoftmaxConfig.with_calibration`. + """ + + _formula: Optional[SkipSoftmaxFormula] = PrivateAttr(default=None) + _layer_overrides: Optional[Dict[str, float]] = PrivateAttr(default=None) + _component_configs: Optional[Dict[str, "SkipSoftmaxConfig"]] = PrivateAttr(default=None) + _resolved_threshold_prefill: Optional[float] = PrivateAttr(default=None) + + @classmethod + def with_calibration( + cls, + *, + threshold_scale_factor: Optional[Union[float, Dict[str, float]]] = None, + target_sparsity: Optional[Union[float, Dict[str, float]]] = None, + formula: Optional[SkipSoftmaxFormula] = None, + layer_overrides: Optional[Dict[str, float]] = None, + component_configs: Optional[Dict[str, "SkipSoftmaxConfig"]] = None, + ) -> "SkipSoftmaxConfig": + """Internal factory used by YAML/checkpoint loaders and tests.""" + kwargs = {} + if threshold_scale_factor is not None: + kwargs["threshold_scale_factor"] = threshold_scale_factor + if target_sparsity is not None: + kwargs["target_sparsity"] = target_sparsity + cfg = cls(**kwargs) + cfg._formula = formula + cfg._layer_overrides = layer_overrides + cfg._component_configs = component_configs + return cfg + + def _with_public_overrides(self, user_cfg: "SkipSoftmaxConfig") -> "SkipSoftmaxConfig": + """Copy user-facing sparse knobs onto calibration loaded from ModelOpt.""" + updates = { + k: v + for k, v in { + "threshold_scale_factor": user_cfg.threshold_scale_factor, + "target_sparsity": user_cfg.target_sparsity, + }.items() + if v is not None + } + if not updates: + return self + + merged = self.model_copy(update=updates) + if self._component_configs: + merged._component_configs = { + name: component.model_copy(update=updates) + for name, component in self._component_configs.items() + } + return merged + + def get_or_resolve_threshold( + self, + checkpoint_formula: Optional[Dict[str, float]] = None, + ) -> Optional[float]: + """Return the resolved prefill threshold, caching after the first call.""" + if self._resolved_threshold_prefill is not None: + return self._resolved_threshold_prefill + if ( + self._component_configs + and self._formula is None + and self.threshold_scale_factor_prefill is None + ): + return None + threshold = self.resolve_threshold_scale_factor(checkpoint_formula) + if threshold is not None and threshold > 0: + self._resolved_threshold_prefill = threshold + return threshold + + def resolve_threshold(self, module_name: str) -> Optional[float]: + """Resolve the threshold for a specific layer by module name.""" + if self._component_configs: + component = self._component_config_for_module_name(module_name) + if component is None: + return None + component_cfg, relative_name = component + return component_cfg.resolve_threshold(relative_name) + + threshold = self.get_or_resolve_threshold() + if threshold is None: + return None + if self._layer_overrides: + candidate_names = self._layer_override_match_names(module_name) + for pattern, override in self._layer_overrides.items(): + if any(fnmatch.fnmatch(name, pattern) for name in candidate_names): + threshold = override + break + return threshold if threshold > 0 else None + + @staticmethod + def _layer_override_match_names(module_name: str) -> tuple[str, ...]: + """Return full and component-relative names for ModelOpt override matching.""" + candidate_names = {module_name, module_name.replace("._orig_mod.", ".")} + for name in tuple(candidate_names): + for prefix in ("transformer.", "transformer_2."): + if name.startswith(prefix): + candidate_names.add(name[len(prefix) :]) + return tuple(candidate_names) + + def _component_config_for_module_name( + self, module_name: str + ) -> Optional[tuple["SkipSoftmaxConfig", str]]: + if not self._component_configs: + return None + normalized_name = module_name.replace("._orig_mod.", ".") + for component_name, component_cfg in sorted( + self._component_configs.items(), key=lambda item: len(item[0]), reverse=True + ): + prefix = f"{component_name}." + if normalized_name == component_name: + return component_cfg, "" + if normalized_name.startswith(prefix): + return component_cfg, normalized_name[len(prefix) :] + return None + + def resolve_threshold_scale_factor( + self, + checkpoint_formula: Optional[Dict[str, float]] = None, + ) -> Optional[float]: + """Resolve to a concrete prefill threshold using the shared LLM resolver.""" + if self.threshold_scale_factor_prefill is not None: + return self.threshold_scale_factor_prefill + + sparsity = self.target_sparsity_prefill + if sparsity is None: + return None + + formula = self._shared_formula(checkpoint_formula) + if formula is None: + raise ValueError( + "SkipSoftmaxConfig: target_sparsity requires calibration formula " + "coefficients. Provide via a ModelOpt sparse YAML " + "(sparse_config_path) or a checkpoint config.json carrying " + "calibrated coefficients." + ) + resolved = SkipSoftmaxAttentionConfig( + algorithm=self.algorithm, + target_sparsity=self.target_sparsity, + ).resolve_for_target_sparsity(formula) + return resolved.threshold_scale_factor_prefill + + def _shared_formula( + self, + checkpoint_formula: Optional[Dict[str, float]], + ) -> Optional[Dict[str, Dict[str, float]]]: + if self._formula: + coeffs = {"a": math.exp(self._formula.log_a), "b": self._formula.b} + elif checkpoint_formula: + coeffs = self._shared_formula_coefficients(checkpoint_formula) + else: + return None + if coeffs is None: + return None + return {"prefill": coeffs, "decode": coeffs} + + @staticmethod + def _shared_formula_coefficients( + formula: Dict[str, float], + ) -> Optional[Dict[str, float]]: + if "log_a" in formula and "b" in formula: + return {"a": math.exp(formula["log_a"]), "b": formula["b"]} + if "a" in formula and "b" in formula: + return {"a": formula["a"], "b": formula["b"]} + return None + + +def _load_sparse_config_group_container(data: Dict[str, Any]) -> Optional[SkipSoftmaxConfig]: + """Load one component's skip-softmax config from a ``config_groups`` container.""" + config_groups = data.get("config_groups", {}) + if not isinstance(config_groups, dict): + return None + + for group in config_groups.values(): + if not isinstance(group, dict) or group.get("sparse_algo") != "softmax_skip": + continue + + tsf = group.get("threshold_scale_factor", {}) + prefill = tsf.get("prefill", {}) if isinstance(tsf, dict) else {} + if "b" not in prefill or ("log_a" not in prefill and "a" not in prefill): + continue + + disabled = group.get("disabled_layers", []) + if not isinstance(disabled, list): + disabled = [] + layer_overrides: Optional[Dict[str, float]] = ( + {str(name): 0.0 for name in disabled} if disabled else None + ) + + formula_kwargs = {k: prefill[k] for k in ("log_a", "a", "b") if k in prefill} + return SkipSoftmaxConfig.with_calibration( + formula=SkipSoftmaxFormula(**formula_kwargs), + layer_overrides=layer_overrides, + ) + + return None + + +def load_sparse_config_from_yaml(yaml_path: str) -> Optional[SkipSoftmaxConfig]: + """Load SkipSoftmaxConfig from a ModelOpt sparse attention YAML file.""" + with open(yaml_path) as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict): + return None + + cfg = _load_sparse_config_group_container(data) + if cfg is not None: + return cfg + + component_configs = {} + for component_name, component_data in data.items(): + if not isinstance(component_data, dict): + continue + cfg = _load_sparse_config_group_container(component_data) + if cfg is not None: + component_configs[component_name] = cfg + + if component_configs: + return SkipSoftmaxConfig.with_calibration(component_configs=component_configs) + + return None + + +def auto_detect_sparse_yaml(checkpoint_dir: str) -> Optional[SkipSoftmaxConfig]: + """Auto-detect the current consolidated ModelOpt sparse YAML at checkpoint root.""" + checkpoint_path = Path(checkpoint_dir) + candidates = [checkpoint_path / "sparse.yaml"] + candidates.extend(checkpoint_path.glob("sparse.*.yaml")) + candidates = sorted({path for path in candidates if path.is_file()}) + if not candidates: + return None + if len(candidates) > 1: + raise ValueError( + "auto_detect_sparse_yaml: multiple sparse YAML files found at " + f"{checkpoint_path}: {candidates}. Pass an explicit `sparse_config_path` " + "to disambiguate." + ) + return load_sparse_config_from_yaml(str(candidates[0])) + + +def auto_detect_sparse_attention_config( + checkpoint_config: Dict[str, Any], +) -> Optional[SkipSoftmaxConfig]: + """Auto-detect sparse attention calibration from ModelOpt checkpoint config.json.""" + sparse_cfg = checkpoint_config.get("sparse_attention_config") + if not isinstance(sparse_cfg, dict): + return None + + tsf = sparse_cfg.get("threshold_scale_factor") + if not isinstance(tsf, dict): + return None + + prefill = tsf.get("prefill") + if not isinstance(prefill, dict): + return None + + if "log_a" in prefill and "b" in prefill: + return SkipSoftmaxConfig.with_calibration( + formula=SkipSoftmaxFormula(log_a=prefill["log_a"], b=prefill["b"]), + ) + if "a" in prefill and "b" in prefill: + return SkipSoftmaxConfig.with_calibration( + formula=SkipSoftmaxFormula(log_a=math.log(prefill["a"]), b=prefill["b"]), + ) + + return None + + +def apply_skip_softmax_overrides(model: "torch.nn.Module", skip_softmax: SkipSoftmaxConfig) -> int: + """Apply component-specific skip-softmax calibration to constructed TRTLLM backends.""" + if skip_softmax._layer_overrides is None and skip_softmax._component_configs is None: + return 0 + + from tensorrt_llm._torch.visual_gen.attention_backend.trtllm import TrtllmAttention + + modified = 0 + for name, module in model.named_modules(): + threshold = skip_softmax.resolve_threshold(name) + attn = getattr(module, "attn", None) + targets = [] + if isinstance(attn, TrtllmAttention): + targets.append(attn) + inner = getattr(attn, "inner_backend", None) + if isinstance(inner, TrtllmAttention): + targets.append(inner) + + for target in targets: + if threshold is not None: + target.sparse_attention_config = SkipSoftmaxAttentionConfig( + threshold_scale_factor={"prefill": threshold, "decode": 0} + ) + else: + target.sparse_attention_config = None + modified += 1 + + return modified diff --git a/tests/unittest/_torch/visual_gen/test_skip_softmax_config.py b/tests/unittest/_torch/visual_gen/test_skip_softmax_config.py new file mode 100644 index 000000000000..99f61eefb42e --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_skip_softmax_config.py @@ -0,0 +1,755 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for SkipSoftmax visual generation config and wiring.""" + +import math +from typing import Final + +import pytest + +from tensorrt_llm.llmapi.llm_args import SkipSoftmaxAttentionConfig +from tensorrt_llm.visual_gen.args import AttentionConfig, QuantAttentionConfig +from tensorrt_llm.visual_gen.sparse_attention import ( + SkipSoftmaxConfig, + SkipSoftmaxFormula, + apply_skip_softmax_overrides, +) + +# ============================================================================= +# SkipSoftmaxFormula — accepts both log_a (diffusion) and a (LLM) formats +# ============================================================================= + + +class TestSkipSoftmaxFormulaFormats: + def test_accepts_log_a(self): + """Diffusion format: log_a stored directly.""" + f = SkipSoftmaxFormula(log_a=-14.409, b=37.457) + assert f.log_a == pytest.approx(-14.409) + assert f.b == pytest.approx(37.457) + + def test_accepts_linear_a_and_normalizes(self): + """LLM format: a is normalized to log_a = log(a).""" + f = SkipSoftmaxFormula(a=7e-5, b=7.929109) + assert f.log_a == pytest.approx(math.log(7e-5)) + assert f.b == pytest.approx(7.929109) + + def test_rejects_both_log_a_and_a(self): + """Specifying both is ambiguous — error rather than silently pick one.""" + with pytest.raises(ValueError, match="not both"): + SkipSoftmaxFormula(log_a=-10.0, a=999.0, b=5.0) + + def test_rejects_non_positive_a(self): + """Linear 'a' must be positive (log of 0/negative is undefined).""" + with pytest.raises(ValueError, match="must be positive"): + SkipSoftmaxFormula(a=0.0, b=5.0) + with pytest.raises(ValueError, match="must be positive"): + SkipSoftmaxFormula(a=-1.0, b=5.0) + + +# ============================================================================= +# SkipSoftmaxConfig construction +# ============================================================================= + + +class TestSkipSoftmaxConfigConstruction: + def test_attention_config_from_full_dict(self): + """Discriminator dispatch from a raw dict populating every user-facing field. + + Subsumes the per-field assignment tests (Pydantic's contract) and the + narrower dict-construction variants. Calibration coefficients + (``formula``, ``layer_overrides``) are intentionally *not* user-facing + — they are loaded from ModelOpt YAML via ``with_calibration`` and + live in private attrs; passing them here would (correctly) fail + StrictBaseModel validation. + """ + cfg = AttentionConfig( + **{ + "backend": "TRTLLM", + "sparse_attention_config": { + "algorithm": "skip_softmax", + "threshold_scale_factor": 5000.0, + "target_sparsity": 0.5, + }, + } + ) + sc = cfg.sparse_attention_config + assert isinstance(sc, SkipSoftmaxConfig) + assert sc.algorithm == "skip_softmax" + assert sc.threshold_scale_factor == 5000.0 + assert sc.target_sparsity == 0.5 + + def test_attention_config_rejects_calibration_fields(self): + """``formula`` and ``layer_overrides`` are not part of the user surface. + + Guards against accidentally re-exposing calibration internals on the + Pydantic model. ``StrictBaseModel`` rejects unknown fields with + ``ValidationError``. + """ + from pydantic import ValidationError + + for field, value in [ + ("formula", {"log_a": math.log(0.0003), "b": 7.5}), + ("layer_overrides", {"transformer_blocks.0.*": 0}), + ]: + with pytest.raises(ValidationError): + AttentionConfig( + backend="TRTLLM", + sparse_attention_config={ + "algorithm": "skip_softmax", + "threshold_scale_factor": 5000.0, + field: value, + }, + ) + + def test_attention_config_round_trip(self): + """``model_dump()`` → ``AttentionConfig(**dumped)`` preserves all fields. + + Guards against future serialization regressions on the user-facing + surface: dropped discriminator, alias drift on the inherited llmapi + fields. Calibration state is private and intentionally outside the + dump/load contract. + """ + original = AttentionConfig( + backend="TRTLLM", + sparse_attention_config=SkipSoftmaxConfig( + threshold_scale_factor=5000.0, + target_sparsity=0.5, + ), + ) + dumped = original.model_dump() + rehydrated = AttentionConfig(**dumped) + assert rehydrated.model_dump() == dumped + + def test_attention_config_no_sparse(self): + cfg = AttentionConfig(backend="VANILLA") + assert cfg.sparse_attention_config is None + + def test_base_class_inheritance(self): + cfg = SkipSoftmaxConfig(threshold_scale_factor=5000.0) + # Inherits from the LLM-shared SkipSoftmaxAttentionConfig (reuse, no duplication) + assert isinstance(cfg, SkipSoftmaxAttentionConfig) + assert cfg.algorithm == "skip_softmax" + + def test_quant_attention_requires_trtllm_backend(self): + """Quantized attention requires backend='TRTLLM'.""" + with pytest.raises(ValueError, match="requires backend='TRTLLM'"): + AttentionConfig(backend="VANILLA", quant_attention_config=QuantAttentionConfig()) + + def test_quant_attention_rejects_unsupported_block_combo(self): + """The validator must reject unsupported quantized-attention recipes.""" + with pytest.raises(ValueError, match="Unsupported quant_attention_config"): + AttentionConfig( + backend="TRTLLM", + quant_attention_config=QuantAttentionConfig( + q_block_size=99, + k_block_size=99, + v_block_size=99, + ), + ) + + +# ============================================================================= +# Use case scenarios +# ============================================================================= + + +class TestUseCaseScenarios: + """End-to-end use case tests matching the PR documentation. + + Case 1: Normal HF checkpoint (no skip_softmax metadata in config.json) + 1a: User provides threshold_scale_factor → all layers get same threshold + 1b: User provides target_sparsity without formula → helpful error + 1c: User provides target_sparsity + formula → resolves correctly + 1d: User provides full config with layer_overrides → per-layer thresholds + + Case 2: ModelOpt checkpoint (has calibrated a, b in config.json) + 2a: User provides nothing → auto-enable from checkpoint + 2b: User provides threshold_scale_factor → user overrides checkpoint + 2c: User provides target_sparsity → uses checkpoint formula + """ + + MODELOPT_CHECKPOINT: Final = { + "sparse_attention_config": { + "config_groups": { + "group_0": { + "sparse_algo": "softmax_skip", + "targets": ["Attention"], + } + }, + "threshold_scale_factor": { + "formula": "a * exp(b * target_sparsity)", + "prefill": {"a": 7.93, "b": 8.61}, + "decode": {"a": 0.12, "b": 9.85}, + }, + "producer": {"name": "modelopt", "version": "0.37.0"}, + } + } + + # --- Case 1: Normal HF checkpoint --- + + def test_case_1a_user_threshold_only(self): + """Normal checkpoint + user threshold → works.""" + cfg = SkipSoftmaxConfig(threshold_scale_factor=5000.0) + result = cfg.resolve_threshold_scale_factor(checkpoint_formula=None) + assert result == 5000.0 + + def test_case_1b_user_target_sparsity_no_formula(self): + """Normal checkpoint + target_sparsity without formula → helpful error.""" + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + with pytest.raises(ValueError, match="calibration formula"): + cfg.resolve_threshold_scale_factor(checkpoint_formula=None) + + def test_case_1c_target_sparsity_with_attached_formula(self): + """target_sparsity + attached calibration formula (from YAML/checkpoint) → resolves. + + The user surface only carries ``target_sparsity``; the formula is + attached by the loader via :meth:`SkipSoftmaxConfig.with_calibration`. + """ + cfg = SkipSoftmaxConfig.with_calibration( + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=math.log(0.0003), b=7.5), + ) + result = cfg.resolve_threshold_scale_factor(checkpoint_formula=None) + expected = 0.0003 * math.exp(7.5 * 0.5) + assert result == pytest.approx(expected) + + def test_case_1d_layer_overrides_from_calibration(self): + """Attached layer_overrides (from ModelOpt ``disabled_layers``) → per-layer thresholds.""" + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"blocks.0*": 0, "blocks.5*": 8000.0}, + ) + assert cfg.resolve_threshold("blocks.0.attn1") is None # disabled + assert cfg.resolve_threshold("blocks.5.attn1") == 8000.0 # override + assert cfg.resolve_threshold("blocks.3.attn1") == 5000.0 # default + + # --- Case 2: ModelOpt checkpoint --- + + def test_case_2a_modelopt_checkpoint_auto_enable(self): + """ModelOpt checkpoint + no user config → auto-enable from checkpoint. + + The pipeline should detect sparse_attention_config in checkpoint + config.json and create a SkipSoftmaxConfig automatically. + """ + from tensorrt_llm.visual_gen.sparse_attention import auto_detect_sparse_attention_config + + ckpt = self.MODELOPT_CHECKPOINT + result = auto_detect_sparse_attention_config(ckpt) + assert result is not None + assert isinstance(result, SkipSoftmaxConfig) + # Calibration formula is attached as a private attr (not user-facing). + assert result._formula is not None + assert result._formula.log_a == pytest.approx(math.log(7.93)) + assert result._formula.b == pytest.approx(8.61) + + def test_case_2b_modelopt_user_threshold_overrides(self): + """ModelOpt checkpoint + user threshold → user wins.""" + cfg = SkipSoftmaxConfig(threshold_scale_factor=3000.0) + ckpt_formula = self.MODELOPT_CHECKPOINT["sparse_attention_config"][ + "threshold_scale_factor" + ]["prefill"] + result = cfg.resolve_threshold_scale_factor(checkpoint_formula=ckpt_formula) + # User threshold takes precedence, checkpoint formula ignored + assert result == 3000.0 + + def test_case_2c_modelopt_user_target_sparsity(self): + """ModelOpt checkpoint + user target_sparsity → uses checkpoint formula.""" + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + ckpt_formula = self.MODELOPT_CHECKPOINT["sparse_attention_config"][ + "threshold_scale_factor" + ]["prefill"] + result = cfg.resolve_threshold_scale_factor(checkpoint_formula=ckpt_formula) + expected = 7.93 * math.exp(8.61 * 0.5) + assert result == pytest.approx(expected) + + def test_case_2a_no_sparse_config_returns_none(self): + """Normal checkpoint (no sparse_attention_config) → returns None.""" + from tensorrt_llm.visual_gen.sparse_attention import auto_detect_sparse_attention_config + + result = auto_detect_sparse_attention_config({}) + assert result is None + + result = auto_detect_sparse_attention_config({"other_key": 123}) + assert result is None + + +# ============================================================================= +# YAML loading +# ============================================================================= + + +class TestYamlLoading: + def test_load_modelopt_yaml(self, tmp_path): + """Load from ModelOpt sparse YAML file.""" + from tensorrt_llm.visual_gen.sparse_attention import load_sparse_config_from_yaml + + yaml_content = """ +config_groups: + group_0: + sparse_algo: softmax_skip + targets: + - WanAttention + threshold_scale_factor: + formula: log_a + b * target_sparsity + prefill: + log_a: -14.14 + b: 36.64 + disabled_layers: + - blocks.0.attn1 + - blocks.0.attn2 + - blocks.39.attn2 +""" + yaml_file = tmp_path / "sparse.yaml" + yaml_file.write_text(yaml_content) + + cfg = load_sparse_config_from_yaml(str(yaml_file)) + assert cfg is not None + # Calibration formula + disabled-layer overrides land in private attrs, + # not on the user-facing surface. + assert cfg._formula.log_a == pytest.approx(-14.14) + assert cfg._formula.b == pytest.approx(36.64) + assert cfg._layer_overrides is not None + assert cfg._layer_overrides["blocks.0.attn1"] == 0 + assert cfg._layer_overrides["blocks.0.attn2"] == 0 + assert cfg._layer_overrides["blocks.39.attn2"] == 0 + assert len(cfg._layer_overrides) == 3 + + def test_load_modelopt_yaml_llm_format_a(self, tmp_path): + """Load from LLM-format YAML where prefill uses 'a' instead of 'log_a'.""" + from tensorrt_llm.visual_gen.sparse_attention import load_sparse_config_from_yaml + + yaml_content = """ +config_groups: + group_0: + sparse_algo: softmax_skip + threshold_scale_factor: + formula: a * exp(b * target_sparsity) + prefill: + a: 7.0e-5 + b: 7.929109 +""" + yaml_file = tmp_path / "sparse.yaml" + yaml_file.write_text(yaml_content) + + cfg = load_sparse_config_from_yaml(str(yaml_file)) + assert cfg is not None + # 'a' should be normalized to log_a = log(a) + assert cfg._formula.log_a == pytest.approx(math.log(7e-5)) + assert cfg._formula.b == pytest.approx(7.929109) + + def test_load_consolidated_modelopt_yaml_component_map(self, tmp_path): + """Load current ModelOpt YAML with separate component calibration.""" + from tensorrt_llm.visual_gen.sparse_attention import load_sparse_config_from_yaml + + yaml_content = """ +transformer: + config_groups: + group_0: + sparse_algo: softmax_skip + threshold_scale_factor: + formula: log_a + b * target_sparsity + prefill: + log_a: -10.0 + b: 2.0 + disabled_layers: + - blocks.0.attn1 +transformer_2: + config_groups: + group_0: + sparse_algo: softmax_skip + threshold_scale_factor: + formula: log_a + b * target_sparsity + prefill: + log_a: -20.0 + b: 4.0 + disabled_layers: + - blocks.1.attn1 +producer: + name: modelopt +""" + yaml_file = tmp_path / "sparse.yaml" + yaml_file.write_text(yaml_content) + + cfg = load_sparse_config_from_yaml(str(yaml_file)) + assert cfg is not None + assert cfg._formula is None + assert cfg._component_configs is not None + assert set(cfg._component_configs) == {"transformer", "transformer_2"} + + merged = cfg._with_public_overrides(SkipSoftmaxConfig(target_sparsity=0.5)) + transformer_threshold = math.exp(-10.0 + 2.0 * 0.5) + transformer_2_threshold = math.exp(-20.0 + 4.0 * 0.5) + assert merged.resolve_threshold("transformer.blocks.0.attn1") is None + assert merged.resolve_threshold("transformer.blocks.2.attn1") == pytest.approx( + transformer_threshold + ) + assert merged.resolve_threshold("transformer_2.blocks.1.attn1") is None + assert merged.resolve_threshold("transformer_2.blocks.2._orig_mod.attn1") == pytest.approx( + transformer_2_threshold + ) + + def test_load_yaml_no_skip_softmax(self, tmp_path): + """YAML without softmax_skip algo returns None.""" + from tensorrt_llm.visual_gen.sparse_attention import load_sparse_config_from_yaml + + yaml_content = """ +config_groups: + group_0: + sparse_algo: other_algo +""" + yaml_file = tmp_path / "sparse.yaml" + yaml_file.write_text(yaml_content) + + cfg = load_sparse_config_from_yaml(str(yaml_file)) + assert cfg is None + + def test_auto_detect_yaml(self, tmp_path): + """Auto-detect sparse YAML files in checkpoint directory.""" + from tensorrt_llm.visual_gen.sparse_attention import auto_detect_sparse_yaml + + yaml_content = """ +config_groups: + group_0: + sparse_algo: softmax_skip + threshold_scale_factor: + prefill: + log_a: -14.14 + b: 36.64 +""" + (tmp_path / "sparse.yaml").write_text(yaml_content) + + cfg = auto_detect_sparse_yaml(str(tmp_path)) + assert cfg is not None + assert cfg._formula is not None + assert cfg._formula.log_a == pytest.approx(-14.14) + + +# ============================================================================= +# resolve_threshold_scale_factor +# ============================================================================= + + +class TestResolveThresholdScaleFactor: + def test_direct_threshold_returns_immediately(self): + cfg = SkipSoftmaxConfig(threshold_scale_factor=5000.0) + assert cfg.resolve_threshold_scale_factor() == 5000.0 + + def test_direct_threshold_ignores_formula(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=math.log(0.0003), b=7.5), + ) + # threshold_scale_factor takes precedence + assert cfg.resolve_threshold_scale_factor() == 5000.0 + + def test_target_sparsity_with_attached_formula(self): + cfg = SkipSoftmaxConfig.with_calibration( + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=math.log(7e-5), b=7.929109), + ) + expected = 7e-5 * math.exp(7.929109 * 0.5) + assert cfg.resolve_threshold_scale_factor() == pytest.approx(expected) + + def test_target_sparsity_with_checkpoint_formula(self): + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + checkpoint = {"a": 7e-5, "b": 7.929109} + expected = 7e-5 * math.exp(7.929109 * 0.5) + assert cfg.resolve_threshold_scale_factor(checkpoint) == pytest.approx(expected) + + def test_attached_formula_overrides_checkpoint(self): + cfg = SkipSoftmaxConfig.with_calibration( + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=math.log(0.001), b=5.0), # attached + ) + checkpoint = {"a": 7e-5, "b": 7.929109} # checkpoint_formula arg (lower priority) + expected = 0.001 * math.exp(5.0 * 0.5) # attached formula wins + assert cfg.resolve_threshold_scale_factor(checkpoint) == pytest.approx(expected) + + def test_modelopt_checkpoint_formula_format(self): + """Test with the actual ModelOpt config.json format.""" + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + # ModelOpt format: sparse_attention_config.threshold_scale_factor.prefill + modelopt_prefill = {"a": 7.93, "b": 8.61} + expected = 7.93 * math.exp(8.61 * 0.5) + assert cfg.resolve_threshold_scale_factor(modelopt_prefill) == pytest.approx(expected) + + def test_no_threshold_no_sparsity_returns_none(self): + cfg = SkipSoftmaxConfig() + assert cfg.resolve_threshold_scale_factor() is None + + def test_target_sparsity_no_formula_raises(self): + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + with pytest.raises(ValueError, match="calibration formula"): + cfg.resolve_threshold_scale_factor() + + def test_target_sparsity_zero(self): + cfg = SkipSoftmaxConfig.with_calibration( + target_sparsity=0.0, + formula=SkipSoftmaxFormula(log_a=math.log(7e-5), b=7.929109), + ) + # exp(0) = 1, so result = a + assert cfg.resolve_threshold_scale_factor() == pytest.approx(7e-5) + + def test_target_sparsity_one(self): + cfg = SkipSoftmaxConfig.with_calibration( + target_sparsity=1.0, + formula=SkipSoftmaxFormula(log_a=math.log(7e-5), b=7.929109), + ) + expected = 7e-5 * math.exp(7.929109) + assert cfg.resolve_threshold_scale_factor() == pytest.approx(expected) + + +# ============================================================================= +# resolve_threshold (layer overrides) +# ============================================================================= + + +class TestResolveThreshold: + def test_no_overrides_returns_default(self): + cfg = SkipSoftmaxConfig(threshold_scale_factor=5000.0) + assert cfg.resolve_threshold("transformer_blocks.5.attn1") == 5000.0 + + def test_matching_override(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"transformer_blocks.0.*": 0}, + ) + assert cfg.resolve_threshold("transformer_blocks.0.attn1") is None # disabled + + def test_non_matching_returns_default(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"transformer_blocks.0.*": 0}, + ) + assert cfg.resolve_threshold("transformer_blocks.5.attn1") == 5000.0 + + def test_override_with_custom_value(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"single_transformer_blocks.*": 8000.0}, + ) + assert cfg.resolve_threshold("single_transformer_blocks.10.attn") == 8000.0 + + def test_first_match_wins(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={ + "transformer_blocks.0.*": 0, + "transformer_blocks.*": 3000.0, + }, + ) + # First pattern matches + assert cfg.resolve_threshold("transformer_blocks.0.attn1") is None + # Second pattern matches for other blocks + assert cfg.resolve_threshold("transformer_blocks.5.attn1") == 3000.0 + + def test_wildcard_patterns(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"*.attn2": 0}, # disable all cross-attention + ) + assert cfg.resolve_threshold("transformer.blocks.3.attn2") is None + assert cfg.resolve_threshold("transformer.blocks.3.attn1") == 5000.0 + + def test_modelopt_relative_patterns_match_transformer_components(self): + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"blocks.0.attn1": 0}, + ) + assert cfg.resolve_threshold("transformer.blocks.0.attn1") is None + assert cfg.resolve_threshold("transformer_2.blocks.0.attn1") is None + assert cfg.resolve_threshold("transformer.blocks.0._orig_mod.attn1") is None + assert cfg.resolve_threshold("transformer.blocks.1.attn1") == 5000.0 + + def test_no_threshold_or_target_returns_none(self): + cfg = SkipSoftmaxConfig() + assert cfg.resolve_threshold("any_layer") is None + + def test_target_sparsity_without_formula_raises(self): + cfg = SkipSoftmaxConfig(target_sparsity=0.5) + with pytest.raises(ValueError, match="calibration formula"): + cfg.get_or_resolve_threshold() + with pytest.raises(ValueError, match="calibration formula"): + cfg.resolve_threshold("any_layer") + + +# ============================================================================= +# apply_skip_softmax_overrides +# ============================================================================= + + +class TestApplySkipSoftmaxOverrides: + def _make_mock_model(self): + """Create a mock model with patched TrtllmAttention instances.""" + from unittest.mock import MagicMock + + import torch.nn as nn + + from tensorrt_llm._torch.visual_gen.attention_backend.trtllm import TrtllmAttention + + def make_mock_backend(): + mock = MagicMock(spec=TrtllmAttention) + mock.sparse_attention_config = None + return mock + + class MockAttentionModule(nn.Module): + def __init__(self): + super().__init__() + self.attn = make_mock_backend() + + class MockModel(nn.Module): + def __init__(self): + super().__init__() + self.block0 = MockAttentionModule() + self.block1 = MockAttentionModule() + self.block2 = MockAttentionModule() + + return MockModel() + + def test_no_overrides_returns_zero(self): + model = self._make_mock_model() + cfg = SkipSoftmaxConfig(threshold_scale_factor=5000.0) + assert apply_skip_softmax_overrides(model, cfg) == 0 + + def test_overrides_applied(self): + model = self._make_mock_model() + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"block0*": 0, "block2*": 8000.0}, + ) + n = apply_skip_softmax_overrides(model, cfg) + assert n == 3 + + # block0: disabled (threshold=0 → None) + assert model.block0.attn.sparse_attention_config is None + # block1: default threshold + assert model.block1.attn.sparse_attention_config is not None + assert model.block1.attn.sparse_attention_config.threshold_scale_factor_prefill == 5000.0 + # block2: overridden to 8000 + assert model.block2.attn.sparse_attention_config is not None + assert model.block2.attn.sparse_attention_config.threshold_scale_factor_prefill == 8000.0 + + def test_overrides_applied_to_compiled_module_names(self): + from unittest.mock import MagicMock + + import torch.nn as nn + + from tensorrt_llm._torch.visual_gen.attention_backend.trtllm import TrtllmAttention + + class MockAttentionModule(nn.Module): + def __init__(self): + super().__init__() + self.attn = MagicMock(spec=TrtllmAttention) + self.attn.sparse_attention_config = None + + class MockCompiledBlock(nn.Module): + def __init__(self): + super().__init__() + self._orig_mod = nn.Module() + self._orig_mod.attn1 = MockAttentionModule() + + class MockModel(nn.Module): + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([MockCompiledBlock()]) + + model = MockModel() + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"blocks.0.attn1": 0}, + ) + n = apply_skip_softmax_overrides(model, cfg) + assert n == 1 + assert model.blocks[0]._orig_mod.attn1.attn.sparse_attention_config is None + + def test_overrides_applied_to_component_relative_names(self): + from unittest.mock import MagicMock + + import torch.nn as nn + + from tensorrt_llm._torch.visual_gen.attention_backend.trtllm import TrtllmAttention + + class MockAttentionModule(nn.Module): + def __init__(self): + super().__init__() + self.attn = MagicMock(spec=TrtllmAttention) + self.attn.sparse_attention_config = None + + class MockCompiledBlock(nn.Module): + def __init__(self): + super().__init__() + self._orig_mod = nn.Module() + self._orig_mod.attn1 = MockAttentionModule() + + class MockTransformer(nn.Module): + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([MockCompiledBlock()]) + + class MockPipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = MockTransformer() + self.transformer_2 = MockTransformer() + + model = MockPipeline() + cfg = SkipSoftmaxConfig.with_calibration( + threshold_scale_factor=5000.0, + layer_overrides={"blocks.0.attn1": 0}, + ) + n = apply_skip_softmax_overrides(model, cfg) + assert n == 2 + assert model.transformer.blocks[0]._orig_mod.attn1.attn.sparse_attention_config is None + assert model.transformer_2.blocks[0]._orig_mod.attn1.attn.sparse_attention_config is None + + def test_component_configs_apply_distinct_thresholds(self): + from unittest.mock import MagicMock + + import torch.nn as nn + + from tensorrt_llm._torch.visual_gen.attention_backend.trtllm import TrtllmAttention + + class MockAttentionModule(nn.Module): + def __init__(self): + super().__init__() + self.attn = MagicMock(spec=TrtllmAttention) + self.attn.sparse_attention_config = None + + class MockTransformer(nn.Module): + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([MockAttentionModule()]) + + class MockPipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = MockTransformer() + self.transformer_2 = MockTransformer() + + model = MockPipeline() + cfg = SkipSoftmaxConfig.with_calibration( + component_configs={ + "transformer": SkipSoftmaxConfig.with_calibration( + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=-10.0, b=2.0), + ), + "transformer_2": SkipSoftmaxConfig.with_calibration( + target_sparsity=0.5, + formula=SkipSoftmaxFormula(log_a=-20.0, b=4.0), + ), + } + ) + n = apply_skip_softmax_overrides(model, cfg) + assert n == 2 + assert model.transformer.blocks[ + 0 + ].attn.sparse_attention_config.threshold_scale_factor_prefill == pytest.approx( + math.exp(-10.0 + 2.0 * 0.5) + ) + assert model.transformer_2.blocks[ + 0 + ].attn.sparse_attention_config.threshold_scale_factor_prefill == pytest.approx( + math.exp(-20.0 + 4.0 * 0.5) + ) From 90694fec72a434ddc2b23740dce4dc9e821a2b0a Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Mon, 25 May 2026 12:17:39 +0800 Subject: [PATCH 002/308] [https://nvbugs/6190759][fix] set env on some gb300 cluster (#14460) Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- .../defs/disaggregated/test_auto_scaling.py | 123 +++++++++++++----- .../defs/disaggregated/test_disaggregated.py | 3 +- tests/integration/test_lists/waives.txt | 17 --- .../apps/_test_disagg_serving_multi_nodes.py | 12 +- 4 files changed, 107 insertions(+), 48 deletions(-) diff --git a/tests/integration/defs/disaggregated/test_auto_scaling.py b/tests/integration/defs/disaggregated/test_auto_scaling.py index 3a1873d8be34..efa0570e44f9 100644 --- a/tests/integration/defs/disaggregated/test_auto_scaling.py +++ b/tests/integration/defs/disaggregated/test_auto_scaling.py @@ -1,9 +1,10 @@ import asyncio import os +import platform import pytest import requests -from defs.conftest import llm_models_root +from defs.conftest import get_sm_version, llm_models_root from disagg_test_utils import (CHECK_STATUS_INTERVAL, request_completion, run_ctx_worker, run_disagg_server, run_gen_worker, terminate, verify_cluster_info, @@ -16,6 +17,29 @@ ROUTER_TYPES = ["round_robin", "load_balancing", "kv_cache_aware"] +def get_ucx_tls(): + """Get UCX_TLS value based on GPU architecture. + + Pre-Hopper GPUs need cuda_ipc excluded from UCX transports. + On some gb300 cluster, we need to set `cuda_copy,cuda_ipc,sm,self,tcp` + explicitly to avoid UCX auto-selection picking a misbehaving NIC transport. + """ + sm = get_sm_version() + if sm == 103 and "aarch" in platform.machine().lower(): + return "cuda_copy,cuda_ipc,sm,self,tcp" + if sm < 90: + return "^cuda_ipc,ib,gdr_copy" + return "^ib,gdr_copy" + + +@pytest.fixture +def worker_env(): + """Per-worker environment with UCX_TLS set for the current GPU arch.""" + env = os.environ.copy() + env["UCX_TLS"] = get_ucx_tls() + return env + + @pytest.fixture def model_name(): model_path = os.path.join(llm_models_root(), @@ -31,16 +55,24 @@ def model_name(): @pytest.mark.parametrize("service_discovery", ["etcd", "http"], indirect=True) async def test_service_discovery(model_name, disagg_server_config, worker_config, router, service_discovery, - disagg_port, work_dir): + disagg_port, work_dir, worker_env): ctx_worker1 = None gen_worker1 = None disagg_server = None try: # initial cluster, 1 ctx, 1 gen, request should succeed - ctx_worker1 = run_ctx_worker(model_name, worker_config, work_dir) - gen_worker1 = run_gen_worker(model_name, worker_config, work_dir) - disagg_server = run_disagg_server(disagg_server_config, work_dir, - disagg_port) + ctx_worker1 = run_ctx_worker(model_name, + worker_config, + work_dir, + env=worker_env) + gen_worker1 = run_gen_worker(model_name, + worker_config, + work_dir, + env=worker_env) + disagg_server = run_disagg_server(disagg_server_config, + work_dir, + disagg_port, + env=worker_env) await wait_for_disagg_server_ready(disagg_port) verify_cluster_info(True, 1, 1, port=disagg_port) response = request_completion(model_name, @@ -60,7 +92,7 @@ async def test_service_discovery(model_name, disagg_server_config, @pytest.mark.timeout(900) async def test_minimal_instances(model_name, disagg_server_config, worker_config, router, service_discovery, - disagg_port, work_dir): + disagg_port, work_dir, worker_env): # the cluster should have at least 2 ctx and 2 gen workers minimal_instances = { "context_servers": 2, @@ -76,10 +108,18 @@ async def test_minimal_instances(model_name, disagg_server_config, gen_worker2 = None disagg_server = None try: - ctx_worker1 = run_ctx_worker(model_name, worker_config, work_dir) - gen_worker1 = run_gen_worker(model_name, worker_config, work_dir) - disagg_server = run_disagg_server(disagg_server_config, work_dir, - disagg_port) + ctx_worker1 = run_ctx_worker(model_name, + worker_config, + work_dir, + env=worker_env) + gen_worker1 = run_gen_worker(model_name, + worker_config, + work_dir, + env=worker_env) + disagg_server = run_disagg_server(disagg_server_config, + work_dir, + disagg_port, + env=worker_env) await wait_for_disagg_server_status(disagg_port, 1, 1) verify_cluster_info(False, 1, 1, port=disagg_port) # with only 1 ctx and 1 gen worker, the request should fail @@ -89,8 +129,14 @@ async def test_minimal_instances(model_name, disagg_server_config, port=disagg_port) print(response) - ctx_worker2 = run_ctx_worker(model_name, worker_config, work_dir) - gen_worker2 = run_gen_worker(model_name, worker_config, work_dir) + ctx_worker2 = run_ctx_worker(model_name, + worker_config, + work_dir, + env=worker_env) + gen_worker2 = run_gen_worker(model_name, + worker_config, + work_dir, + env=worker_env) await wait_for_disagg_server_ready(disagg_port) verify_cluster_info(True, 2, 2, port=disagg_port) response = request_completion(model_name, @@ -108,7 +154,8 @@ async def test_minimal_instances(model_name, disagg_server_config, @pytest.mark.asyncio(loop_scope="module") @pytest.mark.timeout(900) async def test_worker_restart(model_name, disagg_server_config, worker_config, - router, service_discovery, disagg_port, work_dir): + router, service_discovery, disagg_port, work_dir, + worker_env): ctx_worker1 = None ctx_worker2 = None gen_worker1 = None @@ -120,13 +167,17 @@ async def test_worker_restart(model_name, disagg_server_config, worker_config, ctx_worker1 = run_ctx_worker(model_name, worker_config, work_dir, - device=0) + device=0, + env=worker_env) gen_worker1 = run_gen_worker(model_name, worker_config, work_dir, - device=1) - disagg_server = run_disagg_server(disagg_server_config, work_dir, - disagg_port) + device=1, + env=worker_env) + disagg_server = run_disagg_server(disagg_server_config, + work_dir, + disagg_port, + env=worker_env) await wait_for_disagg_server_ready(disagg_port) verify_cluster_info(True, 1, 1, port=disagg_port) response = request_completion(model_name, @@ -149,7 +200,8 @@ async def test_worker_restart(model_name, disagg_server_config, worker_config, worker_config, work_dir, port=0, - device=1) + device=1, + env=worker_env) await wait_for_disagg_server_status(disagg_port, 1, 1) await asyncio.sleep(CHECK_STATUS_INTERVAL) verify_cluster_info(True, 1, 1, port=disagg_port) @@ -171,7 +223,8 @@ async def test_worker_restart(model_name, disagg_server_config, worker_config, worker_config, work_dir, port=0, - device=0) + device=0, + env=worker_env) await wait_for_disagg_server_status(disagg_port, 1, 1) await asyncio.sleep(CHECK_STATUS_INTERVAL) verify_cluster_info(True, 1, 1, port=disagg_port) @@ -185,12 +238,14 @@ async def test_worker_restart(model_name, disagg_server_config, worker_config, worker_config, work_dir, port=0, - device=0) + device=0, + env=worker_env) gen_worker1 = run_gen_worker(model_name, worker_config, work_dir, port=0, - device=1) + device=1, + env=worker_env) await wait_for_disagg_server_status(disagg_port, 2, 2) await asyncio.sleep(CHECK_STATUS_INTERVAL) verify_cluster_info(True, 2, 2, port=disagg_port) @@ -214,16 +269,24 @@ async def test_worker_restart(model_name, disagg_server_config, worker_config, @pytest.mark.timeout(900) async def test_disagg_server_restart(model_name, disagg_server_config, worker_config, router, service_discovery, - disagg_port, work_dir): + disagg_port, work_dir, worker_env): ctx_worker1 = None gen_worker1 = None disagg_server = None try: # initial cluster, 1 ctx, 1 gen, request should succeed - ctx_worker1 = run_ctx_worker(model_name, worker_config, work_dir) - gen_worker1 = run_gen_worker(model_name, worker_config, work_dir) - disagg_server = run_disagg_server(disagg_server_config, work_dir, - disagg_port) + ctx_worker1 = run_ctx_worker(model_name, + worker_config, + work_dir, + env=worker_env) + gen_worker1 = run_gen_worker(model_name, + worker_config, + work_dir, + env=worker_env) + disagg_server = run_disagg_server(disagg_server_config, + work_dir, + disagg_port, + env=worker_env) await wait_for_disagg_server_ready(disagg_port) verify_cluster_info(True, 1, 1, port=disagg_port) response = request_completion(model_name, @@ -246,8 +309,10 @@ async def test_disagg_server_restart(model_name, disagg_server_config, expected_code=500) # restart disagg server, the request should succeed - disagg_server = run_disagg_server(disagg_server_config, work_dir, - disagg_port) + disagg_server = run_disagg_server(disagg_server_config, + work_dir, + disagg_port, + env=worker_env) await wait_for_disagg_server_ready(disagg_port) verify_cluster_info(True, 1, 1, port=disagg_port) response = request_completion(model_name, diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 1fd711ef751e..81b36db52e76 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -16,6 +16,7 @@ import asyncio import json import os +import platform import re import shutil import subprocess @@ -65,7 +66,7 @@ def get_ucx_tls(): """ ON some gb300 cluster, we need to set `cuda_copy,cuda_ipc,sm,self,tcp` for UCX_TLS """ - if sm == 103: + if sm == 103 and "aarch" in platform.machine().lower(): return "cuda_copy,cuda_ipc,sm,self,tcp" if sm < 90: return "^cuda_ipc,ib,gdr_copy" diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6f895057854d..fb8bb7c280b0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -161,26 +161,9 @@ cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-mpi_kvcache cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] SKIP (https://nvbugs/6093820) cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] SKIP (https://nvbugs/6093820) cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (https://nvbugs/5838199) -disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-kv_cache_aware] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing] SKIP (https://nvbugs/6094100) -disaggregated/test_auto_scaling.py::test_worker_restart[http-round_robin] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114612) disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_gentp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114140) diff --git a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py index c14027434607..20f7ef29eea6 100644 --- a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py +++ b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py @@ -1,4 +1,5 @@ import os +import platform import socket import time @@ -9,6 +10,8 @@ from test_common.perf_metrics_utils import (get_timing_metrics, validate_timing_metrics) +from tensorrt_llm._utils import get_sm_version + from ..test_llm import get_model_path from .openai_server import RemoteDisaggOpenAIServer, RemoteOpenAIServer from .utils import (expand_slurm_nodelist, wait_for_endpoint_down, @@ -66,12 +69,19 @@ def is_pytest_node(): def env(): # Remove MPI related environment variables to isolate the ctx/gen processes # so that they will not be in the same MPI communicator, otherwise the rank and world_size may mismatch - return { + e = { k: v for k, v in os.environ.items() if not ('PMI_' in k or 'OMPI_' in k or 'PMIX_' in k or 'SLURM_' in k) and k not in ["UCX_TLS", "UCX_NET_DEVICES"] # avoid UCX failure on oci } + # Some GB300 machines have NICs that misbehave with UCX's default transport + # auto-selection, so UCX_TLS must be set explicitly there. Identify GB300 via + # sm_103 (Blackwell Ultra) + aarch64 (Grace) -- this excludes HGX B300 + # (also sm_103 but x86_64) and avoids depending on the GPU device name string. + if get_sm_version() == 103 and platform.machine().lower() == "aarch64": + e["UCX_TLS"] = "cuda_copy,cuda_ipc,sm,self,tcp" + return e @pytest.fixture(scope="module") From 72bc8ed5f0b71a61ee92d7f57edbce50656bc903 Mon Sep 17 00:00:00 2001 From: tcherckez-nvidia <127761168+tcherckez-nvidia@users.noreply.github.com> Date: Mon, 25 May 2026 08:04:14 +0300 Subject: [PATCH 003/308] [https://nvbugs/6157131][fix] lower the GSM8K accuracy grade for Nano V3 (#14078) Signed-off-by: Tal Cherckez --- tests/integration/defs/accuracy/references/gsm8k.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index a46c91e82678..c7cdaddc297e 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -474,7 +474,7 @@ nvidia/Nemotron-3-Nano: accuracy: 68.73 - quant_algo: NVFP4 kv_cache_quant_algo: FP8 - accuracy: 67.286 + accuracy: 65.428 MiniMaxAI/MiniMax-M2: - accuracy: 93.75 - quant_algo: FP8_BLOCK_SCALES From bdc8b6420aa0c105885844eb10a0605994cb5355 Mon Sep 17 00:00:00 2001 From: Yao Yao Date: Mon, 25 May 2026 13:15:03 +0800 Subject: [PATCH 004/308] [None][fix] Fix KV cache grain slot refinement (#14442) Signed-off-by: Yao Yao --- .../kv_cache_manager_v2/_storage/_core.py | 23 +++++++++-- .../test_kv_cache_manager_v2.py | 39 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage/_core.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage/_core.py index bde42e7d53ec..1db403131ba8 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage/_core.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage/_core.py @@ -755,7 +755,7 @@ def _grains_to_slots( slot_size_list: TypedIndexList[PoolIndex, int], granularity: int, ) -> tuple[int, int]: - """Distribute grains among pools within a pool group. + """Compute the maximum slots that fit in a pool group grain budget. Returns (num_slots, grains_consumed). """ @@ -779,9 +779,24 @@ def _grains_to_slots( remaining_pg_grains -= pool_grains assert remaining_pg_grains == 0 assert num_slots > 0 - return num_slots, CacheLevelStorage._grains_for_slots( - num_slots, slot_size_list, granularity - ) + _s2g = CacheLevelStorage._grains_for_slots + lo = num_slots + step = 1 + hi = lo + step + while _s2g(hi, slot_size_list, granularity) <= pg_grains: + lo = hi + step *= 2 + hi = lo + step + while lo + 1 < hi: + mid = (lo + hi) // 2 + if _s2g(mid, slot_size_list, granularity) <= pg_grains: + lo = mid + else: + hi = mid + used = _s2g(lo, slot_size_list, granularity) + assert used <= pg_grains + assert _s2g(lo + 1, slot_size_list, granularity) > pg_grains + return lo, used @staticmethod def _grains_for_slots( diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py index 1d6a26b9c147..65ccda597d36 100755 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py @@ -65,6 +65,7 @@ from kv_cache_manager_v2._copy_engine import CopyTask, batched_copy from kv_cache_manager_v2._exceptions import OutOfPagesError from kv_cache_manager_v2._life_cycle_registry import SsmLifeCycle + from kv_cache_manager_v2._storage._core import CacheLevelStorage from kv_cache_manager_v2._utils import ( CachedCudaStream, HalfOpenRange, @@ -117,6 +118,7 @@ from tensorrt_llm.runtime.kv_cache_manager_v2._copy_engine import CopyTask, batched_copy from tensorrt_llm.runtime.kv_cache_manager_v2._exceptions import OutOfPagesError from tensorrt_llm.runtime.kv_cache_manager_v2._life_cycle_registry import SsmLifeCycle + from tensorrt_llm.runtime.kv_cache_manager_v2._storage._core import CacheLevelStorage from tensorrt_llm.runtime.kv_cache_manager_v2._utils import ( CachedCudaStream, HalfOpenRange, @@ -189,6 +191,43 @@ def wrapper(self, *args, **kwargs): return wrapper +class TestCacheLevelStorage(unittest.TestCase): + def test_grains_to_slots_refines_proportional_lower_bound(self) -> None: + granularity = 16 << 20 + slot_size_list = [16_252_928, 4_063_232] + min_slots = 157 + + grains = CacheLevelStorage._grains_for_slots(min_slots, slot_size_list, granularity) + slots, used = CacheLevelStorage._grains_to_slots(grains, slot_size_list, granularity) + + self.assertGreaterEqual(slots, min_slots) + self.assertLessEqual(used, grains) + + def test_ratio_to_slot_count_list_preserves_min_slots(self) -> None: + granularity = 16 << 20 + slot_size_lists = [ + [16_252_928, 4_063_232], + [491_520, 126_720, 15_872], + [31_457_280, 7_864_320], + ] + min_slots = [157, 3907, 157] + total_min_grains = sum( + CacheLevelStorage._grains_for_slots(slots, sizes, granularity) + for slots, sizes in zip(min_slots, slot_size_lists) + ) + + slot_counts = CacheLevelStorage.ratio_to_slot_count_list( + total_min_grains * granularity, + slot_size_lists, + [0.2, 0.5, 0.3], + granularity, + min_slots, + ) + + for slot_count, min_slot in zip(slot_counts, min_slots): + self.assertGreaterEqual(slot_count, min_slot) + + def create_config( tokens_per_block: int, gpu_quota: int, From 8e691e89b73c101a0cc85c881ec8509437f0433d Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 25 May 2026 13:48:16 +0800 Subject: [PATCH 005/308] [None][test] Waive 7 failed cases for main in QA CI (#14504) Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index fb8bb7c280b0..f85d8ac7fde5 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -94,6 +94,7 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) +accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[8gpus] SKIP (https://nvbugs/6162328) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) @@ -142,8 +143,13 @@ accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_on-trtllm] SKIP (https://nvbugs/6094068) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] SKIP (https://nvbugs/6177390) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False] SKIP (https://nvbugs/6212252) accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True] SKIP (https://nvbugs/6210714) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_9B::test_bf16[mtp_off] SKIP (https://nvbugs/6212250) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_9B::test_bf16[mtp_on] SKIP (https://nvbugs/6212250) accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response SKIP (https://nvbugs/6200128) +accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6211189) +accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6189416) accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] SKIP (https://nvbugs/6182617) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) From ce788e02086a37f069a744e22eca95c53db2f584 Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 25 May 2026 13:52:33 +0800 Subject: [PATCH 006/308] [None][test] Waive 1 failed cases for main in QA CI (#14503) Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f85d8ac7fde5..d0805c29e652 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -82,6 +82,7 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughpu accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5772993) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[two_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) From e6d4f9f2f910ddad3ff68cd07ed9791c5e87241a Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 25 May 2026 14:30:30 +0800 Subject: [PATCH 007/308] [None][infra] Waive 10 failed cases for main in post-merge 2733 (#14514) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d0805c29e652..27b22bb1b4f1 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -18,6 +18,7 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-t accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-4-trtllm] SKIP (https://nvbugs/6120981) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) +accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) @@ -83,6 +84,7 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKI accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] SKIP (https://nvbugs/6215702) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[two_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) @@ -241,6 +243,11 @@ examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-rec examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5214221) examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5214221) examples/test_recurrentgemma.py::test_llm_recurrentgemma_2gpu[recurrentgemma-2b] SKIP (https://nvbugs/5401233) +examples/test_visual_gen.py::test_flux1_lpips_against_golden SKIP (https://nvbugs/6215688) +examples/test_visual_gen.py::test_flux2_lpips_against_golden SKIP (https://nvbugs/6215688) +examples/test_visual_gen.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6215688) +examples/test_visual_gen.py::test_wan21_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) +examples/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) full:A10/unittest/kv_cache_manager_v2_tests/ SKIP (https://nvbugs/5841954) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6185480) @@ -339,6 +346,9 @@ perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14 stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) +stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) +stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) +test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6114608) From 5f4946e9125043ed7c10933be914207162cd12f0 Mon Sep 17 00:00:00 2001 From: Junpan Wu <77491544+shikicloud@users.noreply.github.com> Date: Mon, 25 May 2026 14:52:56 +0800 Subject: [PATCH 008/308] [https://nvbugs/6094070][fix] Skip ray-marked integration tests when --run-ray is not set (#14226) Signed-off-by: Shiki Wu --- tests/integration/defs/conftest.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index ff92afdc149a..007998ee9b41 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -2260,6 +2260,14 @@ def pytest_collection_modifyitems(session, config, items): if waives_file: apply_waives(waives_file, items, config) + # Skip tests with pytest.mark.ray unless --run-ray is passed. + if not config.getoption("--run-ray"): + skip_marker = pytest.mark.skip( + reason="Ray tests skipped; pass --run-ray to enable") + for item in items: + if item.get_closest_marker("ray") is not None: + item.add_marker(skip_marker) + # We have to remove prefix temporarily before splitting the test list # After that change back the test id. for item in items: From 8c8765d69c9b2ad6241098f5b824efc9e0c010ad Mon Sep 17 00:00:00 2001 From: xingfei xi <95731198+xxi-nv@users.noreply.github.com> Date: Mon, 25 May 2026 15:03:00 +0800 Subject: [PATCH 009/308] [TRTLLM-12635][feat] add bench_moe microbenchmark (#14507) Signed-off-by: xxi --- .../bench_moe/BENCH_MOE_USER_GUIDE.md | 788 ++++++++++++++++ tests/microbenchmarks/bench_moe/__init__.py | 67 ++ tests/microbenchmarks/bench_moe/__main__.py | 65 ++ tests/microbenchmarks/bench_moe/backend.py | 125 +++ tests/microbenchmarks/bench_moe/build.py | 209 +++++ .../microbenchmarks/bench_moe/case_runner.py | 857 ++++++++++++++++++ tests/microbenchmarks/bench_moe/cli.py | 745 +++++++++++++++ tests/microbenchmarks/bench_moe/mapping.py | 185 ++++ tests/microbenchmarks/bench_moe/quantize.py | 61 ++ tests/microbenchmarks/bench_moe/reporting.py | 765 ++++++++++++++++ tests/microbenchmarks/bench_moe/results.py | 445 +++++++++ .../bench_moe/routing/__init__.py | 111 +++ .../bench_moe/routing/builders.py | 365 ++++++++ .../bench_moe/routing/materialize.py | 239 +++++ .../bench_moe/routing/native_logits.py | 289 ++++++ .../bench_moe/routing/parsing.py | 115 +++ tests/microbenchmarks/bench_moe/search.py | 394 ++++++++ tests/microbenchmarks/bench_moe/specs.py | 356 ++++++++ .../bench_moe/timing/__init__.py | 58 ++ .../bench_moe/timing/autotune.py | 65 ++ .../bench_moe/timing/cuda_graph.py | 129 +++ .../microbenchmarks/bench_moe/timing/cupti.py | 167 ++++ .../microbenchmarks/bench_moe/timing/eager.py | 182 ++++ tests/microbenchmarks/bench_moe/utils.py | 186 ++++ tests/microbenchmarks/bench_moe/worker.py | 687 ++++++++++++++ 25 files changed, 7655 insertions(+) create mode 100644 tests/microbenchmarks/bench_moe/BENCH_MOE_USER_GUIDE.md create mode 100644 tests/microbenchmarks/bench_moe/__init__.py create mode 100644 tests/microbenchmarks/bench_moe/__main__.py create mode 100644 tests/microbenchmarks/bench_moe/backend.py create mode 100644 tests/microbenchmarks/bench_moe/build.py create mode 100644 tests/microbenchmarks/bench_moe/case_runner.py create mode 100644 tests/microbenchmarks/bench_moe/cli.py create mode 100644 tests/microbenchmarks/bench_moe/mapping.py create mode 100644 tests/microbenchmarks/bench_moe/quantize.py create mode 100644 tests/microbenchmarks/bench_moe/reporting.py create mode 100644 tests/microbenchmarks/bench_moe/results.py create mode 100644 tests/microbenchmarks/bench_moe/routing/__init__.py create mode 100644 tests/microbenchmarks/bench_moe/routing/builders.py create mode 100644 tests/microbenchmarks/bench_moe/routing/materialize.py create mode 100644 tests/microbenchmarks/bench_moe/routing/native_logits.py create mode 100644 tests/microbenchmarks/bench_moe/routing/parsing.py create mode 100644 tests/microbenchmarks/bench_moe/search.py create mode 100644 tests/microbenchmarks/bench_moe/specs.py create mode 100644 tests/microbenchmarks/bench_moe/timing/__init__.py create mode 100644 tests/microbenchmarks/bench_moe/timing/autotune.py create mode 100644 tests/microbenchmarks/bench_moe/timing/cuda_graph.py create mode 100644 tests/microbenchmarks/bench_moe/timing/cupti.py create mode 100644 tests/microbenchmarks/bench_moe/timing/eager.py create mode 100644 tests/microbenchmarks/bench_moe/utils.py create mode 100644 tests/microbenchmarks/bench_moe/worker.py diff --git a/tests/microbenchmarks/bench_moe/BENCH_MOE_USER_GUIDE.md b/tests/microbenchmarks/bench_moe/BENCH_MOE_USER_GUIDE.md new file mode 100644 index 000000000000..a1e8e00ba852 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/BENCH_MOE_USER_GUIDE.md @@ -0,0 +1,788 @@ + + + +# bench_moe User Guide + +`bench_moe` is the TensorRT-LLM MoE microbenchmark under +`tests/microbenchmarks/bench_moe`. It times `ConfigurableMoE.forward` directly +with synthetic inputs, so you can search a user-defined configuration space and +find the best MoE setup for a fixed model shape and token workload. The search +space can cover backends, communication methods, parallel layouts, routing +shapes, and CUDA Graph settings without loading a HuggingFace checkpoint. +Advanced users can also control source-to-target communication volume and hot +workloads on each local expert. Current kernel analysis records CUDA kernel +statistics and raw samples when CUPTI is available; a planned per-forward +breakdown mode will make the time spent by each kernel in every MoE forward pass +directly visible. + +The preferred invocation style is: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} python3 -m bench_moe ... +``` + +When running from a script that already exports `PYTHONPATH` with +`$PWD/tests/microbenchmarks`, use `python3 -m bench_moe ...` directly. + +## Quick Model + +1. Choose the workload. + + Pick a built-in `--model` or pass explicit shape fields, then set the token + workload with `--balanced_total_num_tokens`. For source-rank skew, use + `--per_rank_num_tokens`. + +2. Choose the search space. + + Use `--search backend`, `--search comm`, `--search parallel`, or + combinations such as `--search backend comm`. Multi-value `--backend`, + `--comm_method`, and `--parallel_mode` flags implicitly enable the matching + search axis. Use `--max_configs` or `--time_budget_minutes` when the + Cartesian product is too large. + +3. Inspect winners and skips. + + The JSON report contains `rankings` grouped by `(num_tokens, parallel_mode)`. + Unsupported or pruned candidates are emitted as `status="skipped"` with a + `skip_reason`. + +## Launch Rules + +For single-rank sanity checks: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} python3 -m bench_moe \ + --world_size 1 \ + --model mixtral_8x7b \ + --backend CUTLASS \ + --balanced_total_num_tokens 8 \ + --no_cuda_graph \ + --analysis none +``` + +For multi-rank runs, prefer an external launcher: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + ... +``` + +`--world_size` must match the number of ranks started by external `mpirun` or +`srun`. Avoid bare `python3 -m bench_moe --world_size N` on OCI / Slurm / Pyxis +systems because that path uses `MPI.COMM_SELF.Spawn`, which is commonly disabled +there. `--word_size` is a typo; the valid option is `--world_size`. + +Do not wrap `bench_moe` with `trtllm-llmapi-launch`. That launcher runs the user +command on rank 0 and starts MGMN workers on the other ranks, which is correct +for LLM API / serving workloads but not for `bench_moe`. `bench_moe` requires +every MPI rank to execute the benchmark worker. + +## Common Cases + +### Case A: single-GPU sanity check + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} python3 -m bench_moe \ + --world_size 1 \ + --model mixtral_8x7b \ + --backend CUTLASS \ + --balanced_total_num_tokens 8 \ + --no_cuda_graph \ + --analysis none +``` + +Use this to validate imports, CUDA availability, backend construction, and basic +timing. + +### Case B: 4-GPU DEP, search all backends + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --parallel_mode DEP \ + --model deepseek_v3 \ + --search backend \ + --balanced_total_num_tokens 64 128 256 512 \ + --output_file out/deepseek_v3_backend_search.json +``` + +`--search backend` without `--backend` expands to all backends. `--backend ALL` +is equivalent. + +### Case C: fixed backend, search forced communication methods + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode DEP \ + --backend TRTLLM \ + --search comm \ + --balanced_total_num_tokens 256 \ + --output_file out/deepseek_v3_comm_search.json +``` + +`--search comm` compares concrete forced communication strategies. It +intentionally excludes `AUTO`, because `AUTO` is an alias resolved by +TensorRT-LLM at runtime. To measure `AUTO`, run a separate single-candidate case +with `--comm_method AUTO`. + +### Case D: search backend and forced communication together + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode DEP \ + --search backend comm \ + --balanced_total_num_tokens 64 128 256 \ + --output_file out/deepseek_v3_backend_comm_full.json +``` + +To limit the Cartesian product, pass subsets such as: + +```bash +--backend CUTLASS DEEPGEMM --comm_method NVLINK_ONE_SIDED DEEPEP +``` + +### Case E: receiver hotspot + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --parallel_mode DEP \ + --model deepseek_v3 \ + --backend TRTLLM \ + --balanced_total_num_tokens 256 \ + --comm_pattern receiver_hotspot,hotness=0.75,rank=0 \ + --output_file out/recv_hotspot.json +``` + +This sends 75% of selected slots to rank 0 and studies receiver-side pressure. + +### Case F: source-rank token skew + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode DEP \ + --backend TRTLLM \ + --balanced_total_num_tokens 896 \ + --per_rank_num_tokens 128 128 512 128 \ + --routing_dump_matrix \ + --output_file out/per_rank_skew.json +``` + +Use this to study max tokens per rank, padding, workspace use, and chunking when +one source rank has many more tokens. + +### Case G: local-only communication baseline + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode DEP \ + --backend TRTLLM \ + --balanced_total_num_tokens 256 \ + --routing_mode forced \ + --comm_pattern local_only \ + --routing_dump_matrix \ + --output_file out/local_only_forced.json +``` + +This forces every selected slot to stay on its source rank. It estimates the +near-zero cross-rank traffic lower bound, but it is a supplied-topk path and is +not directly equivalent to native fused routing. + +### Case H: routing pattern file for arbitrary trace shapes + +Use `--routing_pattern_file` when a trace shape cannot be represented by the +built-in `--comm_pattern` / `--expert_pattern` templates. Production routing is +often irregular: different source ranks may prefer different target ranks, and +each target rank may have a different local-expert hotspot. The CLI templates +are intentionally compact and mostly symmetric; a pattern file is the escape +hatch for exact trace replay. + +```json +{ + "ep_size": 4, + "experts_per_rank": 4, + "slot_dispatch_matrix": [ + [5, 0, 1, 2], + [1, 5, 1, 1], + [0, 2, 4, 2], + [2, 1, 1, 4] + ], + "expert_histogram": [ + [5, 1, 1, 1], + [0, 6, 1, 1], + [0, 0, 7, 0], + [2, 2, 1, 4] + ] +} +``` + +`slot_dispatch_matrix[src][dst]` is the number of selected slots sent from +source rank `src` to target rank `dst`. Each row sum must equal +`per_rank_num_tokens[src] * top_k`. `expert_histogram[dst][local_expert]` +describes how the slots received by target rank `dst` are distributed across +its local experts. + +Key invariants: + +- `slot_dispatch_matrix` has shape `[ep_size][ep_size]`. Each cell counts slots, + where one `(token, selected_expert)` pair is one slot. +- `sum(slot_dispatch_matrix[src]) == per_rank_num_tokens[src] * top_k` for every + source rank. The benchmark validates this during parsing. +- `sum(expert_histogram) == sum(per_rank_num_tokens) * top_k`. The benchmark + also validates this global total. +- For an exact realization, each `expert_histogram[dst]` row sum should match + the corresponding dispatch column sum. If the row sum does not match, the + materializer treats the row as weights and `routing_control.actual` may report + `max_abs_slot_error > 0`. + +In the example above, `slot_dispatch_matrix[0] = [5, 0, 1, 2]` means source rank +0 sends five slots to itself, none to rank 1, one to rank 2, and two to rank 3. +`expert_histogram[2] = [0, 0, 7, 0]` is an extreme compute hotspot: all slots +received by target rank 2 land on local expert 2. + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --parallel_mode DEP \ + --num_experts 16 \ + --top_k 2 \ + --hidden_size 1024 \ + --intermediate_size 4096 \ + --quant FP8 \ + --routing_method RENORMALIZE \ + --backend CUTLASS \ + --balanced_total_num_tokens 16 \ + --routing_mode forced \ + --routing_pattern_file advanced_routing_pattern.json \ + --routing_dump_matrix \ + --output_file out/advanced_routing_pattern.json +``` + +To verify the run, compare `routing_control.actual.observed_slot_dispatch_matrix` +and `routing_control.actual.observed_expert_histogram` in the output JSON with +the matrices in the input file. In `--routing_mode forced`, the supplied-topk +path should reproduce them exactly. In `--routing_mode native`, check +`routing_realization.status`, `max_abs_slot_error`, and `max_relative_slot_error` +to see how closely the model-native routing path matched the requested shape. + +### Case I: JSON config for a dashboard sweep + +```json +{ + "model": "deepseek_v3", + "workload": { + "balanced_total_num_tokens": [64, 128, 256, 512], + "routing_control": { + "comm_pattern": "receiver_hotspot,hotness=0.75,rank=0", + "expert_pattern": "balanced", + "routing_dump_matrix": false, + "seed": 42 + } + }, + "search": { + "backend": ["CUTLASS", "TRTLLM", "DEEPGEMM"], + "parallel_mode": ["DEP", "TEP"], + "comm_method": ["NVLINK_ONE_SIDED", "DEEPEP"] + }, + "analysis": ["kernels"], + "output_file": "out/deepseek_v3_dashboard.json" +} +``` + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --config_file configs/deepseek_v3_dashboard.json +``` + +The config file is useful when sharing a fixed customer or dashboard sweep. Its +`search` block accepts `backend`, `parallel_mode`, and `comm_method`; multi-value +lists enable the matching search axis. Other runtime knobs belong at top level +or under `workload` / `routing_control`. If the same field is set in the config +and on the CLI, the explicit CLI value wins. + +### Case J: 2-node Slurm, world_size 8 + +This form starts eight ranks across two nodes, four GPUs per node. Use it as the +benchmark command inside your own Slurm allocation or batch script. The +environment should already have TensorRT-LLM installed and should run from the +repository root. Do not use internal spawn, and do not wrap the benchmark with +`trtllm-llmapi-launch`. + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +srun --mpi=pmix --nodes=2 --ntasks=8 --ntasks-per-node=4 --gres=gpu:4 \ + python3 -m bench_moe \ + --world_size 8 \ + --model deepseek_v4_flash \ + --quant W4A8_MXFP4_MXFP8 \ + --balanced_total_num_tokens 1024 \ + --backend MEGAMOE_DEEPGEMM \ + --parallel_mode DEP \ + --comm_method AUTO \ + --warmup 1 \ + --iters 2 \ + --analysis none \ + --comm_pattern balanced_alltoall \ + --expert_pattern balanced \ + --per_candidate_timeout_s 120 \ + --checkpoint_every 0 \ + --output_file out/deepseek_v4_flash_w8.json +``` + +## Search Spaces + +`--search` controls which runtime axes are expanded. The benchmark first builds +a Cartesian product, then filters candidates through backend capability checks, +parallel mapping checks, and forced-communication validity checks. + +| `--search` | Expanded axes | Common use | +|---|---|---| +| `none` | No expansion; run only the base config. | Single-candidate sanity or rerun. | +| `backend` | Backends. If `--backend` is omitted, all backends are searched. | Find the fastest backend. | +| `comm` | Forced communication methods: `NVLINK_ONE_SIDED`, `NVLINK_TWO_SIDED`, `DEEPEP`, `DEEPEPLOWLATENCY`, and `ALLGATHER`. `AUTO` is filtered out. | Compare concrete communication strategies. | +| `backend comm` | Backend x forced-communication product; other axes use the base config. | Most common combined search. | +| `parallel` | Parallel layouts: `DEP`, `TEP`, `DTP`, `TTP`, or a subset. | Compare EP/TP layout effects. | +| `full` | Backend, parallel layout, communication, and CUDA Graph on/off. | Full runtime sweep; can be large. | + +Passing more than one value to `--backend`, `--comm_method`, or +`--parallel_mode` automatically enables that search axis. `--comm_method AUTO` +is valid for a single candidate, but it is not part of a communication sweep. + +Use `--max_configs N` to keep only the first `N` valid candidates after pruning. +Use `--time_budget_minutes M` to stop launching new candidates after a wall-clock +deadline. The currently running candidate is not interrupted. + +Recommended search patterns: + +| Question | Command pattern | +|---|---| +| Which backend is fastest? | `--search backend` | +| Which forced communication strategy is fastest under one backend? | `--backend TRTLLM --search comm` | +| Which backend x communication pair wins? | `--search backend comm` | +| Compare only a backend subset. | `--backend CUTLASS DEEPGEMM` | +| Compare only a communication subset. | `--comm_method NVLINK_ONE_SIDED DEEPEP` | +| Check whether parallel layout dominates. | `--search parallel --backend TRTLLM` or `--parallel_mode DEP TEP` | +| Keep a customer sweep small. | List allowed values explicitly in CLI or in the config-file `search` block. | + +Prefer space-separated search axes, for example `--search backend comm parallel`. +Comma-separated spelling such as `--search backend,comm` also parses, but the +space-separated form is consistent with other multi-value options such as +`--balanced_total_num_tokens 64 128 256`. + +## Workload and Routing Shapes + +Workload has two layers: token count controls how many tokens enter MoE, while +routing control shapes how selected experts are distributed across ranks and +local experts. Without routing-control flags, the default is +`native + balanced_alltoall + balanced`. + +Routing control is useful when you need to reproduce a trace shape, stress a +specific communication path, or isolate expert-hotspot behavior without changing +the model definition. It can either keep model-native routing (`native`) or +materialize supplied top-k inputs (`forced`) for exact control experiments. + +| Option | Meaning | +|---|---| +| `--balanced_total_num_tokens` | Global token counts to sweep. Tokens are distributed as evenly as possible across ranks. | +| `--per_rank_num_tokens` | Explicit token count per rank. Length must equal `world_size`; mutually exclusive with `--balanced_total_num_tokens`. | +| `--comm_pattern` | Source-to-target slot traffic shape. Examples: `balanced_alltoall`, `receiver_hotspot`, `pair_hotspot`, `local_only`, `ring`, `random`. | +| `--expert_pattern` | Local expert distribution on each target rank. Examples: `balanced`, `hotspot,hotness=0.5`, `hotspot,active_experts=2`, `random`. | +| `--routing_pattern_file` | JSON file that fixes both dispatch and expert matrices. | +| `--routing_mode native` | Keep model-native logits/fused routing while projecting toward the requested shape. | +| `--routing_mode forced` | Supply top-k ids/scales directly for exact control experiments. | +| `--enable_perfect_router` | Opt into the lower-level MoE perfect-router override. Normal routing-control cases leave it off and use `bench_moe`-projected logits. | + +### Comm Patterns + +`--comm_pattern` controls source-rank to target-rank slot traffic. One slot is +one `(token, selected_expert)` pair. + +| Pattern | Meaning | Example | +|---|---|---| +| `balanced_alltoall` | Each source rank sends selected slots as evenly as possible to all target ranks. This is the default fair baseline. | `--comm_pattern balanced_alltoall` | +| `receiver_hotspot` | Each source row sends a chosen fraction of slots to one target rank. Useful for receiver-side bandwidth and queueing stress. | `--comm_pattern receiver_hotspot,hotness=0.75,rank=0` | +| `pair_hotspot` | Only one source-to-target pair becomes hot. Useful for peer-link hotspots. | `--comm_pattern pair_hotspot,hotness=0.5,src=0,dst=1` | +| `local_only` | All slots stay on their source rank. Useful as a near-zero cross-rank communication baseline. | `--routing_mode forced --comm_pattern local_only` | +| `ring` | Source rank `i` sends to `(i + 1) % ep_size`. Useful for structured peer traffic. | `--routing_mode forced --comm_pattern ring` | +| `random` | Generate deterministic pseudo-random dispatch with `--routing_seed`. | `--comm_pattern random --routing_seed 42` | + +### Expert Patterns + +`--expert_pattern` controls how received slots are distributed over the local +experts of each target rank. + +| Pattern | Meaning | Example | +|---|---|---| +| `balanced` | Slots are approximately balanced across local experts. | `--expert_pattern balanced` | +| `hotspot,hotness=...` | A chosen fraction of slots goes to one local expert on each target rank. | `--expert_pattern hotspot,hotness=0.5` | +| `hotspot,active_experts=...` | Only a chosen number of local experts receive all slots. | `--expert_pattern hotspot,active_experts=2` | +| `random` | Generate deterministic pseudo-random local expert histograms with `--routing_seed`. | `--expert_pattern random --routing_seed 42` | + +Common routing-control combinations: + +| Goal | Flags | +|---|---| +| Fair baseline | `--comm_pattern balanced_alltoall --expert_pattern balanced` | +| Receiver hotspot | `--comm_pattern receiver_hotspot,hotness=0.75,rank=0` | +| Pair hotspot | `--comm_pattern pair_hotspot,hotness=0.5,src=0,dst=1` | +| Local-only baseline | `--routing_mode forced --comm_pattern local_only` | +| Reject inexact native projection | `--routing_mode native --projection_policy reject --comm_pattern local_only` | + +### Native, Forced, and Projection Policy + +| Mode | Best for | Caveat | +|---|---|---| +| `--routing_mode native` | Keep model-native logits and fused routing while pushing routing toward the requested shape. | Some routing methods cannot exactly express a requested shape. The result may be `projected` or `rejected`. | +| `--routing_mode forced` | Exact top-k id/scale control for dispatch matrices, expert hotspots, and trace replay. | It bypasses fused scoring. Use it for controlled experiments, not as a direct equivalent of native fused routing. | + +`--projection_policy` only applies to `native` mode. With `project`, the +benchmark runs the closest valid projection and records the deviation in +`routing_realization`. With `reject`, the candidate is skipped if the requested +shape cannot be represented exactly. + +Routing-method projection capability: + +| Routing method | Capability | Notes | +|---|---|---| +| Default | `exact_ids` | Expert ids can be exact; scales come from softmax. | +| Renormalize / RenormalizeNaive | `exact` | Expert ids and scales can be controlled exactly. | +| SigmoidRenorm | `exact_ids` | Expert ids can be exact; scales are limited by sigmoid behavior. | +| Llama4 | `top1_exact` | Only top-1 is exact; `top_k > 1` may project. | +| MiniMax2 | `exact_with_zero_bias` | Exact ids when score-correction bias is zero. | +| DeepSeekV3 | `projected_or_exact` | `n_group` / `topk_group` constraints can force projection. | +| SparseMixer | `unsupported` | Current routing-control projection treats it as projected. | + +`--enable_perfect_router` is normally not needed for routing-control experiments: + +| Routing mode | Pattern | Perfect router | +|---|---|---| +| `native` | `balanced_alltoall + balanced` | May be enabled for the lower-level perfect-router baseline. | +| `native` | Any custom comm or expert pattern | Keep off to avoid conflicting with projection. | +| `forced` | Any pattern | Keep off; the supplied-topk path already owns the routing inputs. | + +In `forced` mode, `routing_control.actual.observation_source` is `plan_exact` +because the benchmark patches the kernel inputs to consume the materialized +plan. In `native` mode, it is `plan_simulation`: observed matrix fields are +deterministic re-materializations of the requested plan, not a runtime capture +of the kernel's final selected experts. Use `routing_realization.status` and +`max_abs_slot_error` to decide whether the projection is close enough. + +## Reading Outputs + +Rank 0 prints the benchmark header and result rows to stdout. With +`--output_file`, the full payload is written to JSON. By default the benchmark +also writes an Excel workbook next to the JSON. + +The top-level JSON has this shape: + +```json +{ + "benchmark": "bench_moe", + "environment": {}, + "model": {}, + "search": {}, + "base_config": {}, + "results": [], + "rankings": [] +} +``` + +Start with `rankings` when you only need the winner: + +```json +{ + "num_tokens": 256, + "parallel_mode": "DEP", + "best": { + "backend": "TRTLLM", + "requested_backend": "TRTLLM", + "comm_method": "NVLinkOneSided", + "cuda_graph": true, + "score_ms": 0.8502, + "status": "success" + }, + "ranking": [] +} +``` + +`best` is the lowest-scoring successful candidate in the same +`(num_tokens, parallel_mode)` group. Skipped and failed candidates remain in +`ranking`, so a sweep can explain both "what won" and "why other candidates did +not run." + +| Field or sheet | Meaning | +|---|---| +| `rankings` | Best candidates grouped by `(num_tokens, parallel_mode)`. | +| `workload` | Token settings, per-rank token list, and routing-control request. | +| `requested_config` | User-requested or search-expanded candidate configuration. | +| `actual_config` | Backend, communication method, scheduler, EP/TP layout, and chunk count that actually ran. | +| `status` / `skip_reason` | Candidate outcome and the reason for pruning, skipping, timeout, or failure. | +| `latency_ms.score` | Ranking score based on slowest-rank latency with robust outlier handling. | +| `latency_ms.raw_score` | Unfiltered slowest-rank mean. | +| `latency_ms.iter_max_stats` | Mean, median, p90, min, max, and stdev over per-iteration slowest-rank latency. | +| `latency_ms.iter_max_outliers` | Outliers detected by robust scoring, including index, value, center, and modified z-score. | +| `kernel_breakdown` | CUDA kernel-name statistics when `--analysis kernels` succeeds. | +| `raw_data.forward_times_ms` | Per-rank forward latency samples for every timed iteration. | +| `raw_data.kernel_times_ms` | Per-rank per-kernel samples when CUPTI collection succeeds. | +| `routing_control` | Requested routing shape and observed/planned routing shape. | +| `all_results` | Excel sheet with one row per candidate. | +| `best_by_workload` | Excel sheet with winners per workload. | +| `status_summary` | Excel sheet with skip/failure aggregation. | + +Excel workbook sheets: + +| Sheet | Contents | Typical use | +|---|---|---| +| `all_results` | One row per candidate, including requested/actual config, score, raw score, outlier count, kernel count, and routing summary. | Sort, filter, and compare candidates quickly. | +| `best_by_workload` | One winner per `(num_tokens, parallel_mode)` group. | Find winners without reading nested JSON. | +| `status_summary` | Aggregated counts by token, backend, communication method, status, and skip reason. | Check whether a sweep was mostly pruned or failed. | +| `kernel_breakdown` | One row per `(candidate, category, kernel_name, rank)` with mean, median, p90, min, max, and stdev. | Inspect real CUDA kernel-name statistics per rank. | +| `raw_data` | One row per raw sample. `record_type=forward` stores per-forward samples; `record_type=kernel` stores per-kernel samples. | Investigate outliers and build custom plots. | +| `workload_` | Candidate rows split by workload. | Local analysis for a single token count. | + +Raw data boundary: `raw_data.forward_times_ms` is always recorded for successful +timed candidates. `raw_data.kernel_times_ms` is present only when +`--analysis kernels` is enabled and CUPTI/profiler collection succeeds. Older +JSON files that did not save raw data cannot be retroactively expanded into raw +per-iteration samples. + +`routing_control.actual` is the main place to inspect routing-control quality: + +```json +{ + "routing_path": "logits_projected", + "routing_realization": { + "status": "projected", + "reason": "DeepSeekV3 grouped routing: ...", + "max_abs_slot_error": 3, + "max_relative_slot_error": 0.004 + }, + "enable_perfect_router": false, + "max_num_tokens_per_rank": 256, + "num_chunks_observed": 1, + "observed_dispatch_matrix_summary": { + "row_sums": [2048, 2048, 2048, 2048], + "col_sums": [6144, 682, 683, 683], + "off_diagonal_ratio": 0.75, + "max_abs_slot_error": 3 + } +} +``` + +`routing_path` can be `logits_native`, `logits_projected`, +`supplied_topk_apply`, or `supplied_topk_run_moe`. `routing_realization.status` +can be `exact`, `projected`, `rejected`, or `forced_exact`. With +`--routing_dump_matrix`, the row also includes full requested and observed +dispatch/expert matrices. + +## Pitfalls and Limitations + +| Symptom | Cause / fix | +|---|---| +| `Custom shapes ... AUTO has no safe default` | Custom shapes must explicitly set `--routing_method`. | +| Forced communication candidate is skipped | Some communication methods do not make sense for non-DP layouts, MoE TP, or `world_size=1`. | +| `routing_realization.status="projected"` | The routing method cannot exactly express the requested shape. Inspect `max_abs_slot_error`. | +| `routing_realization.status="rejected"` | `--projection_policy reject` was used with an inexact native-routing request. The skipped row still keeps routing-control context. | +| Forced mode under TEP is not directly comparable with native | `forced` supplies top-k inputs and skips fused scoring; use it for controlled execution-path experiments. | +| Routing pattern file row-sum error | Each dispatch row must equal `per_rank_num_tokens[src] * top_k`, and `ep_size` must match runtime EP size. | +| Routing pattern file conflicts with comm/expert pattern | `--routing_pattern_file` already fixes both dispatch and expert matrices; do not combine it with non-default `--comm_pattern` or `--expert_pattern`. | +| `per_rank_num_tokens` length error | The list length must equal `world_size`; it is mutually exclusive with `--balanced_total_num_tokens`. | +| `phase_times_ms` is empty | Scheduler-side phase markers are not implemented yet; use latency and kernel breakdown. | +| Backend appears as skipped | Backend `can_implement()`, mapping, or communication capability gates rejected the candidate. Read `skip_reason`. | +| CUDA Graph timing has no kernel breakdown | CUPTI must initialize before CUDA context creation. If that fails, breakdown is unavailable. | +| Excel `raw_data` sheet only has headers | The JSON may come from an older run without raw data, or the run had no successful candidate. Rerun with current code to capture raw samples. | +| `--search comm` does not include `AUTO` | This is intentional: `AUTO` is a runtime alias, not a concrete forced strategy. Run a separate base case with `--comm_method AUTO` when you need that number. | +| Bare `python3 -m bench_moe --world_size 4` hangs or reports `MPI_ERR_SPAWN` | This path calls `MPI.COMM_SELF.Spawn`. Many Slurm/Pyxis systems disable MPI dynamic spawn. Use external `mpirun` or `srun --mpi=pmix`. | +| `trtllm-llmapi-launch` times out with `bench_moe` | That launcher runs the user process on rank 0 and MGMN workers on other ranks. `bench_moe` requires every rank to run the benchmark worker. | +| `--word_size` does not work | The correct option is `--world_size`. | + +## Built-in Model Presets + +| Model | Experts | `top_k` | Hidden | Intermediate | Default quant | Default routing | +|---|---:|---:|---:|---:|---|---| +| `qwen1.5_moe` | 60 | 4 | 2048 | 1408 | `FP8` | `RENORMALIZE` | +| `deepseek_v2_lite` | 64 | 6 | 2048 | 1408 | `FP8_BLOCK_SCALES` | `DEEPSEEK_V3` | +| `deepseek_v3` | 256 | 8 | 7168 | 2048 | `FP8_BLOCK_SCALES` | `DEEPSEEK_V3` | +| `kimi_k2` | 384 | 8 | 7168 | 2048 | `FP8_BLOCK_SCALES` | `DEEPSEEK_V3` | +| `deepseek_v4_pro` | 384 | 6 | 7168 | 3072 | pass `--quant` | `RENORMALIZE` | +| `deepseek_v4_flash` | 256 | 6 | 4096 | 2048 | pass `--quant` | `RENORMALIZE` | +| `mixtral_8x7b` | 8 | 2 | 4096 | 14336 | `FP8` | `RENORMALIZE` | +| `gpt_oss_120b` | 128 | 4 | 2880 | 2880 | `W4A8_MXFP4_MXFP8` | `RENORMALIZE` | + +Custom shapes can be used instead of `--model`: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --num_experts 384 \ + --top_k 6 \ + --hidden_size 7168 \ + --intermediate_size 3072 \ + --quant FP8_BLOCK_SCALES \ + --routing_method RENORMALIZE \ + --backend TRTLLM \ + --balanced_total_num_tokens 64 128 +``` + +When `--model` is omitted, do not rely on `--routing_method AUTO`; custom shapes +have no model preset from which a safe routing default can be inferred. + +## Backend, Parallel, and Communication Options + +`--backend`, `--comm_method`, and `--parallel_mode` all accept multiple values. +Passing more than one value automatically enables the matching search axis. +`--backend ALL` is a compatibility spelling for "search all backends". + +Parallel modes: + +| Mode | `moe_ep_size` | `moe_tp_size` | `enable_attention_dp` | Meaning | +|---|---:|---:|---|---| +| `DEP` | `world_size` | 1 | true | Data-parallel attention with expert-parallel MoE. | +| `TEP` | `world_size` | 1 | false | Tensor-parallel attention with expert-parallel MoE. | +| `DTP` | 1 | `world_size` | true | Data-parallel attention with MoE tensor parallelism. | +| `TTP` | 1 | `world_size` | false | Tensor-parallel attention with MoE tensor parallelism. | +| `CUSTOM` | user-specified | user-specified | user-specified | Advanced layout through explicit mapping flags. | + +`CUSTOM` is not part of the default `--search parallel` expansion. A bare +`--search parallel` expands only `DEP`, `TEP`, `DTP`, and `TTP`. Use `CUSTOM` +when you need an EP/TP split that is not covered by the four presets. + +Users provide a custom layout through these flags: + +- `--parallel_mode CUSTOM` +- `--moe_ep_size `: MoE expert-parallel size. +- `--moe_tp_size `: MoE tensor-parallel size. +- `--enable_attention_dp`: optional; if omitted, attention DP is disabled for + `CUSTOM`. + +The invariant is `moe_ep_size * moe_tp_size == world_size`. The benchmark +validates this before running a candidate. As a convenience, passing +`--moe_ep_size` or `--moe_tp_size` while the base `--parallel_mode` is still one +of `DEP` / `TEP` / `DTP` / `TTP` is treated as opting into `CUSTOM`, so output +metadata reports the real layout. + +`CUSTOM` must be a single parallel mode. Do not combine it with preset modes in +one `--parallel_mode` list, because one scalar pair of `--moe_ep_size` / +`--moe_tp_size` cannot describe multiple layouts. For JSON configs, keep the +search space in the file if desired, but still provide the custom EP/TP sizes on +the CLI: + +```json +{ + "model": "deepseek_v3", + "workload": { + "balanced_total_num_tokens": [256] + }, + "search": { + "parallel_mode": ["CUSTOM"] + }, + "output_file": "out/custom_layout.json" +} +``` + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --config_file configs/custom_layout.json \ + --moe_ep_size 2 \ + --moe_tp_size 2 \ + --enable_attention_dp +``` + +Equivalent CLI-only custom layout: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode CUSTOM \ + --moe_ep_size 2 \ + --moe_tp_size 2 \ + --enable_attention_dp \ + --backend TRTLLM \ + --balanced_total_num_tokens 256 +``` + +Communication methods are `AUTO`, `NVLINK_ONE_SIDED`, `NVLINK_TWO_SIDED`, +`DEEPEP`, `DEEPEPLOWLATENCY`, and `ALLGATHER`. For a single candidate, `AUTO` +lets TensorRT-LLM choose the concrete method. In a communication sweep, +`bench_moe` uses forced concrete methods and filters `AUTO` out to avoid +duplicating an alias. + +## Timing, Output, and Limits + +| Option | Default | Meaning | +|---|---|---| +| `--no_cuda_graph` | not set | Use eager timing instead of CUDA Graph timing. | +| `--warmup` | `1` | Warmup iterations before timed iterations. | +| `--iters` | `12` | Timed iterations. At least 8 samples enables MAD-based outlier filtering. | +| `--fast_autotune` | `false` | Shorten autotune repeat/warmup for quick debugging. | +| `--per_candidate_timeout_s` | `0.0` | Hard wall-clock timeout per candidate; useful for NCCL/CUDA hangs. | +| `--analysis` | `kernels` | Use `kernels` for CUPTI kernel breakdown or `none` for latency only. | +| `--dtype` | `bfloat16` | Activation dtype. Common values are `bfloat16` and `float16`. | +| `--output_file` | unset | Write the final JSON report and determine the default Excel path. | +| `--analysis_workbook_file` | `.analysis.xlsx` | Write the Excel workbook. | +| `--resume_from` | unset | Read an existing JSON report and skip completed terminal candidates. | +| `--checkpoint_every` | `1` | Write a JSON checkpoint after every N newly completed candidates. Use `0` for final-only output. | +| `--max_configs N` | disabled | After pruning, keep only the first N valid candidates. | +| `--time_budget_minutes M` | disabled | Stop launching new candidates after the deadline. | + +Autotune is an untimed pre-pass. The benchmark runs autotune before formal +timing to populate kernel caches; reported latency does not include the autotune +pass. + +`--per_candidate_timeout_s` is a hard guard around one candidate. If a candidate +hangs in NCCL or CUDA, the watchdog terminates the process; use `--resume_from` +to rerun missing candidates from the last checkpoint. `--max_configs` and +`--time_budget_minutes` are sweep-level limiters and do not interrupt a candidate +that is already running. + +Example bounded sweep: + +```bash +PYTHONPATH=tests/microbenchmarks:${PYTHONPATH:-} \ +mpirun --allow-run-as-root --oversubscribe --bind-to none --map-by slot -np 4 \ + python3 -m bench_moe \ + --world_size 4 \ + --model deepseek_v3 \ + --parallel_mode DEP \ + --search backend comm \ + --balanced_total_num_tokens 128 256 \ + --max_configs 32 \ + --time_budget_minutes 30 \ + --output_file out/bounded_sweep.json +``` diff --git a/tests/microbenchmarks/bench_moe/__init__.py b/tests/microbenchmarks/bench_moe/__init__.py new file mode 100644 index 000000000000..a6ebf3567bab --- /dev/null +++ b/tests/microbenchmarks/bench_moe/__init__.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MoE microbenchmark package entrypoint. + +The implementation lives in focused submodules. This package mirrors the +historical ``bench_moe.*`` import surface by re-exporting every non-dunder +attribute, including underscore-prefixed helpers used by tests and scripts. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +_MICROBENCH_DIR = Path(__file__).resolve().parent.parent +_TESTS_UNITTEST_DIR = _MICROBENCH_DIR.parent / "unittest" +if str(_TESTS_UNITTEST_DIR) not in sys.path: + sys.path.insert(0, str(_TESTS_UNITTEST_DIR)) + +_REPO_ROOT = _MICROBENCH_DIR.parent.parent +if (_REPO_ROOT / "tensorrt_llm" / "bindings").is_dir() and str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +_PACKAGE_PARENT = Path(__file__).resolve().parent.parent +if str(_PACKAGE_PARENT) not in sys.path: + sys.path.insert(0, str(_PACKAGE_PARENT)) + + +def _mirror_module(module_name: str) -> None: + module = importlib.import_module(module_name) + for name, value in module.__dict__.items(): + if not (name.startswith("__") and name.endswith("__")): + globals()[name] = value + + +for _module_name in ( + "bench_moe.backend", + "bench_moe.quantize", + "bench_moe.specs", + "bench_moe.search", + "bench_moe.utils", + "bench_moe.mapping", + "bench_moe.build", + "bench_moe.routing", + "bench_moe.timing", + "bench_moe.results", + "bench_moe.cli", + "bench_moe.case_runner", + "bench_moe.worker", +): + _mirror_module(_module_name) + +del _module_name diff --git a/tests/microbenchmarks/bench_moe/__main__.py b/tests/microbenchmarks/bench_moe/__main__.py new file mode 100644 index 000000000000..ce2bb6b3df2f --- /dev/null +++ b/tests/microbenchmarks/bench_moe/__main__.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script entry point for the MoE microbenchmark package. + +Two invocation styles are supported and tested: + + # As a module (preferred, requires the parent of bench_moe/ on sys.path): + python -m bench_moe --help + + # As a file path (matches CI and legacy script-style invocation): + python tests/microbenchmarks/bench_moe/__main__.py --help + +When invoked by file path, Python sets ``sys.path[0]`` to the directory +containing the script (the ``bench_moe`` package itself), which would shadow +the package's relative imports. We detect that case and prepend the package's +parent directory so ``import bench_moe`` resolves to the package rather than +this single file. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def _ensure_package_importable() -> None: + """Make ``import bench_moe`` resolve to the parent package. + + Only needed when this file is executed directly (``python __main__.py``), + not when run via ``python -m bench_moe`` (which sets ``__package__`` for + us). + """ + if __package__: + return + + pkg_dir = Path(__file__).resolve().parent + parent_dir = pkg_dir.parent + + if str(parent_dir) not in sys.path: + sys.path.insert(0, str(parent_dir)) + + pkg_dir_str = str(pkg_dir) + while pkg_dir_str in sys.path: + sys.path.remove(pkg_dir_str) + + +_ensure_package_importable() + +from bench_moe.worker import main # noqa: E402 + +if __name__ == "__main__": + main() diff --git a/tests/microbenchmarks/bench_moe/backend.py b/tests/microbenchmarks/bench_moe/backend.py new file mode 100644 index 000000000000..d5530186947c --- /dev/null +++ b/tests/microbenchmarks/bench_moe/backend.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local MoE backend registry for ``bench_moe``. + +The benchmark should not import unittest-wide MoE helpers at module import time: +those helpers import every backend, including CUTEDSL, even when a run only asks +for TRTLLM/CUTLASS/MegaMoE. Keep the small registry needed by the benchmark here +and import concrete backend classes lazily. +""" + +from __future__ import annotations + +import importlib +import sys +import types +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class MoeBackendType(str, Enum): + """MoE backend identifiers accepted by bench_moe.""" + + CUTLASS = "CUTLASS" + TRTLLM = "TRTLLM" + CUTEDSL = "CUTEDSL" + DEEPGEMM = "DEEPGEMM" + DENSEGEMM = "DENSEGEMM" + MEGAMOE = "MEGAMOE_DEEPGEMM" + + +@dataclass +class MoeModelConfig: + """MoE model shape used by routing-method construction.""" + + num_experts: int + top_k: int + hidden_size: int + intermediate_size: int + n_group: Optional[int] = None + topk_group: Optional[int] = None + + def __str__(self) -> str: + return f"e{self.num_experts}_k{self.top_k}_h{self.hidden_size}_i{self.intermediate_size}" + + +def resolve_deepseek_group_config(model_config: MoeModelConfig) -> tuple[int, int]: + """Resolve DeepSeek-V3 routing group settings for built-in or custom shapes.""" + if model_config.n_group is not None and model_config.topk_group is not None: + return model_config.n_group, model_config.topk_group + n_group = max(1, model_config.num_experts // 2) + topk_group = min(n_group, max(1, n_group // 2)) + return n_group, topk_group + + +def ensure_cute_dsl_importable_for_benchmark() -> None: + """Install a local sentinel module when optional CUTEDSL imports are absent. + + Production MoE modules import ``fused_moe_cute_dsl`` at module load time. + ``bench_moe`` can still benchmark non-CUTEDSL backends in environments that + do not package CUTLASS DSL; keep that fallback local to the benchmark rather + than weakening the production module. + """ + module_name = "tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl" + if module_name in sys.modules: + return + try: + importlib.import_module(module_name) + return + except ImportError as exc: + import_error = exc + + class CuteDslFusedMoE: + @classmethod + def can_implement(cls, *_args, **_kwargs): + return False, f"CUTLASS DSL is unavailable: {import_error}" + + def __init__(self, *_args, **_kwargs): + raise RuntimeError(f"CUTLASS DSL is unavailable: {import_error}") + + module = types.ModuleType(module_name) + module.CuteDslFusedMoE = CuteDslFusedMoE + sys.modules[module_name] = module + + +def get_backend_class(backend_type: MoeBackendType): + """Import and return the concrete backend class for ``backend_type`` lazily.""" + if backend_type == MoeBackendType.CUTLASS: + from tensorrt_llm._torch.modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE + + return CutlassFusedMoE + if backend_type == MoeBackendType.TRTLLM: + from tensorrt_llm._torch.modules.fused_moe.fused_moe_trtllm_gen import TRTLLMGenFusedMoE + + return TRTLLMGenFusedMoE + if backend_type == MoeBackendType.CUTEDSL: + from tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl import CuteDslFusedMoE + + return CuteDslFusedMoE + if backend_type == MoeBackendType.DEEPGEMM: + from tensorrt_llm._torch.modules.fused_moe.fused_moe_deepgemm import DeepGemmFusedMoE + + return DeepGemmFusedMoE + if backend_type == MoeBackendType.DENSEGEMM: + from tensorrt_llm._torch.modules.fused_moe.fused_moe_densegemm import DenseGEMMFusedMoE + + return DenseGEMMFusedMoE + if backend_type == MoeBackendType.MEGAMOE: + from tensorrt_llm._torch.modules.fused_moe.mega_moe import MegaMoEDeepGemm + + return MegaMoEDeepGemm + raise ValueError(f"unknown MoE backend {backend_type!r}") diff --git a/tests/microbenchmarks/bench_moe/build.py b/tests/microbenchmarks/bench_moe/build.py new file mode 100644 index 000000000000..0715bc783b23 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/build.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MoE module construction and introspection helpers. + +This module owns the "build phase" of one timing case: it instantiates a +fresh :class:`ConfigurableMoE` via :func:`create_moe`, generates the +quantization-aware weights, loads them into the module, and provides the +small set of introspection helpers (``actual_backend`` / ``scheduler_kind`` +/ ``comm_method`` / ``num_chunks``) used by the result schema. +""" + +from __future__ import annotations + +import os +from typing import Dict, List, Optional + +import torch + +from tensorrt_llm._torch.modules.fused_moe.interface import MoESchedulerKind, MoEWeightLoadingMode +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantAlgo + +from .backend import MoeBackendType, ensure_cute_dsl_importable_for_benchmark +from .mapping import _build_model_config, _create_routing_method +from .quantize import get_test_quant_params +from .specs import ConfigSpec, ModelSpec +from .utils import _ensure_dist_for_megamoe + +# Map concrete MoE module class names to short backend identifiers used in +# results and the dashboard. Anything not in this table falls back to the +# upper-case class name. +_BACKEND_CLASS_TO_NAME: Dict[str, str] = { + "CutlassFusedMoE": "CUTLASS", + "TRTLLMGenFusedMoE": "TRTLLM", + "CuteDslFusedMoE": "CUTEDSL", + "DeepGemmFusedMoE": "DEEPGEMM", + "DenseGEMMFusedMoE": "DENSEGEMM", + "MegaMoEDeepGemm": "MEGAMOE_DEEPGEMM", + "VanillaMoE": "VANILLA", +} + + +def _backend_name_from_module(moe) -> str: + """Resolve ``actual_backend`` for both ConfigurableMoE and legacy modules.""" + backend_attr = getattr(moe, "backend", None) + if backend_attr is not None and backend_attr is not moe: + backend_cls = type(backend_attr).__name__ + else: + backend_cls = type(moe).__name__ + return _BACKEND_CLASS_TO_NAME.get(backend_cls, backend_cls.upper()) + + +def _scheduler_kind_name(moe) -> Optional[str]: + """Return ``"EXTERNAL_COMM"`` / ``"FUSED_COMM"`` for the underlying backend.""" + backend = getattr(moe, "backend", None) or moe + kind = getattr(backend, "scheduler_kind", None) + if isinstance(kind, MoESchedulerKind): + return kind.name + return None + + +def _comm_method_name(moe) -> str: + """Return the actual communication strategy class name, or ``"NONE"``.""" + if _scheduler_kind_name(moe) == "FUSED_COMM": + return "NONE" + comm = getattr(moe, "comm", None) + if comm is None: + return "NONE" + return type(comm).__name__ + + +def _calculate_num_chunks_safe(moe, all_rank_num_tokens: List[int]) -> Optional[int]: + """Best-effort lookup of ``num_chunks`` for the case we are about to time.""" + scheduler = getattr(moe, "scheduler", None) + if scheduler is None: + return None + fn = getattr(scheduler, "calculate_num_chunks", None) + if fn is None: + return None + try: + return int(fn(all_rank_num_tokens)) + except Exception: + return None + + +def _create_moe_for_benchmark(**kwargs): + ensure_cute_dsl_importable_for_benchmark() + from tensorrt_llm._torch.modules.fused_moe.create_moe import create_moe + + return create_moe(**kwargs) + + +def _build_moe_module( + *, + model: ModelSpec, + config: ConfigSpec, + mapping: Mapping, + moe_backend: str, + use_cuda_graph: bool, + max_num_tokens: int, + use_low_precision_moe_combine: bool, + enable_perfect_router: bool, + dtype: torch.dtype, + routing_logits_dtype: torch.dtype, + device: torch.device, +): + """Build a fresh ``ConfigurableMoE`` for one ``(backend, num_tokens)`` case. + + Returns ``(moe_module, routing_logits_dtype)``. + """ + if enable_perfect_router: + os.environ["ENABLE_PERFECT_ROUTER"] = "1" + else: + os.environ.pop("ENABLE_PERFECT_ROUTER", None) + + mc = model.to_moe_model_config() + swiglu_gptoss_style = model.swiglu_gptoss_style + + routing_method = _create_routing_method( + model.routing_method_cls, + top_k=mc.top_k, + num_experts=mc.num_experts, + bias_dtype=dtype, + profile_model_config=mc, + ) + + model_config = _build_model_config( + model=model, + mapping=mapping, + moe_backend=moe_backend, + use_cuda_graph=use_cuda_graph, + max_num_tokens=max_num_tokens, + use_low_precision_moe_combine=use_low_precision_moe_combine, + dtype=dtype, + ) + + _ensure_dist_for_megamoe(moe_backend, mapping.rank, mapping.world_size) + + probe_x = torch.randn( + (max(1, mc.hidden_size // 32), mc.hidden_size), dtype=dtype, device=device + ) + backend_type = MoeBackendType(moe_backend.upper()) + quant_algo = model.quant_algo_enum + quantize_util_cls, quant_config, quant_kwargs = get_test_quant_params( + quant_algo, probe_x, backend_type + ) + quant_kwargs.pop("ref_cls", None) + + num_local_experts = mc.num_experts // max(mapping.moe_ep_size, 1) + quantize_util = quantize_util_cls( + num_experts=mc.num_experts, + dtype=dtype, + intermediate_size=mc.intermediate_size, + hidden_size=mc.hidden_size, + quant_config=quant_config, + bias=swiglu_gptoss_style, + swiglu_gptoss_style=swiglu_gptoss_style, + swiglu_alpha=model.swiglu_alpha if swiglu_gptoss_style else None, + swiglu_beta=model.swiglu_beta if swiglu_gptoss_style else None, + swiglu_limit=model.swiglu_limit if swiglu_gptoss_style else None, + num_local_experts=num_local_experts, + ) + + weight_loading_mode = getattr( + quantize_util, "weight_loading_mode", MoEWeightLoadingMode.VANILLA + ) + + swiglu_tensors = quantize_util.get_swiglu_tensors() + + moe = _create_moe_for_benchmark( + routing_method=routing_method, + num_experts=mc.num_experts, + hidden_size=mc.hidden_size, + intermediate_size=mc.intermediate_size, + dtype=dtype, + reduce_results=True, + model_config=model_config, + weight_loading_mode=weight_loading_mode, + bias=swiglu_gptoss_style, + swiglu_alpha=swiglu_tensors["swiglu_alpha"] if swiglu_tensors else None, + swiglu_beta=swiglu_tensors["swiglu_beta"] if swiglu_tensors else None, + swiglu_limit=swiglu_tensors["swiglu_limit"] if swiglu_tensors else None, + ) + + if quant_algo == QuantAlgo.W4A8_MXFP4_MXFP8: + weights, _ref_weights, _ref_kwargs = quantize_util.prepare_weights_from_backend( + moe, **quant_kwargs + ) + else: + weights = quantize_util.create_weights(**quant_kwargs) + + moe.load_weights([weights]) + moe.post_load_weights() + moe.cuda(f"cuda:{torch.cuda.current_device()}") + + return moe, routing_logits_dtype diff --git a/tests/microbenchmarks/bench_moe/case_runner.py b/tests/microbenchmarks/bench_moe/case_runner.py new file mode 100644 index 000000000000..bb3829703bbc --- /dev/null +++ b/tests/microbenchmarks/bench_moe/case_runner.py @@ -0,0 +1,857 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-candidate execution and routing-control orchestration.""" + +from __future__ import annotations + +import hashlib +import json +import os +import traceback +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from tensorrt_llm._torch.autotuner import AutoTuner +from tensorrt_llm._torch.modules.fused_moe.fused_moe_trtllm_gen import TRTLLMGenFusedMoE +from tensorrt_llm._utils import mpi_allgather + +from .build import ( + _backend_name_from_module, + _build_moe_module, + _calculate_num_chunks_safe, + _comm_method_name, + _scheduler_kind_name, +) +from .mapping import _build_mapping_from_config, _resolve_mapping_layout +from .results import ( + _build_latency_block, + _build_raw_data_block, + _classify_bottleneck, + _gather_kernel_timing_blocks, + _gather_per_iteration_times, +) +from .routing import ( + RoutingPlan, + _build_routing_plan, + _materialize_selected_experts_for_rank, + _maybe_install_routing_control_patch, + _observe_routing_metrics, + _observe_summary, + _per_rank_tokens, + _project_router_logits_for_plan, +) +from .specs import ( + _FORCED_COMM_ENV_VALUES, + ConfigSpec, + ModelSpec, + RoutingControlSpec, + RunResult, + WorkloadSpec, +) +from .timing import _run_autotune, _time_moe_forward_cuda_graph, _time_moe_forward_eager +from .utils import _InputCache, _make_inputs, _maybe_print_rank0 + + +def _force_comm_env(comm_method: str, prev: Optional[str]) -> None: + """Push or restore ``TRTLLM_FORCE_COMM_METHOD`` for one case. + + The design notes that env-var forcing is the only available per-case knob + today; subprocess isolation is the recommended long-term path. + """ + upper = comm_method.upper() + if upper in _FORCED_COMM_ENV_VALUES: + os.environ["TRTLLM_FORCE_COMM_METHOD"] = upper + else: + if prev is None: + os.environ.pop("TRTLLM_FORCE_COMM_METHOD", None) + else: + os.environ["TRTLLM_FORCE_COMM_METHOD"] = prev + + +def _gather_status_per_rank(local_status: str) -> Dict[str, str]: + payload = mpi_allgather(local_status) + return {f"rank{i}": s for i, s in enumerate(payload)} + + +@dataclass(frozen=True) +class _RoutingObservations: + """Observation matrices and counters fed into ``_build_routing_control_block``. + + These are the four numbers the consumer cares about: the per-slot dispatch + matrix, per-token dispatch matrix, per-rank expert histogram, and the + optional ``num_chunks`` counter observed during the run. All come from a + deterministic re-materialisation of the canonical ``RoutingPlan`` -- see + ``_build_routing_control_block`` for the contract. + """ + + slot: List[List[int]] + token: List[List[int]] + hist: List[List[int]] + num_chunks: Optional[int] = None + + +def _build_routing_control_block( + *, + spec: RoutingControlSpec, + plan: RoutingPlan, + observations: _RoutingObservations, + routing_path: Optional[str], + realization_status: str, + realization_reason: str, + enable_perfect_router: bool, + max_num_tokens_per_rank: int, + warnings: List[str], + scale_dtype: torch.dtype, + moe_ep_size: int, +) -> Dict[str, Any]: + """Compose the ``routing_control`` block for a result row. + + Always includes ``requested`` and an ``actual`` summary; full slot/token + matrices and histograms are included only when ``spec.routing_dump_matrix`` + is set, to avoid JSON bloat during large sweeps. + + NOTE on ``observations.*`` fields: the dispatch / histogram numbers + reported here are derived from a deterministic re-materialisation of the + canonical ``RoutingPlan`` -- they describe *the plan the bench asked the + kernel to realise*, not what the kernel actually emitted at runtime. The + ``actual.observation_source`` field documents this. In ``forced`` mode the + kernel is patched to consume the exact materialised top-k, so plan == + kernel output by construction. In ``native`` mode the kernel routes via + projected logits and may produce slightly different top-k due to fp ties, + quantisation, or projection-status='projected'; a warning is added so the + consumer does not over-trust the slot/histogram numbers. + """ + routing_mode = spec.routing_mode + dump_full = bool(spec.routing_dump_matrix) + observed_slot = observations.slot + observed_token = observations.token + observed_hist = observations.hist + num_chunks_observed = observations.num_chunks + + requested_slot = [list(row) for row in plan.dispatch_matrix] + max_abs, max_rel = _observe_summary(requested_slot, observed_slot) + row_sums = [sum(row) for row in observed_slot] + col_sums = [sum(observed_slot[s][d] for s in range(moe_ep_size)) for d in range(moe_ep_size)] + diag = sum(observed_slot[i][i] for i in range(moe_ep_size)) if moe_ep_size > 0 else 0 + total = sum(row_sums) + off_diag_ratio = 0.0 if total <= 0 else (1.0 - diag / total) + + flat_hist = [v for row in observed_hist for v in row] + hist_min = min(flat_hist) if flat_hist else 0 + hist_max = max(flat_hist) if flat_hist else 0 + active_experts = sum(1 for v in flat_hist if v > 0) + + warnings_out = list(warnings) + if routing_mode != "forced": + warnings_out.append( + "observed_* fields are derived from RoutingPlan re-materialisation, " + "not from the kernel's actual selected_experts output; in native mode " + "the real top-k may differ from the plan." + ) + + block: Dict[str, Any] = { + "requested": { + "routing_mode": spec.routing_mode, + "projection_policy": spec.projection_policy, + "comm_pattern": spec.comm_pattern, + "expert_pattern": spec.expert_pattern, + "routing_pattern_file": spec.routing_pattern_file, + "per_rank_num_tokens": list(plan.per_rank_num_tokens), + "seed": int(spec.seed), + }, + "actual": { + "routing_path": routing_path, + "routing_realization": { + "status": realization_status, + "reason": realization_reason, + "max_abs_slot_error": int(max_abs), + "max_relative_slot_error": float(max_rel), + }, + "enable_perfect_router": bool(enable_perfect_router), + "effective_src_axis": "dp_rank", + "max_num_tokens_per_rank": int(max_num_tokens_per_rank), + "num_chunks_observed": int(num_chunks_observed) + if isinstance(num_chunks_observed, int) + else None, + "use_dp_padding": False, + # "plan_exact": forced mode (kernel is patched to the materialised + # plan, so observed == plan by construction). + # "plan_simulation": native mode (numbers are a deterministic + # re-materialisation of the plan, NOT the kernel's actual top-k). + "observation_source": ("plan_exact" if routing_mode == "forced" else "plan_simulation"), + "observed_dispatch_matrix_summary": { + "row_sums": row_sums, + "col_sums": col_sums, + "off_diagonal_ratio": float(off_diag_ratio), + "max_abs_slot_error": int(max_abs), + "matrix_dump_path": None, + }, + "observed_expert_histogram_summary": { + "min": int(hist_min), + "max": int(hist_max), + "active_experts": int(active_experts), + }, + "selected_scales": { + "distribution": "uniform", + "dtype": str(scale_dtype), + "seed": int(spec.seed), + }, + "warnings": warnings_out, + }, + } + if dump_full: + block["actual"]["observed_slot_dispatch_matrix"] = [list(r) for r in observed_slot] + block["actual"]["observed_token_dispatch_matrix"] = [list(r) for r in observed_token] + block["actual"]["observed_expert_histogram"] = [list(r) for r in observed_hist] + block["actual"]["requested_slot_dispatch_matrix"] = [list(r) for r in requested_slot] + return block + + +@dataclass +class _RoutingInputs: + """Inputs derived from routing control for one (case, rank). + + ``router_logits`` is the tensor that will be passed to ``moe.forward``; in + forced mode it is left unchanged because the routing kernels are bypassed. + ``materialized_ids`` / ``materialized_scales`` are only populated in forced + mode and are consumed by ``_maybe_install_routing_control_patch``. + """ + + router_logits: torch.Tensor + materialized_ids: Optional[torch.Tensor] = None + materialized_scales: Optional[torch.Tensor] = None + realization_status: str = "exact" + realization_reason: str = "balanced default" + routing_path: Optional[str] = None + warnings: List[str] = field(default_factory=list) + + +def _short_circuit(result: RunResult, status: str, reason: str) -> RunResult: + """Stamp ``status``/``reason`` on ``result`` and broadcast across ranks. + + Every early-return path in ``_run_one_candidate`` must go through this + helper so the per-rank status allgather completes on all ranks and the + output row remains coherent. + """ + result.status = status + result.skip_reason = reason + result.status_per_rank = _gather_status_per_rank(status) + return result + + +def _initial_instrumentation( + analysis: Tuple[str, ...], + config: ConfigSpec, + cupti_ctx: Optional[Any], +) -> Dict[str, Any]: + return { + "level": ",".join(sorted(analysis)) if analysis else "summary", + "cuda_graph": bool(config.cuda_graph), + "cupti_available": bool(cupti_ctx is not None and cupti_ctx.ok), + "phase_timing_available": False, + "kernel_breakdown_available": "kernels" in analysis, + "autotune_status": "not_run", + "latency_source": "cuda_event_external" if config.cuda_graph else "cuda_event_eager", + } + + +def _pick_enable_perfect_router( + rc_spec: RoutingControlSpec, + enable_perfect_router_requested: bool, +) -> bool: + """Decide whether to enable ``ENABLE_PERFECT_ROUTER`` for one case. + + The perfect router is a lower-level MoE override that replaces the incoming + router logits inside ``MoE.forward``. Keep it explicit so normal + routing-control cases use the logits projected by bench_moe itself. + """ + if not enable_perfect_router_requested: + return False + if rc_spec.routing_mode == "forced": + return False + return True + + +def _build_empty_routing_control_block_for_rejection( + *, + rc_spec: RoutingControlSpec, + routing_plan: RoutingPlan, + model: ModelSpec, + moe_ep_size: int, + per_rank: List[int], + num_chunks: Optional[int], + rejected_reason: str, + enable_perfect_router: bool, + act_dtype: torch.dtype, +) -> Dict[str, Any]: + experts_per_rank = int(model.num_experts) // int(moe_ep_size) + ep = int(moe_ep_size) + empty_observations = _RoutingObservations( + slot=[[0] * ep for _ in range(ep)], + token=[[0] * ep for _ in range(ep)], + hist=[[0] * experts_per_rank for _ in range(ep)], + num_chunks=num_chunks, + ) + return _build_routing_control_block( + spec=rc_spec, + plan=routing_plan, + observations=empty_observations, + routing_path=None, + realization_status="rejected", + realization_reason=rejected_reason, + enable_perfect_router=enable_perfect_router, + max_num_tokens_per_rank=max(per_rank) if per_rank else 0, + warnings=[], + scale_dtype=act_dtype, + moe_ep_size=ep, + ) + + +@dataclass +class _RoutingSkip: + """Encodes a routing-control skip with optional dashboard annotation. + + ``skip_reason`` is the human-readable reason written to the result row. + ``rejected_reason`` is set only for ``projection_policy=reject`` skips, in + which case the caller still emits a ``routing_control`` block carrying the + untruncated projection reason. + """ + + skip_reason: str + rejected_reason: Optional[str] = None + + +def _select_routing_inputs( + *, + moe, + model: ModelSpec, + rc_spec: RoutingControlSpec, + routing_plan: RoutingPlan, + rank: int, + moe_ep_size: int, + base_router_logits: torch.Tensor, + device: torch.device, + act_dtype: torch.dtype, + routing_logits_dtype: torch.dtype, +) -> Tuple[Optional[_RoutingInputs], Optional[_RoutingSkip]]: + """Produce the router_logits / materialised top-k tensors for routing control. + + Returns ``(inputs, None)`` on success or ``(None, skip)`` to abort. The + skip object distinguishes a plain error (materialise / projection) from a + ``projection_policy=reject`` rejection that the dashboard wants to see. + """ + experts_per_rank = int(model.num_experts) // int(moe_ep_size) + ep_axis_rank = rank if rank < int(moe_ep_size) else (rank % int(moe_ep_size)) + + if rc_spec.routing_mode == "forced": + try: + ids, scales = _materialize_selected_experts_for_rank( + routing_plan, + src_rank=ep_axis_rank, + top_k=int(model.top_k), + experts_per_rank=experts_per_rank, + moe_ep_size=int(moe_ep_size), + device=device, + scale_dtype=act_dtype, + ) + except Exception as exc: + return None, _RoutingSkip(f"routing materialise error: {type(exc).__name__}: {exc}") + inner_backend = getattr(moe, "backend", moe) + routing_path = ( + "supplied_topk_run_moe" + if isinstance(inner_backend, TRTLLMGenFusedMoE) + else "supplied_topk_apply" + ) + return ( + _RoutingInputs( + router_logits=base_router_logits, + materialized_ids=ids, + materialized_scales=scales, + realization_status="forced_exact", + realization_reason=( + "forced routing_mode: top-k ids and uniform 1/top_k scales materialised " + "from RoutingPlan; native fused scoring is intentionally bypassed" + ), + routing_path=routing_path, + ), + None, + ) + + # Native mode: synthesise router_logits that drive the production routing + # kernel toward the plan; the path is "logits_native" when the projection + # is exact and "logits_projected" when the routing method cannot represent + # the plan exactly. + try: + new_logits, projection_status, projection_reason = _project_router_logits_for_plan( + routing_plan, + src_rank=ep_axis_rank, + routing_method=moe.routing_method, + num_experts=int(model.num_experts), + top_k=int(model.top_k), + experts_per_rank=experts_per_rank, + moe_ep_size=int(moe_ep_size), + device=device, + dtype=routing_logits_dtype, + ) + except Exception as exc: + return None, _RoutingSkip(f"native logits projection error: {type(exc).__name__}: {exc}") + + if projection_status != "exact" and rc_spec.projection_policy == "reject": + return None, _RoutingSkip( + skip_reason=( + f"routing_realization rejected by projection_policy=reject: {projection_reason}" + ), + rejected_reason=projection_reason, + ) + + warnings: List[str] = [] + if projection_status != "exact": + warnings.append(f"routing_realization={projection_status}: {projection_reason}") + + return ( + _RoutingInputs( + router_logits=new_logits, + realization_status=projection_status, + realization_reason=projection_reason, + routing_path=("logits_native" if projection_status == "exact" else "logits_projected"), + warnings=warnings, + ), + None, + ) + + +def _observe_routing_plan( + *, + routing_plan: RoutingPlan, + model: ModelSpec, + moe_ep_size: int, +) -> Tuple[List[List[int]], List[List[int]], List[List[int]]]: + """Materialise the plan on every EP source rank and aggregate the totals. + + We re-materialise on rank 0 (CPU) for *every* EP source instead of relying + on MPI gather: in DTP/TTP modes multiple world ranks share an EP rank, so + a naive allgather would double-count. + """ + experts_per_rank = int(model.num_experts) // int(moe_ep_size) + per_rank_ids: List[Any] = [] + for src in range(int(moe_ep_size)): + try: + src_ids, _ = _materialize_selected_experts_for_rank( + routing_plan, + src_rank=src, + top_k=int(model.top_k), + experts_per_rank=experts_per_rank, + moe_ep_size=int(moe_ep_size), + device=torch.device("cpu"), + scale_dtype=torch.float32, + ) + except Exception: + src_ids = torch.empty((0, int(model.top_k)), dtype=torch.int32) + per_rank_ids.append(src_ids) + return _observe_routing_metrics(routing_plan, per_rank_ids, experts_per_rank, int(moe_ep_size)) + + +def _finalize_routing_control_block( + *, + result: RunResult, + rc_spec: RoutingControlSpec, + routing_plan: RoutingPlan, + routing_inputs: _RoutingInputs, + model: ModelSpec, + moe_ep_size: int, + per_rank: List[int], + enable_perfect_router: bool, + act_dtype: torch.dtype, +) -> None: + observed_slot, observed_token, observed_hist = _observe_routing_plan( + routing_plan=routing_plan, + model=model, + moe_ep_size=int(moe_ep_size), + ) + observations = _RoutingObservations( + slot=observed_slot, + token=observed_token, + hist=observed_hist, + num_chunks=result.num_chunks, + ) + result.routing_control = _build_routing_control_block( + spec=rc_spec, + plan=routing_plan, + observations=observations, + routing_path=routing_inputs.routing_path, + realization_status=routing_inputs.realization_status, + realization_reason=routing_inputs.realization_reason, + enable_perfect_router=enable_perfect_router, + max_num_tokens_per_rank=max(per_rank) if per_rank else 0, + warnings=routing_inputs.warnings, + scale_dtype=act_dtype, + moe_ep_size=int(moe_ep_size), + ) + + +def _resolve_layout_and_plan( + *, + result: RunResult, + model: ModelSpec, + workload: WorkloadSpec, + config: ConfigSpec, + world_size: int, + rc_spec: RoutingControlSpec, + rc_active: bool, +) -> Union[RunResult, Tuple[int, List[int], Optional[RoutingPlan]]]: + """Step 1 of ``_run_one_candidate``: resolve layout, routing plan, per-rank tokens. + + Resolves the EP-axis ``moe_ep_size`` from the candidate config (the + Mapping object built later will agree), validates that routing-control + cases satisfy ``moe_ep_size == world_size`` (so the dispatch_matrix axis + aligns with the world-rank token distribution), and either builds the + canonical ``RoutingPlan`` from ``rc_spec`` or falls back to a uniform + per-rank token split. + + Returns either: + - ``RunResult`` (already short-circuited) on any layout/plan error, or + - ``(moe_ep_size, per_rank, routing_plan)`` on success. + + Routing-control candidates return a world-axis ``per_rank`` list because + non-EP layouts are skipped before plan construction. Non-routing-control + candidates use the regular balanced world-rank split. + """ + try: + moe_ep_size, _moe_tp_size, _enable_dp = _resolve_mapping_layout(config, world_size) + except ValueError as exc: + return _short_circuit(result, "skipped", str(exc)) + + # Routing-control's dispatch_matrix axis is ``moe_ep_size`` while + # ``per_rank_num_tokens`` follows the world (DP source) axis. When the two + # disagree (DTP/TTP/CUSTOM with ``moe_ep_size != world_size``) the plan + # either crashes inside ``_build_routing_plan`` or silently drops the + # tokens of world ranks beyond ``moe_ep_size``. Skip cleanly. + if rc_active and int(moe_ep_size) != int(world_size): + return _short_circuit( + result, + "skipped", + f"routing-control requires moe_ep_size == world_size " + f"(got moe_ep_size={moe_ep_size}, world_size={world_size}); " + "the dispatch_matrix axis would not align with the per-rank token " + "distribution. Use parallel_mode in {DEP, TEP} or drop routing-control.", + ) + + routing_plan: Optional[RoutingPlan] = None + if rc_active: + try: + routing_plan = _build_routing_plan( + rc_spec, + num_tokens=int(workload.num_tokens), + world_size=world_size, + top_k=int(model.top_k), + num_experts=int(model.num_experts), + moe_ep_size=int(moe_ep_size), + ) + except Exception as exc: + reason = f"routing plan error: {type(exc).__name__}: {exc}" + _maybe_print_rank0(f"[bench_moe] {reason}") + return _short_circuit(result, "skipped", reason) + per_rank = list(routing_plan.per_rank_num_tokens) + else: + per_rank = _per_rank_tokens(workload, world_size) + + return int(moe_ep_size), per_rank, routing_plan + + +def _make_candidate_input_seed( + *, + random_seed: int, + rank: int, + model: ModelSpec, + workload: WorkloadSpec, + per_rank: List[int], + local_num_tokens: int, +) -> int: + """Build a stable per-workload input seed shared across runtime candidates.""" + payload = { + "random_seed": int(random_seed), + "rank": int(rank), + "model": model.to_dict(), + "workload": workload.to_dict(per_rank_num_tokens=per_rank), + "local_num_tokens": int(local_num_tokens), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + digest = hashlib.blake2b(encoded, digest_size=8).digest() + return int.from_bytes(digest, byteorder="little") & ((1 << 63) - 1) + + +def _run_one_candidate( + *, + model: ModelSpec, + workload: WorkloadSpec, + config: ConfigSpec, + world_size: int, + rank: int, + device: torch.device, + act_dtype: torch.dtype, + routing_logits_dtype: torch.dtype, + warmup: int, + iters: int, + fast_autotune: bool, + analysis: Tuple[str, ...], + cupti_ctx: Optional[Any], + random_seed: int, + input_cache: Optional[_InputCache], + enable_perfect_router_requested: bool, +) -> RunResult: + """Build, autotune, and time one ``ConfigSpec`` candidate. + + Always returns a ``RunResult``; failures are encoded in the ``status`` / + ``skip_reason`` fields so the caller can write a row even for a failed + case. Every early-return path goes through ``_short_circuit`` so the + per-rank status allgather completes on every rank. + + Pipeline: + Step 1 Resolve EP/TP layout and routing plan (if routing-control) + Step 2 Build mapping, configure AutoTuner, force comm method + Step 3 Build the MoE module and validate the actual backend + Step 4 Synthesise inputs and (if routing-control) pick routing inputs + Step 5 Run autotune and time the forward (eager or CUDA graph) + Step 6 Aggregate latency / kernels / bottleneck and routing observation + """ + result = RunResult(model=model, workload=workload, config=config) + rc_spec = workload.routing_control + rc_active = rc_spec.is_active + + # ---- Step 1: layout + routing plan ---------------------------------- + layout = _resolve_layout_and_plan( + result=result, + model=model, + workload=workload, + config=config, + world_size=world_size, + rc_spec=rc_spec, + rc_active=rc_active, + ) + if isinstance(layout, RunResult): + return layout + moe_ep_size, per_rank, routing_plan = layout + + result.per_rank_num_tokens = list(per_rank) + local_num_tokens = per_rank[rank] if rank < len(per_rank) else 0 + all_rank_num_tokens = list(per_rank) + + result.instrumentation = _initial_instrumentation(analysis, config, cupti_ctx) + + # ---- Step 2: mapping + AutoTuner + comm env ------------------------- + try: + mapping = _build_mapping_from_config(config, world_size) + except ValueError as exc: + return _short_circuit(result, "skipped", str(exc)) + + result.moe_ep_size = int(mapping.moe_ep_size) + result.moe_tp_size = int(mapping.moe_tp_size) + result.enable_attention_dp = bool(mapping.enable_attention_dp) + + AutoTuner.get().setup_distributed_state(mapping) + AutoTuner.get().clear_cache() + + prev_force_comm = os.environ.get("TRTLLM_FORCE_COMM_METHOD") + _force_comm_env(config.comm_method, prev_force_comm) + + enable_perfect_router = _pick_enable_perfect_router( + rc_spec, bool(enable_perfect_router_requested) + ) + + moe = None + try: + # ---- Step 3: build MoE module and validate ---------------------- + try: + moe, _ = _build_moe_module( + model=model, + config=config, + mapping=mapping, + moe_backend=config.backend, + use_cuda_graph=bool(config.cuda_graph), + max_num_tokens=max(int(local_num_tokens), 1), + use_low_precision_moe_combine=bool(config.use_low_precision_moe_combine), + enable_perfect_router=enable_perfect_router, + dtype=act_dtype, + routing_logits_dtype=routing_logits_dtype, + device=device, + ) + except Exception as exc: + reason = f"build error: {type(exc).__name__}: {exc}" + _maybe_print_rank0(f"[bench_moe] build failed: {reason}") + return _short_circuit(result, "failed", reason) + + result.actual_backend = _backend_name_from_module(moe) + result.scheduler_kind = _scheduler_kind_name(moe) + result.actual_comm_method = _comm_method_name(moe) + result.num_chunks = _calculate_num_chunks_safe(moe, all_rank_num_tokens) + + if result.actual_backend != config.backend.upper(): + reason = f"requested backend {config.backend!r} fell back to {result.actual_backend!r}" + _maybe_print_rank0(f"[bench_moe] {reason}") + return _short_circuit(result, "skipped", reason) + + # ---- Step 4: synthetic inputs + routing-control routing inputs -- + input_seed = _make_candidate_input_seed( + random_seed=int(random_seed), + rank=rank, + model=model, + workload=workload, + per_rank=per_rank, + local_num_tokens=local_num_tokens, + ) + result.instrumentation["input_seed"] = int(input_seed) + x, router_logits = _make_inputs( + local_num_tokens, + model.hidden_size, + model.num_experts, + act_dtype, + routing_logits_dtype, + device, + seed=input_seed, + cache=input_cache, + ) + + routing_inputs: Optional[_RoutingInputs] = None + if rc_active and routing_plan is not None: + routing_inputs, rc_skip = _select_routing_inputs( + moe=moe, + model=model, + rc_spec=rc_spec, + routing_plan=routing_plan, + rank=rank, + moe_ep_size=int(moe_ep_size), + base_router_logits=router_logits, + device=device, + act_dtype=act_dtype, + routing_logits_dtype=routing_logits_dtype, + ) + if routing_inputs is None: + assert rc_skip is not None + _maybe_print_rank0(f"[bench_moe] {rc_skip.skip_reason}") + # projection_policy=reject: keep a routing_control block on the + # row so dashboards still see the rejected case. + if rc_skip.rejected_reason is not None: + result.routing_control = _build_empty_routing_control_block_for_rejection( + rc_spec=rc_spec, + routing_plan=routing_plan, + model=model, + moe_ep_size=int(moe_ep_size), + per_rank=per_rank, + num_chunks=result.num_chunks, + rejected_reason=rc_skip.rejected_reason, + enable_perfect_router=enable_perfect_router, + act_dtype=act_dtype, + ) + return _short_circuit(result, "skipped", rc_skip.skip_reason) + router_logits = routing_inputs.router_logits + + materialized_ids = routing_inputs.materialized_ids if routing_inputs else None + materialized_scales = routing_inputs.materialized_scales if routing_inputs else None + + # ---- Step 5: autotune + timed forward --------------------------- + with _maybe_install_routing_control_patch( + moe, + materialized_ids, + materialized_scales, + active=(rc_active and rc_spec.routing_mode == "forced"), + ): + try: + autotune_status = _run_autotune( + moe, x, router_logits, all_rank_num_tokens, bool(fast_autotune) + ) + except Exception as exc: + autotune_status = f"failed:{type(exc).__name__}: {exc}" + _maybe_print_rank0(f"[bench_moe] autotune skipped: {type(exc).__name__}: {exc}") + result.instrumentation["autotune_status"] = autotune_status + + try: + if config.cuda_graph: + fwd_times_ms, detailed_stats = _time_moe_forward_cuda_graph( + moe, + x, + router_logits, + all_rank_num_tokens, + warmup=int(warmup), + iters=int(iters), + cupti_ctx=cupti_ctx, + ) + else: + fwd_times_ms, detailed_stats = _time_moe_forward_eager( + moe, + x, + router_logits, + all_rank_num_tokens, + warmup=int(warmup), + iters=int(iters), + collect_kernels="kernels" in analysis, + ) + except Exception as exc: + reason = f"timed phase error: {type(exc).__name__}: {exc}" + _maybe_print_rank0(f"[bench_moe] {reason}\n{traceback.format_exc()}") + return _short_circuit(result, "failed", reason) + + # ---- Step 6: aggregate latency, kernels, routing observation ---- + # The comm factory may swap moe.comm to AllGatherReduceScatter inside + # dispatch, so refresh ``actual_comm_method`` after the first forward. + result.actual_comm_method = _comm_method_name(moe) + + per_rank_iters = _gather_per_iteration_times(fwd_times_ms) + result.latency_ms = _build_latency_block(per_rank_iters) + + if "kernels" in analysis: + result.kernel_breakdown, raw_kernel_times = _gather_kernel_timing_blocks(detailed_stats) + else: + result.kernel_breakdown = {"moe_forward_kernels": [], "other_kernels": []} + raw_kernel_times = {"moe_forward_kernels": [], "other_kernels": []} + result.raw_data = _build_raw_data_block(per_rank_iters, raw_kernel_times) + + # Phase markers live in moe_scheduler.py (Phase 5 of the design); not + # implemented yet, so emit an empty agg/per_rank with a stable shape. + result.phase_times_ms = {"agg": {}, "per_rank": {}} + result.overlap = {"overlap_ms": None, "overlap_ratio": None} + result.bottleneck = _classify_bottleneck( + result.phase_times_ms["agg"], result.kernel_breakdown, result.latency_ms["score"] + ) + + if rc_active and routing_plan is not None and routing_inputs is not None: + _finalize_routing_control_block( + result=result, + rc_spec=rc_spec, + routing_plan=routing_plan, + routing_inputs=routing_inputs, + model=model, + moe_ep_size=int(moe_ep_size), + per_rank=per_rank, + enable_perfect_router=enable_perfect_router, + act_dtype=act_dtype, + ) + + result.status_per_rank = _gather_status_per_rank("success") + return result + finally: + # Always free GPU memory and restore the per-case env var so the next + # candidate runs from a clean state. + if moe is not None: + try: + moe.destroy() + except Exception: + pass + if prev_force_comm is None: + os.environ.pop("TRTLLM_FORCE_COMM_METHOD", None) + else: + os.environ["TRTLLM_FORCE_COMM_METHOD"] = prev_force_comm diff --git a/tests/microbenchmarks/bench_moe/cli.py b/tests/microbenchmarks/bench_moe/cli.py new file mode 100644 index 000000000000..8cefcfe4f902 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/cli.py @@ -0,0 +1,745 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI and config-file resolution for the MoE microbenchmark.""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import torch + +from tensorrt_llm._torch.modules.fused_moe.routing import DeepSeekV3MoeRoutingMethod +from tensorrt_llm.models.modeling_utils import QuantAlgo + +from .backend import MoeBackendType +from .routing import _per_rank_tokens +from .search import ( + _coerce_str_tuple, + _maybe_auto_enable_search_axes, + _parse_analysis, + _parse_search_axes, + _resolve_search_from_args, +) +from .specs import ( + _ALL_BACKENDS, + _COMM_METHODS, + _ROUTING_METHODS, + BUILT_IN_MODELS, + ConfigSpec, + ModelSpec, + RoutingControlSpec, + SearchSpec, + WorkloadSpec, +) + + +@dataclass(frozen=True) +class _BenchmarkContext: + model: ModelSpec + workloads: List[WorkloadSpec] + base_config: ConfigSpec + search: SearchSpec + analysis: Tuple[str, ...] + act_dtype: torch.dtype + routing_logits_dtype: torch.dtype + + +def _resolve_benchmark_context(args: argparse.Namespace) -> _BenchmarkContext: + analysis = _parse_analysis(args.analysis) + model = _resolve_model_from_args(args) + workloads = _resolve_workloads_from_args(args) + base_config = _resolve_base_config_from_args(args) + search = _resolve_search_from_args(args, base_config) + act_dtype = torch.bfloat16 if args.dtype == "bfloat16" else torch.float16 + routing_logits_dtype = ( + torch.float32 if model.routing_method_cls is DeepSeekV3MoeRoutingMethod else act_dtype + ) + return _BenchmarkContext( + model=model, + workloads=workloads, + base_config=base_config, + search=search, + analysis=analysis, + act_dtype=act_dtype, + routing_logits_dtype=routing_logits_dtype, + ) + + +def _build_worker_header(ctx: _BenchmarkContext, launcher: str, world_size: int) -> Dict[str, Any]: + return { + "benchmark": "bench_moe", + "launcher": launcher, + "model": ctx.model.to_dict(), + "search": ctx.search.to_dict(), + "world_size": world_size, + "analysis": list(ctx.analysis) or ["summary"], + "workloads": [ + w.to_dict(per_rank_num_tokens=_per_rank_tokens(w, world_size)) for w in ctx.workloads + ], + "base_config": ctx.base_config.to_dict(), + } + + +def _add_search_arguments(parser: argparse.ArgumentParser) -> None: + group = parser.add_argument_group("Search and sweep") + group.add_argument( + "--search", + type=lambda s: str(s).lower(), + nargs="+", + default=("none",), + help=( + "Expand one or more runtime axes. Examples: --search backend --backend ALL; " + "--search backend comm; --search full. Comma-separated input is also accepted." + ), + ) + group.add_argument( + "--max_configs", + type=int, + default=None, + help="Run at most this many valid candidate configs after pruning. Example: --max_configs 32.", + ) + group.add_argument( + "--time_budget_minutes", + type=float, + default=None, + help="Stop launching new candidates after this wall-clock budget. Example: --time_budget_minutes 30.", + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="MoE module microbenchmark (MPI). Times ConfigurableMoE.forward.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + launch_group = parser.add_argument_group("Launch") + launch_group.add_argument( + "--world_size", + type=int, + default=None, + help=( + "Number of ranks to run. Under external mpirun/srun, this must match " + "the external MPI world size; without an external launcher, values >1 " + "use the self-spawn launcher." + ), + ) + + model_group = parser.add_argument_group("Model and shape") + model_group.add_argument( + "--model", + type=str, + default=None, + choices=sorted(BUILT_IN_MODELS.keys()), + help=( + "Built-in model shape. Examples: deepseek_v3, qwen1.5_moe. " + "Omit only when passing all custom shape fields below." + ), + ) + model_group.add_argument( + "--num_experts", + type=int, + default=None, + help="Custom-shape total expert count. Required when --model is omitted.", + ) + model_group.add_argument( + "--top_k", + type=int, + default=None, + help="Custom-shape experts selected per token. Required when --model is omitted.", + ) + model_group.add_argument( + "--hidden_size", + type=int, + default=None, + help="Custom-shape hidden size. Required when --model is omitted.", + ) + model_group.add_argument( + "--intermediate_size", + type=int, + default=None, + help="Custom-shape MoE intermediate size. Required when --model is omitted.", + ) + model_group.add_argument( + "--n_group", + type=int, + default=None, + help="DeepSeek-style routing group count for custom grouped routing.", + ) + model_group.add_argument( + "--topk_group", + type=int, + default=None, + help="DeepSeek-style number of routing groups kept per token.", + ) + model_group.add_argument( + "--quant", + type=lambda s: QuantAlgo[str(s).upper()] if s is not None else None, + default=None, + choices=[q.name for q in QuantAlgo], + help="Quantization algorithm. Example: --quant FP8_BLOCK_SCALES.", + ) + model_group.add_argument( + "--routing_method", + type=lambda s: str(s).upper(), + default="AUTO", + choices=sorted(_ROUTING_METHODS) + ["AUTO"], + help=( + "Routing method. Defaults to AUTO: built-in models use the spec " + "default; custom shapes must specify an explicit method." + ), + ) + + workload_group = parser.add_argument_group("Workload shape") + workload_group.add_argument( + "--balanced_total_num_tokens", + "--num_tokens", + dest="balanced_total_num_tokens", + type=int, + nargs="+", + required=False, + help=( + "Global token counts to sweep. Each value is balanced across ranks " + "with any remainder on rank 0. Example: --balanced_total_num_tokens 64 256 1024." + ), + ) + + routing_group = parser.add_argument_group("Routing control") + routing_group.add_argument( + "--routing_mode", + type=lambda s: str(s).lower(), + default="native", + choices=("native", "forced"), + help=( + "native: route through production logits kernels (default); " + "forced: supply top-k ids/scales directly (skips fused scoring)." + ), + ) + routing_group.add_argument( + "--projection_policy", + type=lambda s: str(s).lower(), + default="project", + choices=("project", "reject"), + help=( + "project: when native logits cannot exactly realise the plan, run with " + "the closest legal projection and warn; reject: skip the case instead." + ), + ) + routing_group.add_argument( + "--comm_pattern", + type=str, + default="balanced_alltoall", + help=( + "Source-to-target slot dispatch pattern. balanced_alltoall builds a balanced " + "plan by default; random keeps logits uncontrolled when expert_pattern is also random. " + "Examples: balanced_alltoall, random, " + "receiver_hotspot,hotness=0.75,rank=0, pair_hotspot,hotness=0.5,src=0,dst=1, " + "local_only, ring." + ), + ) + routing_group.add_argument( + "--expert_pattern", + type=str, + default="balanced", + help=( + "Per-target-rank local expert histogram pattern. balanced builds a balanced " + "plan by default; random keeps logits uncontrolled when comm_pattern is also random. " + "Examples: balanced, random, " + "hotspot,hotness=0.5, hotspot,active_experts=2." + ), + ) + routing_group.add_argument( + "--routing_pattern_file", + type=str, + default=None, + help="JSON file that provides both slot_dispatch_matrix and expert_histogram.", + ) + routing_group.add_argument( + "--per_rank_num_tokens", + type=int, + nargs="+", + default=None, + help=( + "Explicit per-rank input token counts. Length must equal world_size " + "and sum defines the workload total. Mutually exclusive with --balanced_total_num_tokens." + ), + ) + routing_group.add_argument( + "--routing_dump_matrix", + action="store_true", + help="Include the full observed slot/token matrix and expert histogram in each result row.", + ) + routing_group.add_argument( + "--routing_seed", + type=int, + default=0, + help="Seed for deterministic routing-plan materialisation; independent from --random_seed.", + ) + routing_group.add_argument( + "--enable_perfect_router", + action="store_true", + help=( + "Use the lower-level ENABLE_PERFECT_ROUTER path to replace router logits with " + "load-balanced logits inside the MoE module. Disabled by default so routing-control " + "patterns use bench_moe's own projected logits." + ), + ) + + parallel_group = parser.add_argument_group("Parallel layout") + parallel_group.add_argument( + "--parallel_mode", + type=str, + nargs="+", + default=("DEP",), + choices=("DEP", "TEP", "DTP", "TTP", "CUSTOM"), + help=( + "Parallel layout(s) to benchmark. Pass multiple values to sweep, e.g. " + "--parallel_mode DEP TEP. DEP=attention DP + MoE EP; TEP=attention TP + " + "MoE EP; DTP/TTP use MoE TP; CUSTOM requires --moe_ep_size and " + "--moe_tp_size and must be passed alone." + ), + ) + parallel_group.add_argument( + "--moe_ep_size", + type=int, + default=None, + help="CUSTOM only: MoE expert-parallel size. Must multiply with --moe_tp_size to world_size.", + ) + parallel_group.add_argument( + "--moe_tp_size", + type=int, + default=None, + help="CUSTOM only: MoE tensor-parallel size. Must multiply with --moe_ep_size to world_size.", + ) + parallel_group.add_argument( + "--enable_attention_dp", + action="store_true", + help="CUSTOM only: enable attention data parallelism for the mapping.", + ) + + runtime_group = parser.add_argument_group("Runtime backend and communication") + runtime_group.add_argument( + "--backend", + type=lambda s: str(s).upper(), + nargs="+", + default=("TRTLLM",), + choices=_ALL_BACKENDS + ["ALL"], + help=( + "MoE backend(s) to benchmark. Pass multiple values to sweep, e.g. " + "--backend CUTLASS DEEPGEMM. ALL expands to every ConfigurableMoE-eligible " + "backend and must be passed alone. Passing >1 value (or ALL) implicitly " + "enables --search backend." + ), + ) + runtime_group.add_argument( + "--comm_method", + type=lambda s: str(s).upper(), + nargs="+", + default=("AUTO",), + choices=_COMM_METHODS, + help=( + "Communication method(s) to benchmark. Pass multiple values to sweep, " + "e.g. --comm_method NVLINK_ONE_SIDED DEEPEP. AUTO lets TensorRT-LLM " + "select; other values force a specific path. Passing >1 value implicitly " + "enables --search comm." + ), + ) + + timing_group = parser.add_argument_group("Timing") + timing_group.add_argument( + "--no_cuda_graph", + dest="cuda_graph", + action="store_false", + default=True, + help="Disable CUDA-Graph capture and use eager timing.", + ) + timing_group.add_argument("--warmup", type=int, default=1, help="Warmup iterations per case.") + timing_group.add_argument("--iters", type=int, default=12, help="Timed iterations per case.") + timing_group.add_argument( + "--fast_autotune", + action="store_true", + help="Use a short autotune pass for smoke tests; may reduce measurement quality.", + ) + timing_group.add_argument( + "--per_candidate_timeout_s", + type=float, + default=0.0, + help=( + "Hard wall-clock budget per candidate (seconds). If a candidate exceeds " + "this, a watchdog thread sends SIGKILL to break suspected NCCL deadlocks " + "or CUDA hangs that ``torch.cuda.synchronize()`` cannot detect. The killed " + "candidate is missing from the checkpoint JSON, so an outer driver that " + "restarts with --resume_from will re-attempt it. 0 disables the watchdog." + ), + ) + timing_group.add_argument( + "--dtype", + type=str, + default="bfloat16", + choices=("bfloat16", "float16"), + help="Activation dtype for synthetic inputs.", + ) + timing_group.add_argument( + "--use_low_precision_moe_combine", + action="store_true", + help="Use low-precision combine where the selected backend supports it.", + ) + timing_group.add_argument( + "--random_seed", + type=int, + default=1234, + help="Seed for synthetic hidden states/router logits; routing-control plans use --routing_seed.", + ) + + analysis_group = parser.add_argument_group("Analysis") + analysis_group.add_argument( + "--analysis", + nargs="+", + default=("kernels",), + choices=("none", "kernels"), + help="Analysis data to collect. Use --analysis none for latency-only output.", + ) + + # ---- Search / sweep ---- + _add_search_arguments(parser) + + output_group = parser.add_argument_group("Output") + output_group.add_argument( + "-c", + "--config_file", + type=str, + default=None, + help="JSON config file. CLI flags override matching config-file fields.", + ) + output_group.add_argument( + "-o", + "--output_file", + type=str, + default=None, + help="Write the final dashboard JSON report to this path.", + ) + output_group.add_argument( + "--analysis_workbook_file", + type=str, + default=None, + help=( + "Write an Excel workbook with all candidate rows, per-workload sheets, " + "best configs, and status summaries. Defaults to .analysis.xlsx." + ), + ) + output_group.add_argument( + "--resume_from", + type=str, + default=None, + help=( + "Read an existing JSON report and skip every (workload, config) candidate " + "whose row is terminal (success/failed, or skipped for a non-upstream reason). " + "Placeholder rows left behind by a prior crash (skip_reason ending in " + "'_upstream') are dropped and re-attempted. Combine with --output_file to " + "write fresh results back into the same file (atomic, checkpointed after " + "every candidate)." + ), + ) + output_group.add_argument( + "--checkpoint_every", + type=int, + default=1, + help=( + "Write the --output_file JSON checkpoint after every N freshly completed " + "candidates. 0 disables incremental checkpointing (only the final JSON is " + "written). Default 1 trades a small amount of JSON I/O for crash-safety: " + "no completed candidate is ever lost to a watchdog SIGKILL or sticky-error " + "exit." + ), + ) + args = parser.parse_args() + provided = set() + argv = sys.argv[1:] + for action in parser._actions: + for option in action.option_strings: + if option in argv or any(arg.startswith(option + "=") for arg in argv): + provided.add(action.dest) + args._cli_provided = provided + args.search = _parse_search_axes(args.search) + _maybe_auto_enable_search_axes(args) + return args + + +def _resolve_model_from_args(args: argparse.Namespace) -> ModelSpec: + base = BUILT_IN_MODELS.get(args.model) if args.model is not None else None + + routing = args.routing_method + if base is None: + # Custom shape requires explicit fields. + missing = [ + f + for f in ("num_experts", "top_k", "hidden_size", "intermediate_size") + if getattr(args, f) is None + ] + if missing: + raise ValueError("No --model selected; you must also pass: " + ", ".join(missing)) + if routing == "AUTO": + raise ValueError( + "Custom shapes (no --model) require an explicit --routing_method; " + "AUTO has no safe default." + ) + quant_name = args.quant.name if args.quant is not None else None + return ModelSpec( + name="custom", + num_experts=int(args.num_experts), + top_k=int(args.top_k), + hidden_size=int(args.hidden_size), + intermediate_size=int(args.intermediate_size), + quant_algo=quant_name, + routing_method=routing, + n_group=args.n_group, + topk_group=args.topk_group, + ) + + # Built-in model with optional per-field overrides. + if routing == "AUTO": + routing = base.routing_method + + quant_name: Optional[str] + if args.quant is not None: + quant_name = args.quant.name + else: + quant_name = base.quant_algo + return ModelSpec( + name=base.name, + num_experts=int(args.num_experts) if args.num_experts is not None else base.num_experts, + top_k=int(args.top_k) if args.top_k is not None else base.top_k, + hidden_size=int(args.hidden_size) if args.hidden_size is not None else base.hidden_size, + intermediate_size=int(args.intermediate_size) + if args.intermediate_size is not None + else base.intermediate_size, + quant_algo=quant_name, + routing_method=routing, + n_group=args.n_group if args.n_group is not None else base.n_group, + topk_group=args.topk_group if args.topk_group is not None else base.topk_group, + swiglu_alpha=base.swiglu_alpha, + swiglu_beta=base.swiglu_beta, + swiglu_limit=base.swiglu_limit, + ) + + +def _resolve_workloads_from_args(args: argparse.Namespace) -> List[WorkloadSpec]: + balanced_total_num_tokens = getattr(args, "balanced_total_num_tokens", None) + if balanced_total_num_tokens is None: + balanced_total_num_tokens = getattr(args, "num_tokens", None) + + # Resolve RoutingControlSpec from explicit CLI/config fields. + per_rank_num_tokens: Optional[Tuple[int, ...]] = None + if getattr(args, "per_rank_num_tokens", None): + if balanced_total_num_tokens: + raise ValueError( + "--balanced_total_num_tokens and --per_rank_num_tokens are mutually exclusive" + ) + raw = args.per_rank_num_tokens + if isinstance(raw, str): + try: + parts = [int(p.strip()) for p in raw.split(",") if p.strip()] + except ValueError as exc: + raise ValueError(f"--per_rank_num_tokens must be integers; got {raw!r}") from exc + else: + parts = [int(v) for v in raw] + per_rank_num_tokens = tuple(parts) + if any(v < 0 for v in per_rank_num_tokens): + raise ValueError("--per_rank_num_tokens entries must be >= 0") + token_values = [sum(per_rank_num_tokens)] + else: + if not balanced_total_num_tokens: + raise ValueError( + "--balanced_total_num_tokens or --per_rank_num_tokens is required " + "(or supply via --config_file)" + ) + token_values = [int(t) for t in balanced_total_num_tokens] + if any(t < 0 for t in token_values): + raise ValueError("--balanced_total_num_tokens entries must be >= 0") + + routing_spec = RoutingControlSpec( + routing_mode=str(getattr(args, "routing_mode", "native")), + projection_policy=str(getattr(args, "projection_policy", "project")), + comm_pattern=str(getattr(args, "comm_pattern", "balanced_alltoall")), + expert_pattern=str(getattr(args, "expert_pattern", "balanced")), + routing_pattern_file=getattr(args, "routing_pattern_file", None), + per_rank_num_tokens=per_rank_num_tokens, + routing_dump_matrix=bool(getattr(args, "routing_dump_matrix", False)), + seed=int(getattr(args, "routing_seed", 0)), + ) + + return [ + WorkloadSpec( + num_tokens=int(t), + routing_control=routing_spec, + ) + for t in token_values + ] + + +def _resolve_base_config_from_args(args: argparse.Namespace) -> ConfigSpec: + # ``--backend``/``--comm_method``/``--parallel_mode`` are ``nargs="+"`` lists. + # The base config holds a single placeholder value; ``_resolve_search_from_args`` + # is responsible for expanding the actual sweep set per axis. + backends_list = _coerce_str_tuple(args.backend) + comm_list = _coerce_str_tuple(args.comm_method) + parallel_list = _coerce_str_tuple(args.parallel_mode) + + comm_method = comm_list[0] if comm_list else "AUTO" + + backend = backends_list[0] if backends_list else MoeBackendType.CUTLASS.value + if backend == "ALL": + backend = MoeBackendType.CUTLASS.value # placeholder; overwritten by search expansion + + parallel_mode = parallel_list[0] if parallel_list else "DEP" + + # parallel_mode CUSTOM if explicit EP/TP overrides are present. + if (args.moe_ep_size is not None or args.moe_tp_size is not None) and parallel_mode in ( + "DEP", + "TEP", + "DTP", + "TTP", + ): + # Treat explicit overrides as opting into CUSTOM so output metadata is honest. + parallel_mode = "CUSTOM" + + if parallel_mode == "CUSTOM" and (args.moe_ep_size is None or args.moe_tp_size is None): + raise ValueError("--parallel_mode=CUSTOM requires both --moe_ep_size and --moe_tp_size") + + return ConfigSpec( + backend=backend, + parallel_mode=parallel_mode, + moe_ep_size=args.moe_ep_size, + moe_tp_size=args.moe_tp_size, + enable_attention_dp=bool(args.enable_attention_dp) if parallel_mode == "CUSTOM" else None, + comm_method=comm_method, + cuda_graph=bool(args.cuda_graph), + use_low_precision_moe_combine=bool(args.use_low_precision_moe_combine), + ) + + +def _maybe_load_config_file(args: argparse.Namespace) -> argparse.Namespace: + """Overlay ``--config_file`` JSON onto ``args``; explicit CLI flags win.""" + if not args.config_file: + args._config_search_axes = {} + return args + with open(args.config_file) as f: + cfg = json.load(f) + + provided = set(getattr(args, "_cli_provided", set())) + + def set_if_unset(dest: str, value: Any) -> None: + if dest not in provided: + setattr(args, dest, value) + + if "model" in cfg: + set_if_unset("model", cfg["model"]) + workload_cfg = cfg.get("workload", {}) or {} + if "balanced_total_num_tokens" in workload_cfg: + set_if_unset("balanced_total_num_tokens", list(workload_cfg["balanced_total_num_tokens"])) + elif "num_tokens" in workload_cfg: + set_if_unset("balanced_total_num_tokens", list(workload_cfg["num_tokens"])) + if "per_rank_num_tokens" in workload_cfg: + prnt = workload_cfg["per_rank_num_tokens"] + set_if_unset( + "per_rank_num_tokens", + [int(v) for v in prnt] if isinstance(prnt, list) else str(prnt), + ) + routing_cfg = workload_cfg.get("routing_control", {}) or {} + if "routing_mode" in routing_cfg: + set_if_unset("routing_mode", str(routing_cfg["routing_mode"]).lower()) + if "projection_policy" in routing_cfg: + set_if_unset("projection_policy", str(routing_cfg["projection_policy"]).lower()) + if "comm_pattern" in routing_cfg: + set_if_unset("comm_pattern", str(routing_cfg["comm_pattern"])) + if "expert_pattern" in routing_cfg: + set_if_unset("expert_pattern", str(routing_cfg["expert_pattern"])) + if "routing_pattern_file" in routing_cfg: + set_if_unset("routing_pattern_file", str(routing_cfg["routing_pattern_file"])) + if "routing_dump_matrix" in routing_cfg: + set_if_unset("routing_dump_matrix", bool(routing_cfg["routing_dump_matrix"])) + if "seed" in routing_cfg: + set_if_unset("routing_seed", int(routing_cfg["seed"])) + search_cfg = cfg.get("search", {}) or {} + unsupported_search_keys = set(search_cfg) - {"backend", "parallel_mode", "comm_method"} + if unsupported_search_keys: + raise ValueError(f"unsupported search key(s): {sorted(unsupported_search_keys)}") + config_search_axes: Dict[str, Tuple[str, ...]] = {} + + def normalize_search_axis(key: str, value: Any) -> Optional[Tuple[str, ...]]: + """Project a config-file search-axis entry onto ``args`` (list form). + + Returns the tuple of canonical values if it should be recorded as a + sweep axis in ``_config_search_axes`` (multi-value or backend=ALL). + Returns ``None`` when the value was instead written to the matching + ``args.`` scalar-list flag. + """ + if key in provided: + return None + values = ( + tuple(str(v).upper() for v in value) + if isinstance(value, list) + else (str(value).upper(),) + ) + # ``backend=ALL`` is the explicit "expand to every backend" sentinel. + if key == "backend" and values == ("ALL",): + set_if_unset("backend", ("ALL",)) + return None + if len(values) == 1: + # Single-value config entry behaves like a CLI scalar default. + set_if_unset(key, values) + return None + return values + + if search_cfg: + for key in ("backend", "parallel_mode", "comm_method"): + if key in search_cfg: + axis = normalize_search_axis(key, search_cfg[key]) + if axis: + config_search_axes[key] = axis + if "search" not in provided: + # Merge config-provided multi-value axes into any axes the CLI + # auto-promoted (e.g. via multi-value --backend). Without merging, + # config-driven axes would clobber CLI auto-promoted ones. + existing = tuple(args.search) if args.search and args.search != ("none",) else () + extra: List[str] = [] + if "backend" in search_cfg and "backend" not in provided: + extra.append("backend") + if "parallel_mode" in search_cfg and "parallel_mode" not in provided: + extra.append("parallel") + if "comm_method" in search_cfg and "comm_method" not in provided: + extra.append("comm") + if extra: + merged: List[str] = [a for a in existing if a not in ("none",)] + for axis in extra: + if axis not in merged: + merged.append(axis) + if merged: + args.search = tuple(merged) + args._config_search_axes = config_search_axes + if "analysis" in cfg: + set_if_unset("analysis", cfg["analysis"]) + if "max_configs" in cfg: + set_if_unset("max_configs", int(cfg["max_configs"])) + if "time_budget_minutes" in cfg: + set_if_unset("time_budget_minutes", float(cfg["time_budget_minutes"])) + if "output_file" in cfg: + set_if_unset("output_file", cfg["output_file"]) + if "analysis_workbook_file" in cfg: + set_if_unset("analysis_workbook_file", cfg["analysis_workbook_file"]) + return args diff --git a/tests/microbenchmarks/bench_moe/mapping.py b/tests/microbenchmarks/bench_moe/mapping.py new file mode 100644 index 000000000000..5074fe0da437 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/mapping.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Construction of TRT-LLM Mapping / ModelConfig / RoutingMethod objects. + +The benchmark plumbs three TRT-LLM-internal config objects into +``create_moe``: + +* :class:`Mapping` — TP / EP / attention-DP topology derived from the + :class:`ConfigSpec` parallel-mode shortcut (``DEP`` / ``TEP`` / ``DTP`` + / ``TTP`` / ``CUSTOM``). +* :class:`ModelConfig` — ``Mapping`` + quant config + per-run knobs + (CUDA graph, max num tokens, low-precision combine). +* A concrete routing-method instance — picked by the registry value + recorded on :class:`ModelSpec`. + +Centralising this construction in one module makes the per-case execution +path easier to follow and keeps the ``ConfigurableMoE`` glue out of +the worker and CLI layers. +""" + +from __future__ import annotations + +from typing import Any, Dict, Tuple + +import torch +from transformers.configuration_utils import PretrainedConfig + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modules.fused_moe.routing import ( + DeepSeekV3MoeRoutingMethod, + DefaultMoeRoutingMethod, + Llama4RenormalizeMoeRoutingMethod, + MiniMaxM2MoeRoutingMethod, + RenormalizeMoeRoutingMethod, + RenormalizeNaiveMoeRoutingMethod, + SigmoidRenormMoeRoutingMethod, +) +from tensorrt_llm._utils import mpi_rank +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantConfig + +from .backend import MoeModelConfig, resolve_deepseek_group_config +from .specs import ConfigSpec, ModelSpec + +_PARALLEL_MODE_LAYOUTS: Dict[str, Dict[str, Any]] = { + "DEP": {"moe_ep_size": "world", "moe_tp_size": 1, "enable_attention_dp": True}, + "TEP": {"moe_ep_size": "world", "moe_tp_size": 1, "enable_attention_dp": False}, + "DTP": {"moe_ep_size": 1, "moe_tp_size": "world", "enable_attention_dp": True}, + "TTP": {"moe_ep_size": 1, "moe_tp_size": "world", "enable_attention_dp": False}, +} + + +def _resolve_mapping_layout(config: ConfigSpec, world_size: int) -> Tuple[int, int, bool]: + """Resolve ``(moe_ep_size, moe_tp_size, enable_attention_dp)`` for a ConfigSpec.""" + if config.parallel_mode == "CUSTOM": + if config.moe_ep_size is None or config.moe_tp_size is None: + raise ValueError("parallel_mode=CUSTOM requires explicit moe_ep_size and moe_tp_size") + moe_ep = int(config.moe_ep_size) + moe_tp = int(config.moe_tp_size) + enable_dp = ( + bool(config.enable_attention_dp) if config.enable_attention_dp is not None else False + ) + else: + layout = _PARALLEL_MODE_LAYOUTS.get(config.parallel_mode) + if layout is None: + raise ValueError(f"Unknown parallel_mode={config.parallel_mode!r}") + moe_ep = world_size if layout["moe_ep_size"] == "world" else int(layout["moe_ep_size"]) + moe_tp = world_size if layout["moe_tp_size"] == "world" else int(layout["moe_tp_size"]) + enable_dp = bool(layout["enable_attention_dp"]) + if moe_ep * moe_tp != world_size: + raise ValueError( + f"moe_ep_size * moe_tp_size = {moe_ep * moe_tp} must equal world_size={world_size}" + ) + return moe_ep, moe_tp, enable_dp + + +def _build_mapping_from_config(config: ConfigSpec, world_size: int) -> Mapping: + """Build ``Mapping`` from a ``ConfigSpec`` + world size; sets ``rank=mpi_rank()``.""" + moe_ep, moe_tp, enable_dp = _resolve_mapping_layout(config, world_size) + mapping = Mapping( + world_size=world_size, + tp_size=world_size, + moe_ep_size=moe_ep, + moe_tp_size=moe_tp, + enable_attention_dp=enable_dp, + ) + mapping.rank = mpi_rank() + return mapping + + +def _create_routing_method( + routing_method_cls, + top_k: int, + num_experts: int, + bias_dtype: torch.dtype, + profile_model_config: MoeModelConfig, +): + """Create a routing-method instance mirroring ``test_moe_module._create_routing_method``.""" + if routing_method_cls in (RenormalizeMoeRoutingMethod, DefaultMoeRoutingMethod): + return routing_method_cls(top_k=top_k, force_enable_pytorch_op=True) + + if routing_method_cls in (RenormalizeNaiveMoeRoutingMethod, Llama4RenormalizeMoeRoutingMethod): + return routing_method_cls(top_k=top_k) + + if routing_method_cls is DeepSeekV3MoeRoutingMethod: + n_group, topk_group = resolve_deepseek_group_config(profile_model_config) + e_score_correction_bias = torch.zeros(num_experts, dtype=bias_dtype, device="cuda") + return routing_method_cls( + top_k=top_k, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=1.0, + callable_e_score_correction_bias=lambda: e_score_correction_bias, + is_fused=False, + ) + + if routing_method_cls is MiniMaxM2MoeRoutingMethod: + e_score_correction_bias = torch.zeros(num_experts, dtype=bias_dtype, device="cuda") + return routing_method_cls( + top_k=top_k, + num_experts=num_experts, + callable_e_score_correction_bias=lambda: e_score_correction_bias, + ) + + if routing_method_cls is SigmoidRenormMoeRoutingMethod: + return routing_method_cls(top_k=top_k, num_experts=num_experts) + + return routing_method_cls(top_k=top_k) + + +def _build_pretrained_config( + num_experts: int, hidden_size: int, intermediate_size: int, dtype: torch.dtype +) -> PretrainedConfig: + """Construct a HF-style ``PretrainedConfig`` for ``ConfigurableMoE``.""" + pc = PretrainedConfig() + pc.num_experts = num_experts + pc.hidden_size = hidden_size + pc.intermediate_size = intermediate_size + pc.torch_dtype = dtype + return pc + + +def _build_model_config( + *, + model: ModelSpec, + mapping: Mapping, + moe_backend: str, + use_cuda_graph: bool, + max_num_tokens: int, + use_low_precision_moe_combine: bool, + dtype: torch.dtype, +) -> ModelConfig: + """Build ``ModelConfig`` plumbed into ``create_moe``.""" + pretrained_config = _build_pretrained_config( + model.num_experts, model.hidden_size, model.intermediate_size, dtype + ) + + quant_algo = model.quant_algo_enum + quant_config = ( + QuantConfig(quant_algo=None) if quant_algo is None else QuantConfig(quant_algo=quant_algo) + ) + + return ModelConfig( + pretrained_config=pretrained_config, + mapping=mapping, + quant_config=quant_config, + moe_backend=moe_backend, + moe_disable_finalize_fusion=False, + max_num_tokens=max(int(max_num_tokens), 1), + use_cuda_graph=use_cuda_graph, + use_low_precision_moe_combine=use_low_precision_moe_combine, + ) diff --git a/tests/microbenchmarks/bench_moe/quantize.py b/tests/microbenchmarks/bench_moe/quantize.py new file mode 100644 index 000000000000..ad6e7cefebe5 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/quantize.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Quantization helper loader for ``bench_moe``. + +The benchmark reuses unittest quantization fixtures for MoE weight generation, +but those fixtures import ``moe_test_utils`` only for a memory skip helper. That +top-level import eagerly loads every backend and can fail when optional CUTEDSL +dependencies are unavailable. This loader provides the one skip helper while +importing the fixture module, then restores the normal module table. +""" + +from __future__ import annotations + +import importlib +import sys +import types + +from .backend import ensure_cute_dsl_importable_for_benchmark + + +def _make_moe_test_utils_stub() -> types.ModuleType: + stub = types.ModuleType("_torch.modules.moe.moe_test_utils") + + def skip_if_insufficient_gpu_memory(*_args, **_kwargs) -> None: + return None + + stub.skip_if_insufficient_gpu_memory = skip_if_insufficient_gpu_memory + return stub + + +def _load_get_test_quant_params(): + ensure_cute_dsl_importable_for_benchmark() + dependency_name = "_torch.modules.moe.moe_test_utils" + original_dependency = sys.modules.get(dependency_name) + inserted_stub = original_dependency is None + if inserted_stub: + sys.modules[dependency_name] = _make_moe_test_utils_stub() + try: + quantize_utils = importlib.import_module("_torch.modules.moe.quantize_utils") + finally: + if inserted_stub: + sys.modules.pop(dependency_name, None) + elif original_dependency is not None: + sys.modules[dependency_name] = original_dependency + return quantize_utils.get_test_quant_params + + +get_test_quant_params = _load_get_test_quant_params() diff --git a/tests/microbenchmarks/bench_moe/reporting.py b/tests/microbenchmarks/bench_moe/reporting.py new file mode 100644 index 000000000000..ec1330172588 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/reporting.py @@ -0,0 +1,765 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import getpass +import json +import os +import platform +import socket +import subprocess +import zipfile +from pathlib import Path +from typing import Any, Optional +from xml.sax.saxutils import escape as _xml_escape + +import torch + + +def build_rankings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Group serialized result rows by workload and rank candidates by score.""" + grouped: dict[tuple[int, str], list[dict[str, Any]]] = {} + for row in rows: + workload = row.get("workload") or {} + requested_cfg = row.get("requested_config") or {} + num_tokens = int(workload.get("num_tokens") or 0) + parallel_mode = str(requested_cfg.get("parallel_mode") or "") + grouped.setdefault((num_tokens, parallel_mode), []).append(row) + + rankings: list[dict[str, Any]] = [] + for (num_tokens, parallel_mode), items in sorted(grouped.items()): + ranking_entries: list[dict[str, Any]] = [] + for row in items: + actual_cfg = row.get("actual_config") or {} + requested_cfg = row.get("requested_config") or {} + instrumentation = row.get("instrumentation") or {} + latency = row.get("latency_ms") or {} + score = latency.get("score") if isinstance(latency, dict) else None + raw_score = latency.get("raw_score") if isinstance(latency, dict) else None + outliers = latency.get("iter_max_outliers") if isinstance(latency, dict) else {} + ranking_entries.append( + { + "backend": actual_cfg.get("backend") or requested_cfg.get("backend"), + "requested_backend": requested_cfg.get("backend"), + "comm_method": actual_cfg.get("comm_method"), + "cuda_graph": requested_cfg.get("cuda_graph"), + "use_low_precision_moe_combine": requested_cfg.get( + "use_low_precision_moe_combine" + ), + "score_ms": float(score) if isinstance(score, (int, float)) else None, + "raw_score_ms": float(raw_score) + if isinstance(raw_score, (int, float)) + else None, + "outlier_count": outliers.get("count") if isinstance(outliers, dict) else None, + "status": row.get("status"), + "skip_reason": row.get("skip_reason"), + "autotune_status": instrumentation.get("autotune_status"), + } + ) + ranking_entries.sort( + key=lambda e: ( + e["score_ms"] is None, + e["score_ms"] if e["score_ms"] is not None else 0.0, + ) + ) + best = next( + ( + e + for e in ranking_entries + if e["score_ms"] is not None + and e["status"] == "success" + and not ( + isinstance(e.get("autotune_status"), str) + and e["autotune_status"].startswith("failed") + ) + ), + None, + ) + rankings.append( + { + "num_tokens": int(num_tokens), + "parallel_mode": parallel_mode, + "best": best, + "ranking": ranking_entries, + } + ) + return rankings + + +def _trtllm_commit(repo_root: Path) -> Optional[str]: + try: + out = subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], + cwd=repo_root, + stderr=subprocess.DEVNULL, + timeout=2.0, + ) + return out.decode().strip() or None + except Exception: + return None + + +def _build_environment_block( + world_size: int, cuda_graph_default: bool, repo_root: Path +) -> dict[str, Any]: + device_name = None + sm = None + if torch.cuda.is_available(): + try: + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + device_name = props.name + sm = int(getattr(props, "major", 0) * 10 + getattr(props, "minor", 0)) + except Exception: + pass + cuda_version = None + try: + cuda_version = torch.version.cuda + except Exception: + pass + driver_version = None + torch_driver_version = getattr(torch.cuda, "_get_driver_version", None) + if torch_driver_version is not None: + try: + driver_version = torch_driver_version() + except Exception: + pass + + try: + host = socket.gethostname() + except Exception: + host = None + try: + user = getpass.getuser() + except Exception: + user = None + + return { + "world_size": int(world_size), + "world_size_per_node": int(min(world_size, max(torch.cuda.device_count(), 1))), + "hostname": host, + "username": user, + "device_name": device_name, + "sm": sm, + "cuda_version": str(cuda_version) if cuda_version else None, + "driver_version": str(driver_version) if driver_version else None, + "torch_version": str(torch.__version__), + "trtllm_commit": _trtllm_commit(repo_root), + "platform": platform.platform(), + "nvlink_topology": "unknown", + "memory_type": "unknown", + "clock_locked": False, + "cuda_graph_default": bool(cuda_graph_default), + } + + +def build_report_payload( + *, + ctx: Any, + rows: list[dict[str, Any]], + world_size: int, + cuda_graph_default: bool, + repo_root: Path, +) -> dict[str, Any]: + """Build the dashboard JSON payload from already-serialized result rows.""" + return { + "benchmark": "bench_moe", + "environment": _build_environment_block(world_size, cuda_graph_default, repo_root), + "model": ctx.model.to_dict(), + "search": ctx.search.to_dict(), + "base_config": ctx.base_config.to_dict(), + "results": list(rows), + "rankings": build_rankings(rows), + } + + +def _excel_safe_sheet_name(name: str, used: set[str]) -> str: + invalid = set("[]:*?/\\") + base = "".join("_" if ch in invalid else ch for ch in str(name)).strip() or "sheet" + base = base[:31] + candidate = base + suffix = 1 + while candidate in used: + tail = f"_{suffix}" + candidate = f"{base[: 31 - len(tail)]}{tail}" + suffix += 1 + used.add(candidate) + return candidate + + +def _excel_col_name(index: int) -> str: + out = "" + index += 1 + while index: + index, rem = divmod(index - 1, 26) + out = chr(ord("A") + rem) + out + return out + + +def _excel_cell_xml(row_idx: int, col_idx: int, value: Any) -> str: + ref = f"{_excel_col_name(col_idx)}{row_idx}" + if value is None: + return f'' + if isinstance(value, bool): + text = "TRUE" if value else "FALSE" + return f'{text}' + if isinstance(value, (int, float)) and not isinstance(value, bool): + if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))): + text = str(value) + return f'{_xml_escape(text)}' + return f'{value}' + text = str(value) + preserve = ' xml:space="preserve"' if text != text.strip() else "" + return f'{_xml_escape(text)}' + + +def _excel_sheet_xml(rows: list[list[Any]]) -> str: + body: list[str] = [] + for row_idx, row in enumerate(rows, start=1): + cells = "".join( + _excel_cell_xml(row_idx, col_idx, value) for col_idx, value in enumerate(row) + ) + body.append(f'{cells}') + return ( + '' + '' + "" + "".join(body) + "" + ) + + +def _write_xlsx_workbook(path: str, sheets: list[tuple[str, list[list[Any]]]]) -> None: + """Write a minimal XLSX workbook using only the Python standard library.""" + used_names: set[str] = set() + named_sheets = [(_excel_safe_sheet_name(name, used_names), rows) for name, rows in sheets] + out_dir = os.path.dirname(path) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + + workbook_sheets = [] + workbook_rels = [] + content_overrides = [] + for idx, (name, _rows) in enumerate(named_sheets, start=1): + workbook_sheets.append( + f'' + ) + workbook_rels.append( + f'' + ) + content_overrides.append( + f'' + ) + + content_types = ( + '' + '' + '' + '' + '' + + "".join(content_overrides) + + "" + ) + root_rels = ( + '' + '' + '' + "" + ) + workbook_xml = ( + '' + '' + "" + "".join(workbook_sheets) + "" + ) + workbook_rels_xml = ( + '' + '' + + "".join(workbook_rels) + + "" + ) + + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr("[Content_Types].xml", content_types) + zf.writestr("_rels/.rels", root_rels) + zf.writestr("xl/workbook.xml", workbook_xml) + zf.writestr("xl/_rels/workbook.xml.rels", workbook_rels_xml) + for idx, (_name, rows) in enumerate(named_sheets, start=1): + zf.writestr(f"xl/worksheets/sheet{idx}.xml", _excel_sheet_xml(rows)) + + +def _latency_rank_value(row: dict[str, Any], rank_name: str, metric: str) -> Optional[float]: + per_rank = ((row.get("latency_ms") or {}).get("per_rank") or {}).get(rank_name) or {} + value = per_rank.get(metric) + return float(value) if isinstance(value, (int, float)) else None + + +def _flatten_result_for_analysis(row: dict[str, Any]) -> dict[str, Any]: + workload = row.get("workload") or {} + requested = row.get("requested_config") or {} + actual = row.get("actual_config") or {} + instrumentation = row.get("instrumentation") or {} + latency = row.get("latency_ms") or {} + routing_actual = (row.get("routing_control") or {}).get("actual") or {} + dispatch_summary = routing_actual.get("observed_dispatch_matrix_summary") or {} + hist_summary = routing_actual.get("observed_expert_histogram_summary") or {} + kernel_breakdown = row.get("kernel_breakdown") or {} + iter_max_stats = latency.get("iter_max_stats") or {} + iter_max_outliers = latency.get("iter_max_outliers") or {} + return { + "num_tokens": workload.get("num_tokens"), + "per_rank_num_tokens": json.dumps(workload.get("per_rank_num_tokens")), + "requested_backend": requested.get("backend"), + "requested_comm_method": requested.get("comm_method"), + "requested_parallel_mode": requested.get("parallel_mode"), + "requested_cuda_graph": requested.get("cuda_graph"), + "requested_low_precision_combine": requested.get("use_low_precision_moe_combine"), + "actual_backend": actual.get("backend"), + "actual_comm_method": actual.get("comm_method"), + "scheduler_kind": actual.get("scheduler_kind"), + "actual_moe_ep_size": actual.get("moe_ep_size"), + "actual_moe_tp_size": actual.get("moe_tp_size"), + "actual_attention_dp": actual.get("enable_attention_dp"), + "num_chunks": actual.get("num_chunks"), + "status": row.get("status"), + "skip_reason": row.get("skip_reason"), + "score_ms": latency.get("score"), + "score_type": latency.get("score_type"), + "raw_score_ms": latency.get("raw_score"), + "raw_score_type": latency.get("raw_score_type"), + "iter_max_median_ms": iter_max_stats.get("median"), + "iter_max_p90_ms": iter_max_stats.get("p90"), + "iter_max_max_ms": iter_max_stats.get("max"), + "iter_max_outlier_count": iter_max_outliers.get("count"), + "rank0_mean_ms": _latency_rank_value(row, "rank0", "mean"), + "rank1_mean_ms": _latency_rank_value(row, "rank1", "mean"), + "rank2_mean_ms": _latency_rank_value(row, "rank2", "mean"), + "rank3_mean_ms": _latency_rank_value(row, "rank3", "mean"), + "rank0_p90_ms": _latency_rank_value(row, "rank0", "p90"), + "rank1_p90_ms": _latency_rank_value(row, "rank1", "p90"), + "rank2_p90_ms": _latency_rank_value(row, "rank2", "p90"), + "rank3_p90_ms": _latency_rank_value(row, "rank3", "p90"), + "autotune_status": instrumentation.get("autotune_status"), + "latency_source": instrumentation.get("latency_source"), + "analysis_level": instrumentation.get("level"), + "kernel_breakdown_available": instrumentation.get("kernel_breakdown_available"), + "bottleneck": row.get("bottleneck"), + "moe_forward_kernel_count": len(kernel_breakdown.get("moe_forward_kernels") or []), + "other_kernel_count": len(kernel_breakdown.get("other_kernels") or []), + "routing_path": routing_actual.get("routing_path"), + "routing_realization_status": (routing_actual.get("routing_realization") or {}).get( + "status" + ), + "routing_observation_source": routing_actual.get("observation_source"), + "routing_off_diagonal_ratio": dispatch_summary.get("off_diagonal_ratio"), + "routing_active_experts": hist_summary.get("active_experts"), + } + + +_ANALYSIS_COLUMNS: tuple[str, ...] = ( + "num_tokens", + "requested_parallel_mode", + "requested_backend", + "requested_comm_method", + "requested_cuda_graph", + "requested_low_precision_combine", + "actual_backend", + "actual_comm_method", + "scheduler_kind", + "actual_moe_ep_size", + "actual_moe_tp_size", + "actual_attention_dp", + "num_chunks", + "status", + "skip_reason", + "score_ms", + "score_type", + "raw_score_ms", + "raw_score_type", + "iter_max_median_ms", + "iter_max_p90_ms", + "iter_max_max_ms", + "iter_max_outlier_count", + "rank0_mean_ms", + "rank1_mean_ms", + "rank2_mean_ms", + "rank3_mean_ms", + "rank0_p90_ms", + "rank1_p90_ms", + "rank2_p90_ms", + "rank3_p90_ms", + "autotune_status", + "latency_source", + "analysis_level", + "kernel_breakdown_available", + "bottleneck", + "moe_forward_kernel_count", + "other_kernel_count", + "routing_path", + "routing_realization_status", + "routing_observation_source", + "routing_off_diagonal_ratio", + "routing_active_experts", + "per_rank_num_tokens", +) + + +def _analysis_table_rows(flat_rows: list[dict[str, Any]]) -> list[list[Any]]: + return [list(_ANALYSIS_COLUMNS)] + [ + [row.get(col) for col in _ANALYSIS_COLUMNS] for row in flat_rows + ] + + +def _best_by_workload_rows(rankings: list[dict[str, Any]]) -> list[list[Any]]: + rows: list[list[Any]] = [ + [ + "num_tokens", + "parallel_mode", + "backend", + "requested_backend", + "comm_method", + "cuda_graph", + "score_ms", + "status", + "skip_reason", + "autotune_status", + ] + ] + for ranking in rankings: + best = ranking.get("best") or {} + rows.append( + [ + ranking.get("num_tokens"), + ranking.get("parallel_mode"), + best.get("backend"), + best.get("requested_backend"), + best.get("comm_method"), + best.get("cuda_graph"), + best.get("score_ms"), + best.get("status"), + best.get("skip_reason"), + best.get("autotune_status"), + ] + ) + return rows + + +def _status_summary_rows(flat_rows: list[dict[str, Any]]) -> list[list[Any]]: + counts: dict[tuple[Any, Any, Any, Any], dict[str, Any]] = {} + for row in flat_rows: + key = ( + row.get("num_tokens"), + row.get("requested_backend"), + row.get("requested_comm_method"), + row.get("status"), + ) + entry = counts.setdefault(key, {"count": 0, "reasons": set()}) + entry["count"] += 1 + if row.get("skip_reason"): + entry["reasons"].add(str(row["skip_reason"])) + out: list[list[Any]] = [ + ["num_tokens", "requested_backend", "requested_comm_method", "status", "count", "reasons"] + ] + for (num_tokens, backend, comm_method, status), entry in sorted( + counts.items(), + key=lambda item: (str(item[0][0]), str(item[0][1]), str(item[0][2]), str(item[0][3])), + ): + out.append( + [ + num_tokens, + backend, + comm_method, + status, + entry["count"], + " | ".join(sorted(entry["reasons"])), + ] + ) + return out + + +_KERNEL_BREAKDOWN_BASE_COLUMNS: tuple[str, ...] = ( + "result_index", + "num_tokens", + "requested_parallel_mode", + "requested_backend", + "requested_comm_method", + "requested_cuda_graph", + "requested_low_precision_combine", + "actual_backend", + "actual_comm_method", + "scheduler_kind", + "status", + "score_ms", + "category", + "kernel_name", + "kernel_count", +) + + +def _rank_sort_key(rank_name: str) -> tuple[int, str]: + prefix = "rank" + if rank_name.startswith(prefix) and rank_name[len(prefix) :].isdigit(): + return (int(rank_name[len(prefix) :]), rank_name) + return (10**9, rank_name) + + +def _kernel_stat(stats: dict[str, Any], metric: str) -> Optional[float]: + value = stats.get(metric) + return float(value) if isinstance(value, (int, float)) else None + + +def _kernel_breakdown_rank_names(rows: list[dict[str, Any]]) -> list[str]: + rank_names: set[str] = set() + for row in rows: + kernel_breakdown = row.get("kernel_breakdown") or {} + for category in ("moe_forward_kernels", "other_kernels"): + for kernel in kernel_breakdown.get(category) or []: + per_rank = kernel.get("per_rank") or {} + rank_names.update(str(rank_name) for rank_name in per_rank) + return sorted(rank_names, key=_rank_sort_key) + + +def _kernel_breakdown_columns(rank_names: list[str]) -> list[str]: + return ( + list(_KERNEL_BREAKDOWN_BASE_COLUMNS) + + [f"{rank_name}_mean_ms" for rank_name in rank_names] + + [f"{rank_name}_median_ms" for rank_name in rank_names] + ) + + +def _kernel_rank_metric_values( + per_rank: dict[str, Any], rank_names: list[str], metric: str +) -> list[Optional[float]]: + values: list[Optional[float]] = [] + for rank_name in rank_names: + stats = per_rank.get(rank_name) + values.append(_kernel_stat(stats, metric) if isinstance(stats, dict) else None) + return values + + +def _kernel_breakdown_rows(rows: list[dict[str, Any]]) -> list[list[Any]]: + rank_names = _kernel_breakdown_rank_names(rows) + out: list[list[Any]] = [_kernel_breakdown_columns(rank_names)] + for result_index, row in enumerate(rows): + workload = row.get("workload") or {} + requested = row.get("requested_config") or {} + actual = row.get("actual_config") or {} + latency = row.get("latency_ms") or {} + common = [ + result_index, + workload.get("num_tokens"), + requested.get("parallel_mode"), + requested.get("backend"), + requested.get("comm_method"), + requested.get("cuda_graph"), + requested.get("use_low_precision_moe_combine"), + actual.get("backend"), + actual.get("comm_method"), + actual.get("scheduler_kind"), + row.get("status"), + latency.get("score"), + ] + kernel_breakdown = row.get("kernel_breakdown") or {} + for category in ("moe_forward_kernels", "other_kernels"): + for kernel in kernel_breakdown.get(category) or []: + per_rank = kernel.get("per_rank") or {} + out.append( + common + + [ + category, + kernel.get("name"), + kernel.get("count"), + ] + + _kernel_rank_metric_values(per_rank, rank_names, "mean") + + _kernel_rank_metric_values(per_rank, rank_names, "median") + ) + return out + + +_RAW_DATA_COLUMNS: tuple[str, ...] = ( + "result_index", + "num_tokens", + "requested_parallel_mode", + "requested_backend", + "requested_comm_method", + "requested_cuda_graph", + "requested_low_precision_combine", + "actual_backend", + "actual_comm_method", + "scheduler_kind", + "status", + "score_ms", + "record_type", + "category", + "kernel_name", + "rank", + "iteration", + "time_ms", +) + + +def _result_common_row(result_index: int, row: dict[str, Any]) -> list[Any]: + workload = row.get("workload") or {} + requested = row.get("requested_config") or {} + actual = row.get("actual_config") or {} + latency = row.get("latency_ms") or {} + return [ + result_index, + workload.get("num_tokens"), + requested.get("parallel_mode"), + requested.get("backend"), + requested.get("comm_method"), + requested.get("cuda_graph"), + requested.get("use_low_precision_moe_combine"), + actual.get("backend"), + actual.get("comm_method"), + actual.get("scheduler_kind"), + row.get("status"), + latency.get("score"), + ] + + +def _raw_sample_value(value: Any) -> Optional[float]: + return float(value) if isinstance(value, (int, float)) else None + + +def _raw_data_rows(rows: list[dict[str, Any]]) -> list[list[Any]]: + out: list[list[Any]] = [list(_RAW_DATA_COLUMNS)] + for result_index, row in enumerate(rows): + common = _result_common_row(result_index, row) + raw_data = row.get("raw_data") or {} + forward_per_rank = (raw_data.get("forward_times_ms") or {}).get("per_rank") or {} + for rank_name, times in sorted( + forward_per_rank.items(), key=lambda item: _rank_sort_key(item[0]) + ): + for iteration, value in enumerate(times or []): + out.append( + common + + [ + "forward", + None, + None, + rank_name, + iteration, + _raw_sample_value(value), + ] + ) + + kernel_times = raw_data.get("kernel_times_ms") or {} + for category in ("moe_forward_kernels", "other_kernels"): + for kernel in kernel_times.get(category) or []: + per_rank = kernel.get("per_rank") or {} + for rank_name, times in sorted( + per_rank.items(), key=lambda item: _rank_sort_key(item[0]) + ): + for iteration, value in enumerate(times or []): + out.append( + common + + [ + "kernel", + category, + kernel.get("name"), + rank_name, + iteration, + _raw_sample_value(value), + ] + ) + return out + + +def _workload_sheets(flat_rows: list[dict[str, Any]]) -> list[tuple[str, list[list[Any]]]]: + grouped: dict[Any, list[dict[str, Any]]] = {} + for row in flat_rows: + grouped.setdefault(row.get("num_tokens"), []).append(row) + sheets: list[tuple[str, list[list[Any]]]] = [] + for num_tokens, rows in sorted(grouped.items(), key=lambda item: str(item[0])): + sorted_rows = sorted( + rows, + key=lambda row: ( + row.get("score_ms") is None, + row.get("score_ms") if row.get("score_ms") is not None else 0.0, + str(row.get("requested_backend")), + str(row.get("requested_comm_method")), + ), + ) + sheets.append((f"workload_{num_tokens}", _analysis_table_rows(sorted_rows))) + return sheets + + +def _build_analysis_workbook_sheets(payload: dict[str, Any]) -> list[tuple[str, list[list[Any]]]]: + rows = payload.get("results") or [] + flat_rows = [_flatten_result_for_analysis(row) for row in rows] + return [ + ("all_results", _analysis_table_rows(flat_rows)), + ("best_by_workload", _best_by_workload_rows(payload.get("rankings") or [])), + ("status_summary", _status_summary_rows(flat_rows)), + ("kernel_breakdown", _kernel_breakdown_rows(rows)), + ("raw_data", _raw_data_rows(rows)), + *_workload_sheets(flat_rows), + ] + + +def default_analysis_workbook_path(output_file: Optional[str]) -> Optional[str]: + if not output_file: + return None + root, _ext = os.path.splitext(output_file) + return f"{root}.analysis.xlsx" + + +def write_analysis_workbook(payload: dict[str, Any], path: str) -> None: + _write_xlsx_workbook(path, _build_analysis_workbook_sheets(payload)) + + +def write_final_report( + *, + args: Any, + ctx: Any, + rows: list[dict[str, Any]], + world_size: int, + repo_root: Path, +) -> None: + """Produce the final report payload and write it to disk + workbook.""" + out_payload = build_report_payload( + ctx=ctx, + rows=rows, + world_size=world_size, + cuda_graph_default=bool(args.cuda_graph), + repo_root=repo_root, + ) + if args.output_file: + out_dir = os.path.dirname(args.output_file) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + tmp = args.output_file + ".final.tmp" + with open(tmp, "w") as f: + json.dump(out_payload, f, indent=2) + os.replace(tmp, args.output_file) + print(f"Report written to {args.output_file}", flush=True) + workbook_file = getattr( + args, "analysis_workbook_file", None + ) or default_analysis_workbook_path(args.output_file) + if workbook_file: + write_analysis_workbook(out_payload, workbook_file) + print(f"Analysis workbook written to {workbook_file}", flush=True) + else: + print(json.dumps({"rankings": out_payload["rankings"]}, indent=2), flush=True) + workbook_file = getattr(args, "analysis_workbook_file", None) + if workbook_file: + write_analysis_workbook(out_payload, workbook_file) + print(f"Analysis workbook written to {workbook_file}", flush=True) diff --git a/tests/microbenchmarks/bench_moe/results.py b/tests/microbenchmarks/bench_moe/results.py new file mode 100644 index 000000000000..e5637edefe18 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/results.py @@ -0,0 +1,445 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Result scoring, bottleneck labelling, and row serialization.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from tensorrt_llm._utils import mpi_allgather + +from .routing import _per_rank_tokens +from .specs import ConfigSpec, ModelSpec, RunResult, WorkloadSpec +from .utils import _compute_stats + +_ROBUST_SCORE_MIN_SAMPLES = 8 +_ROBUST_SCORE_MAD_Z_THRESHOLD = 3.5 +_ROBUST_SCORE_ZERO_SPREAD_REL_TOL = 0.05 + + +def _gather_per_iteration_times(times_ms: List[float]) -> List[List[float]]: + """All-gather raw per-iteration latencies; returns ``[ [rank0_iters], ... ]``.""" + return mpi_allgather(times_ms) + + +def _slowest_rank_iter_maxes(per_rank_iters: List[List[float]]) -> List[float]: + """Return ``max_r(latency_ms[rank=r][iteration=i])`` for each common iteration. + + Falls back gracefully when ranks reported different iteration counts; the + common length is used and trailing entries are ignored. + """ + if not per_rank_iters: + return [] + lengths = [len(r) for r in per_rank_iters if r] + if not lengths: + return [] + n = min(lengths) + if n == 0: + return [] + iter_max: List[float] = [] + for i in range(n): + per_iter_vals = [r[i] for r in per_rank_iters if i < len(r)] + iter_max.append(max(per_iter_vals)) + return iter_max + + +def _slowest_rank_mean_score(per_rank_iters: List[List[float]]) -> float: + """Compute raw ``mean_i(max_r(latency_ms[rank=r][iteration=i]))``.""" + iter_max = _slowest_rank_iter_maxes(per_rank_iters) + return sum(iter_max) / len(iter_max) if iter_max else 0.0 + + +def _median(values: List[float]) -> float: + if not values: + return 0.0 + s = sorted(values) + n = len(s) + mid = n // 2 + if n % 2: + return float(s[mid]) + return float((s[mid - 1] + s[mid]) / 2.0) + + +def _robust_slowest_rank_score(iter_max: List[float]) -> Tuple[float, str, Dict[str, Any]]: + """Score distributed latency while reporting, not hiding, extreme samples.""" + if not iter_max: + return ( + 0.0, + "slowest_rank_trimmed_mean", + { + "method": "modified_z_score_mad", + "count": 0, + "samples_used": 0, + "samples_total": 0, + "items": [], + }, + ) + + if len(iter_max) < _ROBUST_SCORE_MIN_SAMPLES: + return ( + _median(iter_max), + "slowest_rank_median", + { + "method": "disabled_insufficient_samples", + "count": 0, + "samples_used": len(iter_max), + "samples_total": len(iter_max), + "items": [], + }, + ) + + center = _median(iter_max) + deviations = [abs(v - center) for v in iter_max] + mad = _median(deviations) + outliers: List[Dict[str, Any]] = [] + keep: List[float] = [] + + for idx, value in enumerate(iter_max): + diff = value - center + score: Optional[float] + if mad > 0.0: + score = 0.6745 * diff / mad + is_outlier = abs(score) > _ROBUST_SCORE_MAD_Z_THRESHOLD + else: + tolerance = max(abs(center) * _ROBUST_SCORE_ZERO_SPREAD_REL_TOL, 1.0e-12) + score = None + is_outlier = abs(diff) > tolerance + + if is_outlier: + outliers.append( + { + "index": int(idx), + "value": float(value), + "center": float(center), + "modified_z_score": float(score) if score is not None else None, + "absolute_deviation": float(abs(diff)), + "direction": "high" if diff > 0 else "low", + } + ) + else: + keep.append(value) + + samples = keep if keep else iter_max + score = sum(samples) / len(samples) + return ( + float(score), + "slowest_rank_trimmed_mean", + { + "method": "modified_z_score_mad", + "threshold": float(_ROBUST_SCORE_MAD_Z_THRESHOLD), + "center": float(center), + "mad": float(mad), + "count": len(outliers), + "samples_used": len(samples), + "samples_total": len(iter_max), + "items": outliers, + }, + ) + + +def _build_latency_block(per_rank_iters: List[List[float]]) -> Dict[str, Any]: + iter_max = _slowest_rank_iter_maxes(per_rank_iters) + raw_score = sum(iter_max) / len(iter_max) if iter_max else 0.0 + score, score_type, outliers = _robust_slowest_rank_score(iter_max) + return { + "score": float(score), + "score_type": score_type, + "raw_score": float(raw_score), + "raw_score_type": "slowest_rank_mean", + "iter_max_stats": _compute_stats(iter_max), + "iter_max_outliers": outliers, + "per_rank": {f"rank{i}": _compute_stats(times) for i, times in enumerate(per_rank_iters)}, + } + + +def _gather_kernel_breakdown( + detailed_stats: Dict[str, Any], +) -> Dict[str, List[Dict[str, Any]]]: + """All-gather per-kernel timings and produce per-rank summary stats.""" + kernel_breakdown, _raw_kernel_times = _gather_kernel_timing_blocks(detailed_stats) + return kernel_breakdown + + +def _kernel_times_payload(detailed_stats: Dict[str, Any]) -> Dict[str, Dict[str, List[float]]]: + categories = ("moe_forward_kernels", "other_kernels") + local_payload: Dict[str, Dict[str, List[float]]] = {} + for cat in categories: + local_payload[cat] = { + kernel["name"]: kernel.get("_times", []) for kernel in detailed_stats.get(cat, []) + } + return local_payload + + +def _kernel_breakdown_from_payload( + all_payload: List[Dict[str, Dict[str, List[float]]]], +) -> Dict[str, List[Dict[str, Any]]]: + categories = ("moe_forward_kernels", "other_kernels") + merged: Dict[str, List[Dict[str, Any]]] = {} + for cat in categories: + seen = set() + kernel_names: List[str] = [] + for rank_payload in all_payload: + for name in rank_payload.get(cat, {}): + if name not in seen: + seen.add(name) + kernel_names.append(name) + + kernels: List[Dict[str, Any]] = [] + for name in kernel_names: + per_rank_times: List[List[float]] = [] + for rank_payload in all_payload: + times = rank_payload.get(cat, {}).get(name, []) + per_rank_times.append(times if isinstance(times, list) else []) + + per_rank = {f"rank{i}": _compute_stats(times) for i, times in enumerate(per_rank_times)} + kernels.append( + { + "name": name, + "count": max((len(times) for times in per_rank_times), default=0), + "per_rank": per_rank, + } + ) + merged[cat] = kernels + return merged + + +def _raw_kernel_times_from_payload( + all_payload: List[Dict[str, Dict[str, List[float]]]], +) -> Dict[str, List[Dict[str, Any]]]: + categories = ("moe_forward_kernels", "other_kernels") + merged: Dict[str, List[Dict[str, Any]]] = {} + for cat in categories: + seen = set() + kernel_names: List[str] = [] + for rank_payload in all_payload: + for name in rank_payload.get(cat, {}): + if name not in seen: + seen.add(name) + kernel_names.append(name) + + kernels: List[Dict[str, Any]] = [] + for name in kernel_names: + per_rank: Dict[str, List[float]] = {} + for rank_idx, rank_payload in enumerate(all_payload): + times = rank_payload.get(cat, {}).get(name, []) + per_rank[f"rank{rank_idx}"] = list(times) if isinstance(times, list) else [] + kernels.append( + { + "name": name, + "count": max((len(times) for times in per_rank.values()), default=0), + "per_rank": per_rank, + } + ) + merged[cat] = kernels + return merged + + +def _gather_kernel_timing_blocks( + detailed_stats: Dict[str, Any], +) -> Tuple[Dict[str, List[Dict[str, Any]]], Dict[str, List[Dict[str, Any]]]]: + """All-gather kernel timings once and return summary plus raw timing blocks.""" + all_payload = mpi_allgather(_kernel_times_payload(detailed_stats)) + return _kernel_breakdown_from_payload(all_payload), _raw_kernel_times_from_payload(all_payload) + + +def _build_raw_data_block( + per_rank_iters: List[List[float]], + raw_kernel_times: Dict[str, List[Dict[str, Any]]], +) -> Dict[str, Any]: + """Build the raw sample block written to JSON and the analysis workbook.""" + return { + "forward_times_ms": { + "per_rank": { + f"rank{i}": list(times) if isinstance(times, list) else [] + for i, times in enumerate(per_rank_iters) + } + }, + "kernel_times_ms": raw_kernel_times + or { + "moe_forward_kernels": [], + "other_kernels": [], + }, + } + + +# --------------------------------------------------------------------------- +# Bottleneck classification (best-effort, phase-data dependent) +# --------------------------------------------------------------------------- + + +# Heuristic thresholds for ``_classify_bottleneck``. Tuned for "label looks +# right at a glance on the dashboard"; not intended as hard cutoffs. +_BOTTLENECK_COMM_FRACTION = 0.5 +_BOTTLENECK_COMPUTE_FRACTION = 0.5 +_BOTTLENECK_ROUTING_FRACTION = 0.4 +_BOTTLENECK_LAUNCH_MIN_KERNELS = 50 +_BOTTLENECK_LAUNCH_MAX_KERNEL_MS = 0.005 # 5 us per launch on rank0 + + +def _classify_bottleneck( + phase_times_ms_agg: Dict[str, Dict[str, float]], + kernel_breakdown: Dict[str, Any], + forward_score_ms: float, +) -> Optional[str]: + """Return a coarse bottleneck label. + + The classification is intentionally minimal: with no scheduler-side phase + markers (Phase 5 of the design) we can only inspect kernel breakdown. When + we can identify dispatch/combine/GEMM heuristically by name we use that; + otherwise we return ``None`` so the dashboard can show ``unknown``. + """ + if phase_times_ms_agg: + comm_ms = sum( + v.get("score", 0.0) + for k, v in phase_times_ms_agg.items() + if k in ("dispatch", "combine", "all_reduce_or_reduce_results") + ) + gemm_ms = phase_times_ms_agg.get("backend_run_moe", {}).get( + "score", 0.0 + ) or phase_times_ms_agg.get("fused_comm_backend_run_moe", {}).get("score", 0.0) + routing_ms = phase_times_ms_agg.get("routing", {}).get("score", 0.0) + total = comm_ms + gemm_ms + routing_ms + if total <= 0: + return None + if comm_ms / total > _BOTTLENECK_COMM_FRACTION: + return "communication_bound" + if gemm_ms / total > _BOTTLENECK_COMPUTE_FRACTION: + return "compute_bound" + if routing_ms / total > _BOTTLENECK_ROUTING_FRACTION: + return "routing_bound" + return "unknown" + + # No phase markers: inspect kernel breakdown for a rough hint. + moe_kernels = kernel_breakdown.get("moe_forward_kernels", []) + if not moe_kernels: + return None + total_count = sum(k.get("count", 0) for k in moe_kernels) + rank0_total = 0.0 + rank0_count = 0 + for k in moe_kernels: + rank0_stats = k.get("per_rank", {}).get("rank0", {}) + mean_ms = float(rank0_stats.get("mean", 0.0)) + count = int(k.get("count", 0)) + rank0_total += mean_ms * count + rank0_count += count + avg_ms = (rank0_total / rank0_count) if rank0_count else 0.0 + if ( + total_count > _BOTTLENECK_LAUNCH_MIN_KERNELS + and 0.0 < avg_ms < _BOTTLENECK_LAUNCH_MAX_KERNEL_MS + and forward_score_ms > 0 + ): + return "launch_overhead_bound" + return None + + +def _runresult_to_row(result: RunResult) -> Dict[str, Any]: + """Convert ``RunResult`` to the v2 row schema.""" + return { + "workload": result.workload.to_dict(per_rank_num_tokens=result.per_rank_num_tokens), + "requested_config": result.config.to_dict(), + "actual_config": { + "backend": result.actual_backend, + "comm_method": result.actual_comm_method, + "comm_fallback_reason": result.actual_comm_fallback_reason, + "scheduler_kind": result.scheduler_kind, + "moe_ep_size": result.moe_ep_size, + "moe_tp_size": result.moe_tp_size, + "enable_attention_dp": result.enable_attention_dp, + "num_chunks": result.num_chunks, + }, + "status": result.status, + "skip_reason": result.skip_reason, + "status_per_rank": result.status_per_rank, + "instrumentation": result.instrumentation, + "latency_ms": result.latency_ms + or { + "score": None, + "score_type": "slowest_rank_trimmed_mean", + "raw_score": None, + "raw_score_type": "slowest_rank_mean", + "iter_max_stats": {}, + "iter_max_outliers": {}, + "per_rank": {}, + }, + "phase_times_ms": result.phase_times_ms or {"agg": {}, "per_rank": {}}, + "overlap": result.overlap or {"overlap_ms": None, "overlap_ratio": None}, + "bottleneck": result.bottleneck, + "kernel_breakdown": result.kernel_breakdown + or { + "moe_forward_kernels": [], + "other_kernels": [], + }, + "raw_data": result.raw_data + or { + "forward_times_ms": {"per_rank": {}}, + "kernel_times_ms": { + "moe_forward_kernels": [], + "other_kernels": [], + }, + }, + "routing_control": result.routing_control or None, + } + + +def _make_skipped_run_result( + *, + model: ModelSpec, + workload: WorkloadSpec, + config: ConfigSpec, + world_size: int, + analysis: Tuple[str, ...], + reason: str, +) -> RunResult: + """Build a worker-level skipped row without entering MPI collectives.""" + r = RunResult(model=model, workload=workload, config=config) + r.status = "skipped" + r.skip_reason = reason + r.per_rank_num_tokens = _per_rank_tokens(workload, world_size) + r.status_per_rank = {f"rank{i}": "skipped" for i in range(world_size)} + r.instrumentation = { + "level": ",".join(sorted(analysis)) if analysis else "summary", + "cuda_graph": bool(config.cuda_graph), + "cupti_available": False, + "phase_timing_available": False, + "kernel_breakdown_available": False, + } + return r + + +def _make_upstream_skipped_row( + *, + model: ModelSpec, + workload: WorkloadSpec, + config: ConfigSpec, + world_size: int, + analysis: Tuple[str, ...], + reason: str, +) -> Dict[str, Any]: + """Build a row marking a candidate as not-attempted due to an upstream crash. + + Uses a reason ending in ``_upstream`` (or starting with one of the + upstream prefixes) so :func:`_is_completed_for_resume` treats it as + not-done; a subsequent ``--resume_from`` run will retry it. + """ + placeholder = _make_skipped_run_result( + model=model, + workload=workload, + config=config, + world_size=world_size, + analysis=analysis, + reason=reason, + ) + return _runresult_to_row(placeholder) diff --git a/tests/microbenchmarks/bench_moe/routing/__init__.py b/tests/microbenchmarks/bench_moe/routing/__init__.py new file mode 100644 index 000000000000..7a78354af983 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/routing/__init__.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Routing-control package. + +This package implements the planning + materialisation + injection pipeline for +advanced routing-control benchmarks. +Submodules split the pipeline into cohesive stages: + +* :mod:`bench_moe.routing.parsing` -- CLI-style pattern spec parsing +* :mod:`bench_moe.routing.builders` -- plan types and construction +* :mod:`bench_moe.routing.materialize` -- selected-experts realisation +* :mod:`bench_moe.routing.native_logits` -- native logits projection and + supplied-topk forward patches + +The package re-exports every public symbol; the historical +``bench_moe.`` import paths continue to work via the package +entrypoint re-export. +""" + +from .builders import ( + RoutingPlan, + RoutingProjectionResult, + _build_dispatch_matrix, + _build_expert_histogram, + _build_per_rank_num_tokens, + _build_routing_plan, + _largest_remainder_split, + _load_2d_matrix_json, + _load_dispatch_matrix_file, + _load_expert_histogram_file, + _load_routing_pattern_file, + _per_rank_tokens, +) +from .materialize import ( + _flatten_plan_slots_for_rank, + _make_uniform_topk_scales, + _materialize_selected_experts_for_rank, + _observe_routing_metrics, + _observe_summary, + _pack_slots_column_major, + _repair_duplicate_experts, + _split_slot_count_to_experts, +) +from .native_logits import ( + _NATIVE_PROJECTION_CAPABILITIES, + _align_topk_to_batch, + _classify_native_projection, + _make_supplied_topk_apply, + _make_supplied_topk_run_moe, + _maybe_install_routing_control_patch, + _project_router_logits_for_plan, +) +from .parsing import ( + _COMM_PATTERN_NAMES, + _EXPERT_PATTERN_NAMES, + _parse_comm_pattern, + _parse_expert_pattern, + _parse_pattern_spec, + _parse_typed_pattern, + _pop_hotness_kwarg, +) + +__all__ = [ + "RoutingPlan", + "RoutingProjectionResult", + "_COMM_PATTERN_NAMES", + "_EXPERT_PATTERN_NAMES", + "_NATIVE_PROJECTION_CAPABILITIES", + "_align_topk_to_batch", + "_build_dispatch_matrix", + "_build_expert_histogram", + "_build_per_rank_num_tokens", + "_build_routing_plan", + "_classify_native_projection", + "_flatten_plan_slots_for_rank", + "_largest_remainder_split", + "_load_2d_matrix_json", + "_load_dispatch_matrix_file", + "_load_expert_histogram_file", + "_load_routing_pattern_file", + "_make_supplied_topk_apply", + "_make_supplied_topk_run_moe", + "_make_uniform_topk_scales", + "_materialize_selected_experts_for_rank", + "_maybe_install_routing_control_patch", + "_observe_routing_metrics", + "_observe_summary", + "_pack_slots_column_major", + "_parse_comm_pattern", + "_parse_expert_pattern", + "_parse_pattern_spec", + "_parse_typed_pattern", + "_per_rank_tokens", + "_pop_hotness_kwarg", + "_project_router_logits_for_plan", + "_repair_duplicate_experts", + "_split_slot_count_to_experts", +] diff --git a/tests/microbenchmarks/bench_moe/routing/builders.py b/tests/microbenchmarks/bench_moe/routing/builders.py new file mode 100644 index 000000000000..af3d792d756a --- /dev/null +++ b/tests/microbenchmarks/bench_moe/routing/builders.py @@ -0,0 +1,365 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Construction of the canonical :class:`RoutingPlan`. + +The builders translate :class:`RoutingControlSpec` plus the runtime topology +into the normalised plan shape (per-rank token counts, slot dispatch matrix, +expert histogram) consumed by the materialiser. JSON sidecar loaders for the +``--routing_pattern_file`` knob also live here so all plan-construction paths +share the same validation surface. +""" + +from __future__ import annotations + +import json +import random +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import torch + +from ..specs import RoutingControlSpec, WorkloadSpec, _to_jsonable_dict +from ..utils import _distribute_tokens, _validate_per_rank_token_list +from .parsing import _parse_comm_pattern, _parse_expert_pattern + + +@dataclass(frozen=True) +class RoutingPlan: + """Canonical normalised routing plan. + + ``per_rank_num_tokens[src]`` is the local input token count on source rank + ``src``. ``dispatch_matrix[src][dst]`` is the *slot* count (each selected + (token, expert) slot counts once) sent from ``src`` to ``dst``. Row sums + are ``per_rank_num_tokens[src] * top_k``. ``expert_histogram[dst][le]`` is + the global slot count owned by local expert ``le`` on rank ``dst``. + """ + + per_rank_num_tokens: Tuple[int, ...] + dispatch_matrix: Tuple[Tuple[int, ...], ...] + expert_histogram: Tuple[Tuple[int, ...], ...] + seed: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return _to_jsonable_dict(self) + + +@dataclass(frozen=True) +class RoutingProjectionResult: + """Outcome of trying to realise a ``RoutingPlan`` for a given routing method.""" + + router_logits: Optional[torch.Tensor] + status: str # "exact" | "projected" | "rejected" | "forced_exact" | "not_applicable" + reason: str + observed_slot_dispatch_matrix: Tuple[Tuple[int, ...], ...] + observed_token_dispatch_matrix: Tuple[Tuple[int, ...], ...] + observed_expert_histogram: Tuple[Tuple[int, ...], ...] + max_abs_slot_error: int + max_relative_slot_error: float + selected_experts: Optional[torch.Tensor] + selected_scales: Optional[torch.Tensor] + warnings: Tuple[str, ...] = () + + +def _largest_remainder_split(total: int, weights: List[float]) -> List[int]: + """Split ``total`` integer units among bins using largest-remainder method. + + All weights must be non-negative. Zero-weight bins always receive zero + units. The result is deterministic for ties (ties broken by lower index). + """ + n = len(weights) + if n == 0: + return [] + total = int(total) + if total <= 0: + return [0] * n + s = sum(weights) + if s <= 0.0: + # Distribute evenly when all weights are zero. + base = total // n + rem = total - base * n + out = [base] * n + for i in range(rem): + out[i] += 1 + return out + raw = [total * (w / s) for w in weights] + floors = [int(x) for x in raw] + used = sum(floors) + remainders = sorted( + ((raw[i] - floors[i], -i) for i in range(n)), key=lambda pair: pair[0], reverse=True + ) + for k in range(total - used): + _, neg_idx = remainders[k % n] + floors[-neg_idx] += 1 + return floors + + +def _random_weights(n: int, *, seed: int, label: str, index: int) -> List[float]: + rng = random.Random(f"bench_moe:{int(seed)}:{label}:{int(index)}") + return [rng.random() for _ in range(n)] + + +def _build_per_rank_num_tokens( + spec: RoutingControlSpec, + num_tokens: int, + world_size: int, +) -> List[int]: + """Resolve ``per_rank_num_tokens`` for a workload. + + Explicit ``spec.per_rank_num_tokens`` wins; otherwise tokens are split + evenly across ranks with any remainder on rank 0. + """ + if spec.per_rank_num_tokens is None: + return _distribute_tokens(int(num_tokens), world_size) + return _validate_per_rank_token_list( + spec.per_rank_num_tokens, world_size=world_size, expected_total=int(num_tokens) + ) + + +def _per_rank_tokens(workload: WorkloadSpec, world_size: int) -> List[int]: + """Materialize the ``per_rank_num_tokens`` list for a workload + world size.""" + return _build_per_rank_num_tokens( + workload.routing_control, int(workload.num_tokens), world_size + ) + + +def _build_dispatch_matrix( + comm_pattern: str, + per_rank_num_tokens: List[int], + top_k: int, + ep_size: int, + seed: int = 0, +) -> List[List[int]]: + """Build the canonical slot ``dispatch_matrix`` for ``comm_pattern``. + + Row sums always equal ``per_rank_num_tokens[src] * top_k``. The matrix is + a pure planning artefact: it does not enforce per-token uniqueness yet. + That constraint is checked at materialisation time. + """ + name, kwargs = _parse_comm_pattern(comm_pattern) + matrix: List[List[int]] = [[0] * ep_size for _ in range(ep_size)] + for src in range(ep_size): + row_total = int(per_rank_num_tokens[src]) * int(top_k) + if row_total == 0: + continue + if name == "file": + # Loaded separately by ``_load_dispatch_matrix_file``. + raise ValueError( + "file: dispatch matrices are loaded via _load_dispatch_matrix_file" + ) + elif name == "random": + weights = _random_weights(ep_size, seed=seed, label="comm", index=src) + elif name == "balanced_alltoall": + weights = [1.0] * ep_size + elif name == "receiver_hotspot": + hot_rank = int(kwargs["rank"]) + if not 0 <= hot_rank < ep_size: + raise ValueError(f"receiver_hotspot rank={hot_rank} out of range [0, {ep_size})") + hotness = float(kwargs["hotness"]) + weights = [(1.0 - hotness) / max(ep_size, 1)] * ep_size + weights[hot_rank] += hotness + elif name == "pair_hotspot": + pair_src = int(kwargs["src"]) + pair_dst = int(kwargs["dst"]) + if not 0 <= pair_src < ep_size or not 0 <= pair_dst < ep_size: + raise ValueError( + f"pair_hotspot src/dst must be in [0, {ep_size}); got src={pair_src}, dst={pair_dst}" + ) + hotness = float(kwargs["hotness"]) + if src == pair_src: + weights = [(1.0 - hotness) / max(ep_size, 1)] * ep_size + weights[pair_dst] += hotness + else: + weights = [1.0] * ep_size + elif name == "local_only": + weights = [0.0] * ep_size + weights[src] = 1.0 + elif name == "ring": + weights = [0.0] * ep_size + weights[(src + 1) % ep_size] = 1.0 + else: + raise ValueError(f"unknown comm_pattern {name!r}") + matrix[src] = _largest_remainder_split(row_total, weights) + return matrix + + +def _build_expert_histogram( + expert_pattern: str, + dispatch_matrix: List[List[int]], + experts_per_rank: int, + ep_size: int, + seed: int = 0, +) -> List[List[int]]: + """Build the canonical global ``expert_histogram[ep_size][experts_per_rank]``.""" + name, kwargs = _parse_expert_pattern(expert_pattern) + histogram: List[List[int]] = [[0] * experts_per_rank for _ in range(ep_size)] + # Per-target slot totals come from column sums of the dispatch matrix. + col_sums = [sum(dispatch_matrix[src][dst] for src in range(ep_size)) for dst in range(ep_size)] + for dst in range(ep_size): + target_total = int(col_sums[dst]) + if target_total <= 0: + continue + if name == "file": + raise ValueError( + "file: expert histograms are loaded via _load_expert_histogram_file" + ) + elif name == "random": + weights = _random_weights(experts_per_rank, seed=seed, label="expert", index=dst) + elif name == "balanced": + weights = [1.0] * experts_per_rank + elif name == "hotspot": + if "active_experts" in kwargs: + active = int(kwargs["active_experts"]) + active = max(1, min(active, experts_per_rank)) + weights = [1.0 if le < active else 0.0 for le in range(experts_per_rank)] + else: + hotness = float(kwargs["hotness"]) + # Concentrate ``hotness`` fraction onto expert 0; spread the + # remainder uniformly. + weights = [(1.0 - hotness) / max(experts_per_rank, 1)] * experts_per_rank + weights[0] += hotness + else: + raise ValueError(f"unknown expert_pattern {name!r}") + histogram[dst] = _largest_remainder_split(target_total, weights) + return histogram + + +def _load_2d_matrix_json( + path: str, + *, + matrix_key: str, + n_rows: int, + n_cols: int, + label: str, + extra_dims: Tuple[Tuple[str, int], ...] = (), +) -> List[List[int]]: + """Load and validate a 2D integer matrix from a JSON sidecar. + + ``extra_dims`` is a list of ``(payload_key, expected_value)`` checks that + happen alongside the always-present ``ep_size`` guard, so the dispatch and + histogram loaders share their full structural-validation pipeline. + """ + with open(path) as f: + payload = json.load(f) + for key, expected in (("ep_size", n_rows),) + tuple(extra_dims): + if int(payload.get(key, -1)) != int(expected): + raise ValueError( + f"{label} {path!r}: {key}={payload.get(key)} mismatches " + f"runtime {key}={int(expected)}" + ) + matrix = payload.get(matrix_key) + if ( + not isinstance(matrix, list) + or len(matrix) != n_rows + or any(not isinstance(row, list) or len(row) != n_cols for row in matrix) + ): + raise ValueError(f"{label} {path!r}: {matrix_key} must be a {n_rows}x{n_cols} integer list") + return [[int(v) for v in row] for row in matrix] + + +def _load_dispatch_matrix_file(path: str, ep_size: int) -> List[List[int]]: + """Load and validate a ``file:`` slot dispatch matrix.""" + return _load_2d_matrix_json( + path, + matrix_key="slot_dispatch_matrix", + n_rows=ep_size, + n_cols=ep_size, + label="dispatch matrix", + ) + + +def _load_expert_histogram_file(path: str, ep_size: int, experts_per_rank: int) -> List[List[int]]: + """Load and validate a ``file:`` expert histogram.""" + return _load_2d_matrix_json( + path, + matrix_key="expert_histogram", + n_rows=ep_size, + n_cols=experts_per_rank, + label="expert histogram", + extra_dims=(("experts_per_rank", experts_per_rank),), + ) + + +def _load_routing_pattern_file( + path: str, ep_size: int, experts_per_rank: int +) -> Tuple[List[List[int]], List[List[int]]]: + """Load a single file that fixes both dispatch traffic and expert histogram.""" + return ( + _load_dispatch_matrix_file(path, ep_size), + _load_expert_histogram_file(path, ep_size, experts_per_rank), + ) + + +def _build_routing_plan( + spec: RoutingControlSpec, + num_tokens: int, + world_size: int, + top_k: int, + num_experts: int, + moe_ep_size: int, +) -> RoutingPlan: + """Translate a ``RoutingControlSpec`` into a canonical normalised plan.""" + if moe_ep_size <= 0 or num_experts % moe_ep_size != 0: + raise ValueError( + f"num_experts ({num_experts}) must be divisible by moe_ep_size ({moe_ep_size})" + ) + experts_per_rank = num_experts // moe_ep_size + if top_k > num_experts: + raise ValueError(f"top_k ({top_k}) must be <= num_experts ({num_experts})") + per_rank = _build_per_rank_num_tokens(spec, num_tokens, world_size) + # The dispatch matrix is indexed by EP rank on both axes. The current + # worker only calls routing-control planning when ``moe_ep_size`` equals + # ``world_size`` so that this EP-axis matrix also matches the user-visible + # per-rank token list. + if spec.routing_pattern_file: + default_patterns = {("balanced_alltoall", "balanced"), ("random", "random")} + if (spec.comm_pattern, spec.expert_pattern) not in default_patterns: + raise ValueError( + "--routing_pattern_file cannot be combined with non-default " + "--comm_pattern or --expert_pattern" + ) + dispatch_matrix, expert_histogram = _load_routing_pattern_file( + spec.routing_pattern_file, moe_ep_size, experts_per_rank + ) + else: + dispatch_matrix = _build_dispatch_matrix( + spec.comm_pattern, per_rank, top_k, moe_ep_size, seed=spec.seed + ) + expert_histogram = _build_expert_histogram( + spec.expert_pattern, dispatch_matrix, experts_per_rank, moe_ep_size, seed=spec.seed + ) + + # Per-row sums are an invariant; emit a clearer error than the materialiser would. + for src in range(moe_ep_size): + expected = int(per_rank[src]) * int(top_k) if src < len(per_rank) else 0 + actual = sum(dispatch_matrix[src]) + if actual != expected: + raise ValueError( + f"dispatch_matrix row {src} sums to {actual}, expected per_rank_num_tokens[{src}] * top_k = {expected}" + ) + # Global expert histogram total must match total slots. + total_slots = sum(int(t) for t in per_rank) * int(top_k) + hist_total = sum(sum(row) for row in expert_histogram) + if hist_total != total_slots: + raise ValueError( + f"expert_histogram sum={hist_total} must equal sum(per_rank_num_tokens) * top_k = {total_slots}" + ) + + return RoutingPlan( + per_rank_num_tokens=tuple(int(v) for v in per_rank), + dispatch_matrix=tuple(tuple(int(v) for v in row) for row in dispatch_matrix), + expert_histogram=tuple(tuple(int(v) for v in row) for row in expert_histogram), + seed=int(spec.seed), + ) diff --git a/tests/microbenchmarks/bench_moe/routing/materialize.py b/tests/microbenchmarks/bench_moe/routing/materialize.py new file mode 100644 index 000000000000..fe43eedf9336 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/routing/materialize.py @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Materialise a :class:`RoutingPlan` into runtime tensors and observers. + +The materialiser: + +* turns the per-rank slot dispatch matrix into a flat list of expert ids, +* repacks it column-major so each token row spans different destinations, +* runs a small repair pass to enforce per-token expert uniqueness, +* derives uniform top-k scales, +* observes the realised plan to compute slot / token traffic and per-rank + expert histograms used for the result schema's accuracy metrics. +""" + +from __future__ import annotations + +from typing import Dict, List, Tuple + +import torch + +from .builders import RoutingPlan, _largest_remainder_split + + +def _split_slot_count_to_experts( + slot_count: int, + target_histogram_row: List[int], +) -> List[int]: + """Allocate ``slot_count`` slots across local experts proportionally. + + Largest-remainder over ``target_histogram_row`` ensures the per-local-expert + distribution within this (src, dst) cell tracks the global histogram for + the target rank. Returns a list of length ``len(target_histogram_row)``. + """ + weights = [float(v) for v in target_histogram_row] + return _largest_remainder_split(int(slot_count), weights) + + +def _flatten_plan_slots_for_rank( + plan: RoutingPlan, + src_rank: int, + top_k: int, + experts_per_rank: int, + moe_ep_size: int, +) -> List[int]: + """Flatten one plan row into expert ids while preserving slot counts.""" + local_num_tokens = int(plan.per_rank_num_tokens[src_rank]) + row = list(plan.dispatch_matrix[src_rank]) + if sum(row) != local_num_tokens * top_k: + raise ValueError( + f"dispatch_matrix row sum ({sum(row)}) must equal local_num_tokens*top_k " + f"({local_num_tokens * top_k}) for rank {src_rank}" + ) + + flat: List[int] = [] + for dst in range(moe_ep_size): + cell = int(row[dst]) + if cell == 0: + continue + target_hist = list(plan.expert_histogram[dst]) + per_le = _split_slot_count_to_experts(cell, target_hist) + for le, cnt in enumerate(per_le): + if cnt <= 0: + continue + expert_id = dst * experts_per_rank + le + flat.extend([expert_id] * int(cnt)) + + expected = local_num_tokens * top_k + if len(flat) != expected: + raise ValueError( + f"materialiser flat length {len(flat)} != local_num_tokens*top_k={expected}" + ) + return flat + + +def _pack_slots_column_major(flat: List[int], local_num_tokens: int, top_k: int) -> List[List[int]]: + """Pack flat slots as k-major columns to spread destinations across tokens.""" + out = [[0] * top_k for _ in range(local_num_tokens)] + for i, val in enumerate(flat): + k_idx = i // local_num_tokens + t_idx = i % local_num_tokens + out[t_idx][k_idx] = val + return out + + +def _repair_duplicate_experts(out: List[List[int]], top_k: int) -> None: + """Best-effort repair so each token row has distinct selected experts.""" + max_passes = 4 + local_num_tokens = len(out) + for _pass in range(max_passes): + any_repair = False + for t in range(local_num_tokens): + seen: Dict[int, int] = {} + for k in range(top_k): + eid = out[t][k] + if eid in seen: + # Prefer swapping with the same k slot in another row; this + # preserves per-k distribution better than reshuffling the row. + target_k = k + swapped = False + for t2 in range(local_num_tokens): + if t2 == t: + continue + partner = out[t2][target_k] + if partner == eid: + continue + if partner in seen: + continue + out[t][target_k], out[t2][target_k] = partner, eid + swapped = True + any_repair = True + break + if not swapped: + # Last-resort intra-row swap. Some pathological plans + # cannot be repaired, and tests intentionally document + # those duplicate-producing cases. + for k2 in range(top_k): + if k2 == k: + continue + alt = out[t][k2] + if alt == eid or alt in seen: + continue + out[t][k], out[t][k2] = alt, eid + any_repair = True + break + seen[out[t][k]] = k + else: + seen[eid] = k + if not any_repair: + break + + +def _make_uniform_topk_scales( + local_num_tokens: int, + top_k: int, + *, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + return torch.full((local_num_tokens, top_k), 1.0 / max(top_k, 1), dtype=dtype, device=device) + + +def _materialize_selected_experts_for_rank( + plan: RoutingPlan, + src_rank: int, + top_k: int, + experts_per_rank: int, + moe_ep_size: int, + device: torch.device, + scale_dtype: torch.dtype, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Materialise ``[local_num_tokens, top_k]`` expert ids + uniform scales. + + The algorithm: + 1. Flatten ``dispatch_matrix[src_rank]`` into a slot-count-per-(dst, le) + table by splitting the row counts across local experts proportional + to the target rank's global histogram. + 2. Build a flat list of expert ids of length ``local_num_tokens * top_k``. + 3. Reshape column-major (k=0 first across tokens, then k=1, ...) so + that within a row consecutive slots come from different "buckets" and + per-token expert ids stay distinct in practice. + 4. Run a small repair pass that swaps duplicated expert ids between + rows until each token has ``top_k`` distinct experts. + """ + local_num_tokens = int(plan.per_rank_num_tokens[src_rank]) + if local_num_tokens == 0: + ids = torch.zeros((0, top_k), dtype=torch.int32, device=device) + scales = torch.zeros((0, top_k), dtype=scale_dtype, device=device) + return ids, scales + + flat = _flatten_plan_slots_for_rank(plan, src_rank, top_k, experts_per_rank, moe_ep_size) + out = _pack_slots_column_major(flat, local_num_tokens, top_k) + _repair_duplicate_experts(out, top_k) + + ids = torch.tensor(out, dtype=torch.int32, device=device) + scales = _make_uniform_topk_scales(local_num_tokens, top_k, device=device, dtype=scale_dtype) + return ids, scales + + +def _observe_routing_metrics( + plan: RoutingPlan, + selected_experts_per_rank: List[torch.Tensor], + experts_per_rank: int, + moe_ep_size: int, +) -> Tuple[List[List[int]], List[List[int]], List[List[int]]]: + """Derive observed slot/token traffic and expert histogram from materialised ids.""" + slot_traffic = [[0] * moe_ep_size for _ in range(moe_ep_size)] + token_traffic = [[0] * moe_ep_size for _ in range(moe_ep_size)] + expert_hist = [[0] * experts_per_rank for _ in range(moe_ep_size)] + for src, ids in enumerate(selected_experts_per_rank): + if ids is None or ids.numel() == 0: + continue + ids_cpu = ids.detach().cpu().numpy() if not isinstance(ids, list) else ids + for row in ids_cpu: + dst_visited = set() + for eid in row: + eid_int = int(eid) + dst = eid_int // experts_per_rank + le = eid_int % experts_per_rank + if 0 <= dst < moe_ep_size and 0 <= le < experts_per_rank: + slot_traffic[src][dst] += 1 + expert_hist[dst][le] += 1 + if dst not in dst_visited: + token_traffic[src][dst] += 1 + dst_visited.add(dst) + return slot_traffic, token_traffic, expert_hist + + +def _observe_summary( + requested_slot: List[List[int]], + observed_slot: List[List[int]], +) -> Tuple[int, float]: + """Return ``(max_abs_slot_error, max_relative_slot_error)``.""" + max_abs = 0 + max_rel = 0.0 + for src in range(len(observed_slot)): + for dst in range(len(observed_slot[src])): + req = int(requested_slot[src][dst]) if src < len(requested_slot) else 0 + obs = int(observed_slot[src][dst]) + abs_err = abs(obs - req) + if abs_err > max_abs: + max_abs = abs_err + denom = max(req, 1) + rel_err = abs_err / denom + if rel_err > max_rel: + max_rel = rel_err + return int(max_abs), float(max_rel) diff --git a/tests/microbenchmarks/bench_moe/routing/native_logits.py b/tests/microbenchmarks/bench_moe/routing/native_logits.py new file mode 100644 index 000000000000..449c38dd71ac --- /dev/null +++ b/tests/microbenchmarks/bench_moe/routing/native_logits.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native logits projection and supplied-topk forward patches. + +When ``RoutingControlSpec.routing_mode == "native"`` the benchmark constructs +synthetic ``router_logits`` that drive the production routing kernels to the +requested plan (and labels the outcome ``"exact"`` / ``"projected"`` / +``"rejected"`` based on the routing method's capability). When +``routing_mode == "forced"`` the routing kernels are bypassed entirely via +the supplied-topk patches in this module. +""" + +from __future__ import annotations + +import contextlib +from typing import Dict, Optional, Tuple + +import torch + +from tensorrt_llm._torch.modules.fused_moe.fused_moe_trtllm_gen import TRTLLMGenFusedMoE +from tensorrt_llm.tools.layer_wise_benchmarks.runner import make_forward_impl_check + +from .builders import RoutingPlan +from .materialize import _materialize_selected_experts_for_rank + +_NATIVE_PROJECTION_CAPABILITIES: Dict[str, str] = { + "DefaultMoeRoutingMethod": "exact_ids", + "RenormalizeMoeRoutingMethod": "exact", + "RenormalizeNaiveMoeRoutingMethod": "exact", + "SigmoidRenormMoeRoutingMethod": "exact_ids", + "Llama4RenormalizeMoeRoutingMethod": "top1_exact", + "MiniMaxM2MoeRoutingMethod": "exact_with_zero_bias", + "DeepSeekV3MoeRoutingMethod": "projected_or_exact", + "SparseMixerMoeRoutingMethod": "unsupported", +} + + +def _classify_native_projection( + *, + routing_method, + ids: torch.Tensor, + num_experts: int, + top_k: int, +) -> Tuple[str, str]: + """Return projection status/reason for a native routing method.""" + method_name = type(routing_method).__name__ + capability = _NATIVE_PROJECTION_CAPABILITIES.get(method_name, "unsupported") + status = "exact" + reason = "high/low logits drive top-k to plan" + + if capability == "exact": + return status, reason + if capability == "exact_ids": + return ( + status, + "expert ids exactly realised; selected_scales follow native routing kernel and are not " + "matrix-controlled", + ) + if capability == "top1_exact": + if top_k > 1: + return ( + "projected", + "Llama4 native realisation is only exact for top1; multi-target plans are projected", + ) + return status, reason + if capability == "exact_with_zero_bias": + return status, "MiniMax2 exact realisation assumes zero score-correction bias" + if capability == "projected_or_exact": + routing_impl = getattr(routing_method, "routing_impl", None) + n_group = getattr(routing_impl, "n_group", 1) if routing_impl is not None else 1 + topk_group = getattr(routing_impl, "topk_group", 1) if routing_impl is not None else 1 + if n_group > 1 and topk_group >= 1: + experts_per_group = num_experts // n_group + ids_cpu = ids.detach().cpu().tolist() + for row in ids_cpu: + groups = {int(eid) // experts_per_group for eid in row} + if len(groups) > topk_group: + return ( + "projected", + f"DeepSeekV3 grouped routing: row needs experts in {len(groups)} groups " + f"but topk_group={topk_group}", + ) + return status, reason + if capability == "unsupported": + return ( + "projected", + f"{method_name} native logits realisation is unsupported in v1; falling back to high/low logits", + ) + return "projected", f"{method_name}: unknown capability" + + +def _project_router_logits_for_plan( + plan: RoutingPlan, + src_rank: int, + routing_method, + num_experts: int, + top_k: int, + experts_per_rank: int, + moe_ep_size: int, + device: torch.device, + dtype: torch.dtype, + high_logit: float = 10.0, + low_logit: float = -10.0, +) -> Tuple[torch.Tensor, str, str]: + """Build router_logits for ``routing_method.apply`` matching the plan. + + Construct logits such that ``routing_method.apply`` yields a top-k matching + ``plan``'s ``[local_num_tokens, top_k]`` materialised expert ids. + + Returns ``(router_logits, status, reason)`` where ``status`` is one of + ``"exact"``, ``"projected"``, or ``"rejected"``. + """ + local_num_tokens = int(plan.per_rank_num_tokens[src_rank]) + if local_num_tokens == 0: + return ( + torch.empty((0, num_experts), dtype=dtype, device=device), + "exact", + "no local tokens; trivial logits", + ) + + # Materialise target experts using the canonical plan, then construct + # logits that drive the routing kernels towards those experts. + ids, _ = _materialize_selected_experts_for_rank( + plan, + src_rank=src_rank, + top_k=top_k, + experts_per_rank=experts_per_rank, + moe_ep_size=moe_ep_size, + device=device, + scale_dtype=dtype if dtype.is_floating_point else torch.bfloat16, + ) + + # Base low / high pattern with a small monotone perturbation by k index so + # that ties inside a row are broken consistently. + logits = torch.full((local_num_tokens, num_experts), low_logit, dtype=dtype, device=device) + k_offsets = torch.linspace(0.0, 1.0, steps=top_k, device=device, dtype=dtype) * 0.01 + # Score per slot decreases with k_idx so that top-k tie-breaking yields the + # same expert ordering when scales are derived from logits. + score_per_k = high_logit + (k_offsets.flip(dims=(0,)) - 0.005) + row_idx = ( + torch.arange(local_num_tokens, device=device).unsqueeze(1).expand(local_num_tokens, top_k) + ) + logits[row_idx, ids.long()] = score_per_k.unsqueeze(0).expand_as(ids).to(dtype) + + status, reason = _classify_native_projection( + routing_method=routing_method, ids=ids, num_experts=num_experts, top_k=top_k + ) + return logits, status, reason + + +def _align_topk_to_batch( + local: torch.Tensor, scales: torch.Tensor, batch_rows: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """Align materialised routing tensors to runtime batch rows. + + CUDA graph capture may pad or trim the local batch dimension. Repeating the + final row is only used for over-allocation and keeps the synthetic routing + payload well-formed without changing the steady-state path. + """ + if local.shape[0] == batch_rows: + return local, scales + if local.shape[0] >= batch_rows: + return local[:batch_rows], scales[:batch_rows] + pad_rows = batch_rows - local.shape[0] + return ( + torch.cat([local, local[-1:].expand(pad_rows, -1).clone()], dim=0), + torch.cat([scales, scales[-1:].expand(pad_rows, -1).clone()], dim=0), + ) + + +def _make_supplied_topk_run_moe( + moe_module, + run_moe_orig, + materialized_ids: torch.Tensor, + materialized_scales: torch.Tensor, +): + """Return a ``run_moe`` wrapper that injects pre-materialised top-k tensors. + + Mirrors the ``make_balanced_run_moe`` helper in the layer-wise benchmark + runner, but feeds the routing-control plan instead of the legacy + balanced/imbalanced selection helpers. + """ + + def supplied_run_moe( + x, token_selected_experts, token_final_scales, x_sf, router_logits, do_finalize, moe_output + ): + if getattr(moe_module, "_routing_results_replaced_at", None) is not None: + return run_moe_orig( + x, + token_selected_experts, + token_final_scales, + x_sf, + router_logits, + do_finalize, + moe_output, + ) + local, scales = _align_topk_to_batch(materialized_ids, materialized_scales, x.shape[0]) + local = local.to(device=x.device, dtype=torch.int32) + scales = scales.to(device=x.device) + final_hidden_states = run_moe_orig(x, local, scales, x_sf, None, do_finalize, moe_output) + if not do_finalize: + final_hidden_states = ( + final_hidden_states[0], + scales, + final_hidden_states[2], + ) + moe_module._routing_results_replaced_at = "make_supplied_topk_run_moe" + return final_hidden_states + + return supplied_run_moe + + +def _make_supplied_topk_apply( + moe_module, + materialized_ids: torch.Tensor, + materialized_scales: torch.Tensor, +): + """Return a ``routing_method.apply`` wrapper returning the plan directly.""" + + def supplied_apply(router_logits): + local = materialized_ids + scales = materialized_scales + if router_logits is not None: + local, scales = _align_topk_to_batch(local, scales, router_logits.shape[0]) + device = router_logits.device if router_logits is not None else local.device + moe_module._routing_results_replaced_at = "make_supplied_topk_apply" + return local.to(device=device, dtype=torch.int32), scales.to(device=device) + + return supplied_apply + + +@contextlib.contextmanager +def _maybe_install_routing_control_patch( + moe, + materialized_ids: Optional[torch.Tensor], + materialized_scales: Optional[torch.Tensor], + active: bool, +): + """Install supplied-topk patches when routing control is active in forced mode. + + For non-TRTLLM backends we override ``routing_method.apply`` to return the + pre-materialised ``(ids, scales)`` pair. For ``TRTLLMGenFusedMoE`` the + fused TEP path needs ``run_moe`` to be patched as well, mirroring the + layer-wise benchmark's ``make_balanced_run_moe`` flow. + + ``active=False`` makes this a no-op pass-through so legacy callers keep + behaving exactly as before. + """ + if not active or materialized_ids is None or materialized_scales is None: + yield + return + + routing_target = moe + apply_method_orig = routing_target.routing_method.apply + inner_backend = getattr(moe, "backend", moe) + run_moe_orig = None + forward_impl_orig = moe.forward_impl + + try: + routing_target.routing_method.apply = _make_supplied_topk_apply( + routing_target, materialized_ids, materialized_scales + ) + + if isinstance(inner_backend, TRTLLMGenFusedMoE): + run_moe_orig = inner_backend.run_moe + inner_backend.run_moe = _make_supplied_topk_run_moe( + inner_backend, run_moe_orig, materialized_ids, materialized_scales + ) + + moe.forward_impl = make_forward_impl_check(moe, forward_impl_orig) + yield + finally: + routing_target.routing_method.apply = apply_method_orig + if run_moe_orig is not None: + inner_backend.run_moe = run_moe_orig + moe.forward_impl = forward_impl_orig diff --git a/tests/microbenchmarks/bench_moe/routing/parsing.py b/tests/microbenchmarks/bench_moe/routing/parsing.py new file mode 100644 index 000000000000..006548bc970d --- /dev/null +++ b/tests/microbenchmarks/bench_moe/routing/parsing.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pattern-spec parsers for ``--comm_pattern`` / ``--expert_pattern``.""" + +from __future__ import annotations + +from typing import Any, Dict, Tuple + +_COMM_PATTERN_NAMES: Tuple[str, ...] = ( + "random", + "balanced_alltoall", + "receiver_hotspot", + "pair_hotspot", + "local_only", + "ring", +) + +_EXPERT_PATTERN_NAMES: Tuple[str, ...] = ("random", "balanced", "hotspot") + + +def _parse_pattern_spec(spec: str) -> Tuple[str, Dict[str, str]]: + """Parse ``name,k1=v1,k2=v2`` into ``(name, {k1: v1, k2: v2})``. + + File-based routing control is handled by ``--routing_pattern_file``. + """ + raw = str(spec).strip() + if not raw: + raise ValueError("empty pattern spec") + if raw.startswith("file:"): + return "file", {"path": raw[len("file:") :]} + parts = [p.strip() for p in raw.split(",") if p.strip()] + if not parts: + raise ValueError(f"invalid pattern spec: {spec!r}") + name = parts[0] + kwargs: Dict[str, str] = {} + for part in parts[1:]: + if "=" not in part: + raise ValueError(f"invalid pattern fragment {part!r} in {spec!r}; expected k=v") + k, v = part.split("=", 1) + kwargs[k.strip()] = v.strip() + return name, kwargs + + +def _pop_hotness_kwarg(raw: Dict[str, str], kwargs: Dict[str, Any], *, label: str) -> None: + """Parse the optional ``hotness=`` shared by comm/expert patterns.""" + if "hotness" not in raw: + return + value = float(raw["hotness"]) + if not 0.0 <= value <= 1.0: + raise ValueError(f"{label} hotness must be in [0, 1]; got {value}") + kwargs["hotness"] = value + + +def _parse_typed_pattern( + spec: str, *, label: str, valid_names: Tuple[str, ...] +) -> Tuple[str, Dict[str, str]]: + """Common prefix of ``_parse_comm_pattern`` / ``_parse_expert_pattern``. + + Parses the ``name[:k=v,...]`` form, rejects the legacy ``file:`` + prefix (now handled via ``--routing_pattern_file``), and validates that + ``name`` is one of the supported pattern names for ``label``. + """ + name, raw = _parse_pattern_spec(spec) + if name == "file": + raise ValueError(f"{label} no longer accepts file:; use --routing_pattern_file") + if name not in valid_names: + raise ValueError(f"unknown {label} {name!r}; supported: {valid_names}") + return name, raw + + +def _parse_comm_pattern(spec: str) -> Tuple[str, Dict[str, Any]]: + name, raw = _parse_typed_pattern(spec, label="comm_pattern", valid_names=_COMM_PATTERN_NAMES) + kwargs: Dict[str, Any] = {} + _pop_hotness_kwarg(raw, kwargs, label="comm_pattern") + for int_key in ("rank", "src", "dst"): + if int_key in raw: + kwargs[int_key] = int(raw[int_key]) + if name == "receiver_hotspot": + if "hotness" not in kwargs: + raise ValueError("receiver_hotspot requires hotness=") + kwargs.setdefault("rank", 0) + if name == "pair_hotspot": + if "hotness" not in kwargs or "src" not in kwargs or "dst" not in kwargs: + raise ValueError("pair_hotspot requires hotness=, src=, dst=") + return name, kwargs + + +def _parse_expert_pattern(spec: str) -> Tuple[str, Dict[str, Any]]: + name, raw = _parse_typed_pattern( + spec, label="expert_pattern", valid_names=_EXPERT_PATTERN_NAMES + ) + kwargs: Dict[str, Any] = {} + _pop_hotness_kwarg(raw, kwargs, label="expert_pattern") + if "active_experts" in raw: + kwargs["active_experts"] = int(raw["active_experts"]) + if kwargs["active_experts"] <= 0: + raise ValueError("expert_pattern active_experts must be > 0") + if name == "hotspot" and "hotness" not in kwargs and "active_experts" not in kwargs: + raise ValueError( + "expert_pattern hotspot requires hotness= or active_experts=" + ) + return name, kwargs diff --git a/tests/microbenchmarks/bench_moe/search.py b/tests/microbenchmarks/bench_moe/search.py new file mode 100644 index 000000000000..49f539f212e7 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/search.py @@ -0,0 +1,394 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Search-space expansion and candidate pruning for bench_moe.""" + +from __future__ import annotations + +import argparse +import itertools +from dataclasses import replace +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import torch + +from tensorrt_llm.models.modeling_utils import QuantAlgo + +from .backend import MoeBackendType, get_backend_class +from .mapping import _resolve_mapping_layout +from .specs import _ALL_BACKENDS, _FORCED_COMM_ENV_VALUES, ConfigSpec, ModelSpec, SearchSpec + +_FUSED_COMM_BACKENDS = frozenset({"MEGAMOE_DEEPGEMM"}) + + +def _check_backend_can_implement( + backend_str: str, + quant_algo: Optional[QuantAlgo], + dtype_activation: torch.dtype, + swiglu_gptoss_style: bool, +) -> Tuple[bool, Optional[str]]: + """Resolve backend_str to its MoE class and forward to can_implement.""" + try: + backend_cls = get_backend_class(MoeBackendType(backend_str.upper())) + except (ImportError, KeyError, RuntimeError, ValueError) as exc: + return False, f"unknown MoE backend {backend_str!r}: {exc}" + try: + return backend_cls.can_implement( + quant_algo=quant_algo, + dtype_activation=dtype_activation, + swiglu_gptoss_style=swiglu_gptoss_style, + ) + except Exception as exc: + return False, (f"{backend_cls.__name__}.can_implement raised {type(exc).__name__}: {exc}") + + +def _expand_axis(values: Iterable[Any], default: Any) -> Tuple[Any, ...]: + out = tuple(values) + return out if out else (default,) + + +def _comm_axis_for_backend(backend: Any, comm_methods: Tuple[Any, ...]) -> Tuple[Any, ...]: + if str(backend).upper() in _FUSED_COMM_BACKENDS: + return ("NONE",) + return comm_methods + + +def expand_search( + base_config: ConfigSpec, + search: SearchSpec, + world_size: int, +) -> List[ConfigSpec]: + """Cartesian-product candidate generation, then explicit pruning. + + ``base_config`` carries the *non-search* fields (cuda_graph default, + combine precision default). Search axes + explicitly listed on ``search`` override the base values. + """ + backends = _expand_axis(search.backends, base_config.backend) + parallel_modes = _expand_axis(search.parallel_modes, base_config.parallel_mode) + comm_methods = _expand_axis(search.comm_methods, base_config.comm_method) + cuda_graph_options = _expand_axis(search.cuda_graph_options, base_config.cuda_graph) + combine_options = _expand_axis( + search.combine_precision_options, base_config.use_low_precision_moe_combine + ) + + candidates: List[ConfigSpec] = [] + for backend, pmode, cgraph, combine in itertools.product( + backends, parallel_modes, cuda_graph_options, combine_options + ): + for comm in _comm_axis_for_backend(backend, comm_methods): + candidate = replace( + base_config, + backend=str(backend).upper(), + parallel_mode=str(pmode).upper(), + comm_method=str(comm).upper(), + cuda_graph=bool(cgraph), + use_low_precision_moe_combine=bool(combine), + ) + candidates.append(candidate) + return candidates + + +def is_candidate_valid( + config: ConfigSpec, + model: ModelSpec, + world_size: int, + act_dtype: torch.dtype, +) -> Tuple[bool, Optional[str]]: + """Return ``(ok, reason)`` based on backend / mapping / comm gates.""" + # Backend can_implement gate. + ok, reason = _check_backend_can_implement( + config.backend, model.quant_algo_enum, act_dtype, model.swiglu_gptoss_style + ) + if not ok: + return False, reason + + # Mapping layout gate. + try: + moe_ep, moe_tp, enable_dp = _resolve_mapping_layout(config, world_size) + except ValueError as exc: + return False, str(exc) + + # Forced communication on non-DP / MoE-TP paths. + forced = config.comm_method.upper() + if forced not in ("AUTO", "NONE"): + if not enable_dp: + return False, f"comm_method={forced} requires enable_attention_dp=True" + if moe_tp != 1 and forced != "ALLGATHER": + return False, f"comm_method={forced} requires moe_tp_size=1 (got {moe_tp})" + if world_size == 1: + return False, f"comm_method={forced} has no effect at world_size=1" + + return True, None + + +def expand_and_prune( + base_config: ConfigSpec, + search: SearchSpec, + model: ModelSpec, + world_size: int, + act_dtype: torch.dtype, + max_configs: Optional[int] = None, +) -> Tuple[List[ConfigSpec], Dict[ConfigSpec, str]]: + """Expand search and split into ``(valid_candidates, skip_reasons_for_invalid)``. + + ``max_configs`` truncates the *valid* list after pruning; the skipped / + invalid candidates are reported in full so dashboard rows do not silently + disappear. + """ + raw = expand_search(base_config, search, world_size) + valid: List[ConfigSpec] = [] + skipped: Dict[ConfigSpec, str] = {} + for cand in raw: + ok, reason = is_candidate_valid(cand, model, world_size, act_dtype) + if ok: + valid.append(cand) + else: + skipped[cand] = reason or "invalid" + if max_configs is not None and max_configs >= 0 and len(valid) > max_configs: + truncated = valid[max_configs:] + valid = valid[:max_configs] + for cand in truncated: + skipped[cand] = f"truncated by --max_configs={max_configs}" + return valid, skipped + + +def _maybe_auto_enable_search_axes(args: argparse.Namespace) -> None: + """Promote multi-value runtime flags into ``--search`` axes when needed. + + Rules: + - Passing >1 value to ``--backend``/``--comm_method``/``--parallel_mode`` + (or ``--backend ALL``) implicitly enables the matching ``--search`` axis + so users do not have to repeat themselves. + - A single value keeps the prior single-config behavior intact. + - An explicit ``--search none`` together with a multi-value flag is an + error -- the conflicting intent is surfaced rather than silently + overridden. + - ``--search full`` already covers every axis; the auto-promote logic is + a no-op in that case. + """ + provided = set(getattr(args, "_cli_provided", set())) + current = tuple(args.search) if args.search else ("none",) + + if current == ("full",): + return + + promote: List[str] = [] + backends = _coerce_str_tuple(getattr(args, "backend", ())) + if "backend" in provided and (len(backends) > 1 or backends == ("ALL",)): + promote.append("backend") + comm_methods = _coerce_str_tuple(getattr(args, "comm_method", ())) + if "comm_method" in provided and len(comm_methods) > 1: + promote.append("comm") + parallel_modes = _coerce_str_tuple(getattr(args, "parallel_mode", ())) + if "parallel_mode" in provided and len(parallel_modes) > 1: + promote.append("parallel") + + # Reject CUSTOM in a multi-value parallel sweep -- CUSTOM still needs scalar + # --moe_ep_size / --moe_tp_size so it cannot be combined with other modes. + if len(parallel_modes) > 1 and "CUSTOM" in parallel_modes: + raise ValueError( + "--parallel_mode CUSTOM must be passed alone; it requires --moe_ep_size " + f"and --moe_tp_size and cannot be combined with other modes (got {list(parallel_modes)})." + ) + + if not promote: + return + + if current == ("none",) and "search" in provided: + raise ValueError( + "--search none conflicts with a multi-value runtime flag " + f"(would promote axes {promote}). Drop --search none or pass a single value per axis." + ) + + if current == ("none",): + new_axes = tuple(promote) + else: + merged: List[str] = list(current) + for axis in promote: + if axis not in merged: + merged.append(axis) + new_axes = tuple(merged) + + args.search = _parse_search_axes(new_axes) + + +def _coerce_str_tuple(val: Any) -> Tuple[str, ...]: + """Normalize ``nargs="+"`` argparse fields into an upper-cased str tuple. + + Accepts the post-parse value of ``--backend``/``--comm_method``/ + ``--parallel_mode`` regardless of whether it came in as a single string + (e.g. from a config file or default) or a list (argparse ``nargs="+"``). + """ + if val is None: + return () + if isinstance(val, (list, tuple)): + return tuple(str(v).upper() for v in val if str(v).strip()) + return (str(val).upper(),) + + +_SEARCH_AXES = ("backend", "comm", "parallel") +_SEARCH_MODES = ("none",) + _SEARCH_AXES + ("full",) + + +def _normalize_csv_tokens(value: Any) -> List[str]: + """Normalise a ``nargs='+'`` / comma-separated / scalar input into tokens. + + Splits on commas and whitespace, strips, lowercases, and drops empty + fragments. Preserves input order so the caller can deduplicate while + keeping a stable axis order in error messages. + """ + items = value if isinstance(value, (list, tuple)) else [value] + return [ + part.strip().lower() + for item in items + for part in str(item).replace(",", " ").split() + if part.strip() + ] + + +def _parse_search_axes(value: Any) -> Tuple[str, ...]: + parts = _normalize_csv_tokens(value) + if not parts: + return ("none",) + + out: List[str] = [] + for part in parts: + if part not in _SEARCH_MODES: + raise ValueError(f"unknown --search axis {part!r}; valid: {list(_SEARCH_MODES)}") + if part not in out: + out.append(part) + + if "none" in out and len(out) > 1: + raise ValueError("--search none cannot be combined with other axes") + if "full" in out and len(out) > 1: + raise ValueError("--search full cannot be combined with other axes") + return tuple(out) + + +_DEFAULT_PARALLEL_AXIS_VALUES: Tuple[str, ...] = ("DEP", "TEP", "DTP", "TTP") + + +def _axis_values_from_args( + args: argparse.Namespace, + *, + cli_dest: str, + cli_flag_name: str, + config_key: Optional[str], + full_set: Tuple[str, ...], +) -> Tuple[str, ...]: + """Resolve the value set for a search axis. + + Resolution order (highest priority first): + 1. ``args._config_search_axes[config_key]`` if the JSON config provided a list + and the user did not also pass the corresponding CLI flag. + 2. The CLI flag list if the user explicitly provided it (``ALL`` expands to + ``full_set`` for the backend axis). + 3. ``full_set`` -- the default when the axis is enabled but no explicit + subset was given. This replaces the previous footgun where a bare + ``--search backend`` would silently expand to a single default value. + """ + config_axes = getattr(args, "_config_search_axes", {}) or {} + provided = set(getattr(args, "_cli_provided", set())) + + if config_key is not None and config_key in config_axes and cli_dest not in provided: + return tuple(config_axes[config_key]) + + if cli_dest in provided: + values = _coerce_str_tuple(getattr(args, cli_dest)) + if not values: + return full_set + if cli_dest == "backend" and "ALL" in values: + if len(values) != 1: + raise ValueError(f"{cli_flag_name} ALL must be passed alone (got {list(values)}).") + return full_set + return values + + return full_set + + +def _resolve_search_from_args(args: argparse.Namespace, base_config: ConfigSpec) -> SearchSpec: + search_axes = _parse_search_axes(args.search) + + if search_axes == ("none",): + return SearchSpec(mode="none") + + full_search = search_axes == ("full",) + enabled_axes = set(_SEARCH_AXES if full_search else search_axes) + mode = "full" if full_search else ",".join(search_axes) + + backends: Tuple[str, ...] = () + parallel_modes: Tuple[str, ...] = () + comm_methods: Tuple[str, ...] = () + cuda_graph_options: Tuple[bool, ...] = () + combine_options: Tuple[bool, ...] = () + + if "backend" in enabled_axes: + backends = _axis_values_from_args( + args, + cli_dest="backend", + cli_flag_name="--backend", + config_key="backend", + full_set=tuple(_ALL_BACKENDS), + ) + if "parallel" in enabled_axes: + parallel_modes = _axis_values_from_args( + args, + cli_dest="parallel_mode", + cli_flag_name="--parallel_mode", + config_key="parallel_mode", + full_set=_DEFAULT_PARALLEL_AXIS_VALUES, + ) + if "comm" in enabled_axes: + comm_methods = _axis_values_from_args( + args, + cli_dest="comm_method", + cli_flag_name="--comm_method", + config_key="comm_method", + full_set=_FORCED_COMM_ENV_VALUES, + ) + comm_methods = tuple(v for v in comm_methods if str(v).upper() != "AUTO") + if not comm_methods: + raise ValueError( + "--search comm does not include AUTO because AUTO aliases one of the concrete " + "communication strategies; pass a forced --comm_method value or disable comm search." + ) + # ``cuda_graph`` and ``combine_precision`` axes are reserved for ``full``; + # leave them empty by default so the base value is used. + if full_search: + cuda_graph_options = (True, False) + + return SearchSpec( + mode=mode, + backends=backends, + parallel_modes=parallel_modes, + comm_methods=comm_methods, + cuda_graph_options=cuda_graph_options, + combine_precision_options=combine_options, + ) + + +def _parse_analysis(value: Any) -> Tuple[str, ...]: + parts = _normalize_csv_tokens(value) + valid = {"none", "kernels"} + out: List[str] = [] + for p in parts: + if p not in valid: + raise ValueError(f"unknown --analysis dimension {p!r}; valid: {sorted(valid)}") + if p == "none": + continue + if p not in out: + out.append(p) + return tuple(out) diff --git a/tests/microbenchmarks/bench_moe/specs.py b/tests/microbenchmarks/bench_moe/specs.py new file mode 100644 index 000000000000..36134c9b429a --- /dev/null +++ b/tests/microbenchmarks/bench_moe/specs.py @@ -0,0 +1,356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Structured specs used throughout the bench_moe pipeline. + +Spec dataclasses are intentionally light: they hold values only, expose tiny +helpers for JSON serialisation (used by the dashboard schema), and rely on +sibling modules to convert them into runtime objects (Mappings, MoE modules, +RoutingMethods, etc.). +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List, Optional, Tuple + +from tensorrt_llm._torch.modules.fused_moe.routing import ( + DeepSeekV3MoeRoutingMethod, + DefaultMoeRoutingMethod, + Llama4RenormalizeMoeRoutingMethod, + MiniMaxM2MoeRoutingMethod, + RenormalizeMoeRoutingMethod, + RenormalizeNaiveMoeRoutingMethod, + SigmoidRenormMoeRoutingMethod, +) +from tensorrt_llm.models.modeling_utils import QuantAlgo + +from .backend import MoeBackendType, MoeModelConfig + +_ROUTING_METHODS: Dict[str, type] = { + "DEFAULT": DefaultMoeRoutingMethod, + "RENORMALIZE": RenormalizeMoeRoutingMethod, + "RENORMALIZE_NAIVE": RenormalizeNaiveMoeRoutingMethod, + "LLAMA4_RENORMALIZE": Llama4RenormalizeMoeRoutingMethod, + "DEEPSEEK_V3": DeepSeekV3MoeRoutingMethod, + "MINIMAX_M2": MiniMaxM2MoeRoutingMethod, + "SIGMOID_RENORM": SigmoidRenormMoeRoutingMethod, +} + +_ROUTING_NAME_BY_CLS: Dict[type, str] = {cls: name for name, cls in _ROUTING_METHODS.items()} +_ALL_BACKENDS: List[str] = [b.value for b in MoeBackendType] +_COMM_METHODS: Tuple[str, ...] = ( + "AUTO", + "NVLINK_ONE_SIDED", + "NVLINK_TWO_SIDED", + "DEEPEP", + "DEEPEPLOWLATENCY", + "ALLGATHER", +) +_FORCED_COMM_ENV_VALUES: Tuple[str, ...] = ( + "NVLINK_ONE_SIDED", + "NVLINK_TWO_SIDED", + "DEEPEP", + "DEEPEPLOWLATENCY", + "ALLGATHER", +) + + +def _to_jsonable_dict(obj: Any) -> Dict[str, Any]: + """``dataclasses.asdict`` with nested tuples converted to lists. + + Several specs use ``Tuple[...]`` fields for hashability/immutability. + ``asdict`` preserves tuples; downstream consumers serialize the result + to JSON, which treats tuples and lists identically, but list form is + the historical wire format and avoids surprising callers that may + later mutate. + """ + + def _walk(value: Any) -> Any: + if isinstance(value, (tuple, list)): + return [_walk(v) for v in value] + if isinstance(value, dict): + return {k: _walk(v) for k, v in value.items()} + return value + + return _walk(asdict(obj)) + + +@dataclass(frozen=True) +class ModelSpec: + """Static MoE model description. + + A built-in name resolves to one of the entries in ``BUILT_IN_MODELS``. + Custom shapes pass ``name="custom"`` and fill the remaining fields + explicitly. ``routing_method`` is the registry key from + ``_ROUTING_METHODS``; resolution to a concrete class happens lazily so the + spec stays JSON-serializable. + """ + + name: str + num_experts: int + top_k: int + hidden_size: int + intermediate_size: int + quant_algo: Optional[str] + routing_method: str + n_group: Optional[int] = None + topk_group: Optional[int] = None + swiglu_alpha: float = 1.0 + swiglu_beta: float = 0.0 + swiglu_limit: float = float("inf") + + @property + def routing_method_cls(self) -> type: + return _ROUTING_METHODS[self.routing_method] + + @property + def quant_algo_enum(self) -> Optional[QuantAlgo]: + return QuantAlgo[self.quant_algo] if self.quant_algo is not None else None + + @property + def swiglu_gptoss_style(self) -> bool: + return ( + self.swiglu_alpha != 1.0 or self.swiglu_beta != 0.0 or self.swiglu_limit != float("inf") + ) + + def to_moe_model_config(self) -> MoeModelConfig: + return MoeModelConfig( + num_experts=int(self.num_experts), + top_k=int(self.top_k), + hidden_size=int(self.hidden_size), + intermediate_size=int(self.intermediate_size), + n_group=self.n_group, + topk_group=self.topk_group, + ) + + def to_dict(self) -> Dict[str, Any]: + d = _to_jsonable_dict(self) + d["routing_method_class"] = self.routing_method_cls.__name__ + return d + + +@dataclass(frozen=True) +class RoutingControlSpec: + """Advanced routing-control knobs for one workload. + + ``comm_pattern`` and ``expert_pattern`` describe the requested traffic + shape; ``routing_mode`` picks between native logits realization and forced + supplied-topk; ``projection_policy`` controls what happens when native + logits cannot exactly express the requested pattern. + + By default, balanced patterns build a deterministic RoutingPlan that is + projected to native logits or supplied directly, depending on + ``routing_mode``. Set both ``comm_pattern=random`` and + ``expert_pattern=random`` to leave routing uncontrolled and use random + router logits. + """ + + routing_mode: str = "native" # "native" | "forced" + projection_policy: str = "project" # "project" | "reject" + comm_pattern: str = "balanced_alltoall" + expert_pattern: str = "balanced" + routing_pattern_file: Optional[str] = None + per_rank_num_tokens: Optional[Tuple[int, ...]] = None + routing_dump_matrix: bool = False + seed: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return _to_jsonable_dict(self) + + @property + def is_active(self) -> bool: + """True when this spec asks for planned routing instead of random logits. + + Used to decide whether to dispatch through routing-control planning or + keep the normal benchmark path. + """ + return ( + self.routing_mode != "native" + or self.routing_pattern_file is not None + or self.per_rank_num_tokens is not None + or not (self.comm_pattern == "random" and self.expert_pattern == "random") + ) + + +@dataclass(frozen=True) +class WorkloadSpec: + """Workload for one timing case after the model is fixed.""" + + num_tokens: int + routing_control: RoutingControlSpec = field(default_factory=RoutingControlSpec) + + def to_dict(self, per_rank_num_tokens: Optional[List[int]] = None) -> Dict[str, Any]: + return { + "num_tokens": int(self.num_tokens), + "per_rank_num_tokens": ( + [int(v) for v in per_rank_num_tokens] if per_rank_num_tokens is not None else None + ), + "routing_control": self.routing_control.to_dict(), + } + + +@dataclass(frozen=True) +class ConfigSpec: + """One executable MoE runtime configuration.""" + + backend: str + parallel_mode: str # "DEP" | "TEP" | "DTP" | "TTP" | "CUSTOM" + moe_ep_size: Optional[int] = None + moe_tp_size: Optional[int] = None + enable_attention_dp: Optional[bool] = None + comm_method: str = "AUTO" + cuda_graph: bool = True + use_low_precision_moe_combine: bool = False + + def to_dict(self) -> Dict[str, Any]: + return _to_jsonable_dict(self) + + +@dataclass(frozen=True) +class SearchSpec: + """Description of which ConfigSpec axes to expand into candidates.""" + + mode: str = "none" # "none" | "backend" | "comm" | "parallel" | "full" | comma-joined axes + backends: Tuple[str, ...] = () + parallel_modes: Tuple[str, ...] = () + comm_methods: Tuple[str, ...] = () + cuda_graph_options: Tuple[bool, ...] = () + combine_precision_options: Tuple[bool, ...] = () + + def to_dict(self) -> Dict[str, Any]: + return _to_jsonable_dict(self) + + +@dataclass +class RunResult: + """Result of timing a single ``(model, workload, config)`` triple. + + Stays a regular dataclass on the worker side so we can mutate fields while + incrementally collecting data. + """ + + model: ModelSpec + workload: WorkloadSpec + config: ConfigSpec + status: str = "success" # "success" | "skipped" | "failed" + skip_reason: Optional[str] = None + actual_backend: Optional[str] = None + actual_comm_method: Optional[str] = None + actual_comm_fallback_reason: Optional[str] = None + scheduler_kind: Optional[str] = None + moe_ep_size: Optional[int] = None + moe_tp_size: Optional[int] = None + enable_attention_dp: Optional[bool] = None + num_chunks: Optional[int] = None + per_rank_num_tokens: List[int] = field(default_factory=list) + status_per_rank: Dict[str, str] = field(default_factory=dict) + instrumentation: Dict[str, Any] = field(default_factory=dict) + latency_ms: Dict[str, Any] = field(default_factory=dict) + phase_times_ms: Dict[str, Any] = field(default_factory=dict) + kernel_breakdown: Dict[str, Any] = field(default_factory=dict) + raw_data: Dict[str, Any] = field(default_factory=dict) + overlap: Dict[str, Any] = field(default_factory=dict) + bottleneck: Optional[str] = None + routing_control: Dict[str, Any] = field(default_factory=dict) + + +# Built-in MoE model registry. Each entry is a ready-to-run ``ModelSpec`` for a +# publicly known MoE checkpoint. + +BUILT_IN_MODELS: Dict[str, ModelSpec] = { + "qwen1.5_moe": ModelSpec( + name="qwen1.5_moe", + num_experts=60, + top_k=4, + hidden_size=2048, + intermediate_size=1408, + quant_algo="FP8", + routing_method="RENORMALIZE", + ), + "deepseek_v2_lite": ModelSpec( + name="deepseek_v2_lite", + num_experts=64, + top_k=6, + hidden_size=2048, + intermediate_size=1408, + quant_algo="FP8_BLOCK_SCALES", + routing_method="DEEPSEEK_V3", + ), + "deepseek_v3": ModelSpec( + name="deepseek_v3", + num_experts=256, + top_k=8, + hidden_size=7168, + intermediate_size=2048, + quant_algo="FP8_BLOCK_SCALES", + routing_method="DEEPSEEK_V3", + n_group=8, + topk_group=4, + ), + "kimi_k2": ModelSpec( + name="kimi_k2", + num_experts=384, + top_k=8, + hidden_size=7168, + intermediate_size=2048, + quant_algo="FP8_BLOCK_SCALES", + routing_method="DEEPSEEK_V3", + ), + # DeepSeek-V4-Pro: 1.6T total / 49B activated. quant_algo intentionally + # left None: pass --quant on the CLI to pin the mode (the released + # checkpoint mixes FP4 experts with FP8 elsewhere which has no single + # QuantAlgo match). + "deepseek_v4_pro": ModelSpec( + name="deepseek_v4_pro", + num_experts=384, + top_k=6, + hidden_size=7168, + intermediate_size=3072, + quant_algo=None, + routing_method="RENORMALIZE", + ), + # DeepSeek-V4-Flash: 284B total / 13B activated. + "deepseek_v4_flash": ModelSpec( + name="deepseek_v4_flash", + num_experts=256, + top_k=6, + hidden_size=4096, + intermediate_size=2048, + quant_algo=None, + routing_method="RENORMALIZE", + ), + "mixtral_8x7b": ModelSpec( + name="mixtral_8x7b", + num_experts=8, + top_k=2, + hidden_size=4096, + intermediate_size=14336, + quant_algo="FP8", + routing_method="RENORMALIZE", + ), + "gpt_oss_120b": ModelSpec( + name="gpt_oss_120b", + num_experts=128, + top_k=4, + hidden_size=2880, + intermediate_size=2880, + quant_algo="W4A8_MXFP4_MXFP8", + routing_method="RENORMALIZE", + swiglu_alpha=1.702, + swiglu_beta=1.0, + swiglu_limit=7.0, + ), +} diff --git a/tests/microbenchmarks/bench_moe/timing/__init__.py b/tests/microbenchmarks/bench_moe/timing/__init__.py new file mode 100644 index 000000000000..da5ef4e92c4c --- /dev/null +++ b/tests/microbenchmarks/bench_moe/timing/__init__.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-case timing primitives. + +The benchmark runs three timing variants over the same MoE module: + +* :mod:`bench_moe.timing.autotune` -- untimed pre-pass that warms the kernel + caches before any iteration is timed. +* :mod:`bench_moe.timing.eager` -- eager ``moe.forward`` timing via + ``torch.cuda.Event`` plus an optional Kineto profiler pass for kernel + breakdown. +* :mod:`bench_moe.timing.cuda_graph` -- CUDA Graph capture+replay timing with + graph-internal external CUDA events. +* :mod:`bench_moe.timing.cupti` -- optional CUPTI activity tracking used + by the CUDA Graph path to get a kernel-level breakdown for replays that + Kineto cannot see. +""" + +from .autotune import _run_autotune +from .cuda_graph import _time_moe_forward_cuda_graph +from .cupti import ( + _build_cuda_graph_kernel_stats_cupti, + _CuptiContext, + _demangle_names, + _try_init_cupti, +) +from .eager import ( + _kernel_times_to_summary_list, + _l2_flush_buffer, + _parse_profiler_events_moe, + _time_moe_forward_eager, +) + +__all__ = [ + "_CuptiContext", + "_build_cuda_graph_kernel_stats_cupti", + "_demangle_names", + "_kernel_times_to_summary_list", + "_l2_flush_buffer", + "_parse_profiler_events_moe", + "_run_autotune", + "_time_moe_forward_cuda_graph", + "_time_moe_forward_eager", + "_try_init_cupti", +] diff --git a/tests/microbenchmarks/bench_moe/timing/autotune.py b/tests/microbenchmarks/bench_moe/timing/autotune.py new file mode 100644 index 000000000000..f993021be407 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/timing/autotune.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Untimed autotune pre-pass.""" + +from __future__ import annotations + +import os +import tempfile +from typing import List + +import torch + +from tensorrt_llm._torch.autotuner import AutoTuner, autotune + + +def _run_autotune( + moe, + x: torch.Tensor, + router_logits: torch.Tensor, + all_rank_num_tokens: List[int], + fast_autotune: bool, +) -> str: + """One untimed forward pass under ``autotune(...)`` to populate kernel caches. + + Returns an autotune status string, one of: + - ``"success"`` : ran with the project default tuner settings + - ``"success:fast"`` : ran with ``--fast_autotune`` overrides (lower quality) + - ``"failed:"``: the autotune pass raised; caller decides whether to + trust the subsequent timings + + The function always restores ``AutoTuner`` singleton state on exit so that + ``--fast_autotune`` set for one case does not leak into the next. + """ + tuner = AutoTuner.get() + saved_warmup = tuner.warmup + saved_repeat = tuner.repeat + saved_stream_delay = tuner.stream_delay_micro_secs + if fast_autotune: + tuner.warmup = 0 + tuner.repeat = 1 + tuner.stream_delay_micro_secs = 10 + + cache_path = os.path.join(tempfile.gettempdir(), "bench_moe_autotuner_cache.json") + try: + with torch.inference_mode(), autotune(cache_path=cache_path): + moe.forward(x, router_logits, all_rank_num_tokens=all_rank_num_tokens) + torch.cuda.synchronize() + return "success:fast" if fast_autotune else "success" + finally: + tuner.warmup = saved_warmup + tuner.repeat = saved_repeat + tuner.stream_delay_micro_secs = saved_stream_delay diff --git a/tests/microbenchmarks/bench_moe/timing/cuda_graph.py b/tests/microbenchmarks/bench_moe/timing/cuda_graph.py new file mode 100644 index 000000000000..b468f2cdd10b --- /dev/null +++ b/tests/microbenchmarks/bench_moe/timing/cuda_graph.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CUDA Graph capture+replay timing path with external CUDA events.""" + +from __future__ import annotations + +import ctypes +from typing import Any, Dict, List, Optional, Tuple + +import torch + +from ..utils import _sync +from .cupti import _build_cuda_graph_kernel_stats_cupti +from .eager import _l2_flush_buffer + + +def _time_moe_forward_cuda_graph( + moe, + x: torch.Tensor, + router_logits: torch.Tensor, + all_rank_num_tokens: List[int], + *, + warmup: int, + iters: int, + cupti_ctx: Optional[Any] = None, + flush_l2: bool = True, +) -> Tuple[List[float], Dict[str, Any]]: + """Time ``moe.forward`` inside an unrolled CUDA graph with EXTERNAL events.""" + device = x.device if x.numel() > 0 else torch.device("cuda") + l2_buffer = _l2_flush_buffer(device) if flush_l2 else None + + if cupti_ctx is not None: + _cupti = cupti_ctx.module + _cupti_kernels = cupti_ctx.kernels + _cupti_events = cupti_ctx.events + _cupti_available = cupti_ctx.ok + else: + _cupti = None + _cupti_kernels = [] + _cupti_events = [] + _cupti_available = False + + # ---- 1. Pre-capture eager pass for shape discovery and lazy init/codegen. + with torch.inference_mode(): + eager_out = moe.forward(x, router_logits, all_rank_num_tokens=all_rank_num_tokens) + if not isinstance(eager_out, torch.Tensor): + raise RuntimeError( + "CUDA-Graph timing requires a tensor output from moe.forward; got " + f"{type(eager_out).__name__}. Use --no_cuda_graph for this case." + ) + torch.cuda.synchronize() + + # ---- 2. Pre-create events; ``cudaEventRecordWithFlags`` makes graph-internal + # events queryable via elapsed_time(). + _cudart = ctypes.CDLL("libcudart.so") + _cudart.cudaEventRecordWithFlags.restype = ctypes.c_int + _cudart.cudaEventRecordWithFlags.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint] + _CUDA_EVENT_RECORD_EXTERNAL = 0x1 + + def _record_external(event: torch.cuda.Event) -> None: + stream = torch.cuda.current_stream() + ret = _cudart.cudaEventRecordWithFlags( + event.cuda_event, stream.cuda_stream, _CUDA_EVENT_RECORD_EXTERNAL + ) + if ret != 0: + raise RuntimeError(f"cudaEventRecordWithFlags failed with code {ret}") + + starts = [torch.cuda.Event(enable_timing=True) for _ in range(iters)] + ends = [torch.cuda.Event(enable_timing=True) for _ in range(iters)] + for evt in starts + ends: + evt.record() + torch.cuda.synchronize() + + big_graph = torch.cuda.CUDAGraph() + with torch.inference_mode(), torch.cuda.graph(big_graph): + for i in range(iters): + if l2_buffer is not None: + l2_buffer.zero_() + _record_external(starts[i]) + moe.forward(x, router_logits, all_rank_num_tokens=all_rank_num_tokens) + _record_external(ends[i]) + + _sync() + for _ in range(warmup): + big_graph.replay() + _sync() + + if _cupti_available: + _cupti.activity_flush_all(0) + _cupti_kernels.clear() + _cupti_events.clear() + + big_graph.replay() + _sync() + + if _cupti_available: + _cupti.activity_flush_all(0) + + forward_times_ms = [starts[i].elapsed_time(ends[i]) for i in range(iters)] + + if _cupti_available: + _cupti_kernels.sort(key=lambda k: k[1]) + _cupti_events.sort() + cupti_stats = _build_cuda_graph_kernel_stats_cupti(_cupti_kernels, _cupti_events, iters) + if cupti_stats is not None: + cupti_times = cupti_stats.pop("moe_times_ms") + forward_times_ms = [ + ct if ct is not None else et for ct, et in zip(cupti_times, forward_times_ms) + ] + detailed_stats = cupti_stats + else: + detailed_stats = {"moe_forward_kernels": [], "other_kernels": []} + else: + detailed_stats = {"moe_forward_kernels": [], "other_kernels": []} + + return forward_times_ms, detailed_stats diff --git a/tests/microbenchmarks/bench_moe/timing/cupti.py b/tests/microbenchmarks/bench_moe/timing/cupti.py new file mode 100644 index 000000000000..0c533c14069b --- /dev/null +++ b/tests/microbenchmarks/bench_moe/timing/cupti.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Optional CUPTI activity tracking for CUDA-Graph kernel breakdown. + +Both the ``cupti.cupti`` and ``cxxfilt`` packages are optional: when they are +missing on the host we degrade gracefully and the CUDA-Graph timing path +falls back to pure external-event timing without kernel breakdown. +""" + +from __future__ import annotations + +import functools +import importlib +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +from ..utils import _maybe_print_rank0 +from .eager import _kernel_times_to_summary_list + + +def _try_import(module_path: str) -> Any: + """Return the imported module or ``None`` on any failure.""" + try: + return importlib.import_module(module_path) + except Exception: + return None + + +_cupti = _try_import("cupti.cupti") +_cxxfilt = _try_import("cxxfilt") + + +class _CuptiContext(NamedTuple): + """Initialised CUPTI handles + capture buffers. + + Returned by ``_try_init_cupti``. ``ok`` is False when CUPTI was missing + or activity registration failed; in that case the other fields are + empty/None and callers fall back to PyTorch-event-only timing. + """ + + module: Any + kernels: List[Tuple[str, int, int]] + events: List[int] + ok: bool + + +def _try_init_cupti() -> _CuptiContext: + """Initialize CUPTI activity tracking before any CUDA context is created. + + Returns a ``_CuptiContext``. When the ``cupti`` Python package is missing + or registration fails the function degrades gracefully to + ``_CuptiContext(None, [], [], False)`` and the caller falls back to + PyTorch-event-only timing without kernel breakdown. + + The two activity kinds we register are: + - ``CONCURRENT_KERNEL``: every kernel actually executed on the GPU, + including those replayed from a captured CUDA graph (Kineto cannot see + these because there is no Python frame during replay). + - ``CUDA_EVENT``: device-side timestamps for ``cudaEventRecord`` calls; + we use them to delimit which kernels fall inside each timed iteration. + """ + if _cupti is None: + return _CuptiContext(None, [], [], False) + try: + _cupti_kernels: List[Tuple[str, int, int]] = [] + _cupti_events: List[int] = [] + + def _buf_requested(): + return 8 * 1024 * 1024, 0 + + def _buf_completed(kernels, events, activities): + for act in activities: + if act.kind == _cupti.ActivityKind.CONCURRENT_KERNEL: + kernels.append((act.name, act.start, act.end)) + elif act.kind == _cupti.ActivityKind.CUDA_EVENT: + events.append(act.device_timestamp) + + _cupti.activity_enable(_cupti.ActivityKind.CONCURRENT_KERNEL) + _cupti.activity_enable(_cupti.ActivityKind.CUDA_EVENT) + _cupti.activity_enable_cuda_event_device_timestamps(1) + _cupti.activity_register_callbacks( + _buf_requested, + functools.partial(_buf_completed, _cupti_kernels, _cupti_events), + ) + return _CuptiContext(_cupti, _cupti_kernels, _cupti_events, True) + except Exception: + return _CuptiContext(None, [], [], False) + + +def _demangle_names(names: List[str]) -> Dict[str, str]: + """Demangle C++ symbol names via ``cxxfilt`` when available.""" + if _cxxfilt is None: + return {n: n for n in names} + try: + return {n: _cxxfilt.demangle(n) for n in names} + except Exception: + return {n: n for n in names} + + +def _build_cuda_graph_kernel_stats_cupti( + cupti_kernels: List[Tuple[str, int, int]], + cupti_events: List[int], + iters: int, +) -> Optional[Dict[str, Any]]: + """Categorize replay kernels into moe_forward/other windows via CUPTI.""" + expected_events = 2 * iters + if len(cupti_events) != expected_events: + _maybe_print_rank0( + f"[bench_moe] CUPTI breakdown skipped: expected {expected_events} CUDA_EVENT " + f"records ({iters} iters x 2) but got {len(cupti_events)}. " + "Most likely CUPTI was registered after CUDA context creation." + ) + return None + if not cupti_kernels: + return None + + starts_abs = [cupti_events[2 * i + 0] for i in range(iters)] + ends_abs = [cupti_events[2 * i + 1] for i in range(iters)] + + unique_names = list({name for name, _, _ in cupti_kernels}) + dm = _demangle_names(unique_names) + + moe_kernel_times: Dict[str, List[float]] = {} + other_kernel_times: Dict[str, List[float]] = {} + + iter_span: List[List[Optional[int]]] = [[None, None] for _ in range(iters)] + + for name, k_start, k_end in cupti_kernels: + demangled = dm.get(name, name) + # CUPTI timestamps are nanoseconds; convert to milliseconds. + device_time_ms = (k_end - k_start) / 1e6 + + iter_idx = -1 + for i in range(iters): + if k_start >= starts_abs[i] and k_end <= ends_abs[i]: + iter_idx = i + break + + if iter_idx >= 0: + span = iter_span[iter_idx] + span[0] = k_start if span[0] is None else min(span[0], k_start) + span[1] = k_end if span[1] is None else max(span[1], k_end) + moe_kernel_times.setdefault(demangled, []).append(device_time_ms) + else: + other_kernel_times.setdefault(demangled, []).append(device_time_ms) + + moe_times_ms = [ + (span[1] - span[0]) / 1e6 if span[0] is not None else None for span in iter_span + ] + + return { + "moe_forward_kernels": _kernel_times_to_summary_list(moe_kernel_times), + "other_kernels": _kernel_times_to_summary_list(other_kernel_times), + "moe_times_ms": moe_times_ms, + } diff --git a/tests/microbenchmarks/bench_moe/timing/eager.py b/tests/microbenchmarks/bench_moe/timing/eager.py new file mode 100644 index 000000000000..22543584f354 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/timing/eager.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eager-path timing and Kineto kernel breakdown.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +import torch +from torch.autograd import DeviceType + +from ..utils import _maybe_print_rank0, _sync + + +def _kernel_times_to_summary_list( + kernel_times: Dict[str, List[float]], +) -> List[Dict[str, Any]]: + """Convert ``{kernel_name: [times_ms]}`` into the dashboard summary shape. + + Sorted by per-kernel mean duration descending; entries keep the raw + ``_times`` list so downstream mpi_allgather can recompute per-rank stats. + Shared by the Kineto (``_parse_profiler_events_moe``) and CUPTI + (``_build_cuda_graph_kernel_stats_cupti``) paths so the wire format stays + in lockstep across the two backends. + """ + out = [{"name": n, "count": len(t), "_times": t} for n, t in kernel_times.items()] + out.sort( + key=lambda entry: (sum(entry["_times"]) / len(entry["_times"])) if entry["_times"] else 0.0, + reverse=True, + ) + return out + + +def _l2_flush_buffer(device: torch.device) -> torch.Tensor: + """Allocate a 2x-L2 flush buffer to clear L2 between iterations.""" + l2_size = torch.cuda.get_device_properties(device).L2_cache_size + l2_flush_size = (l2_size * 2) // 4 + return torch.empty(l2_flush_size, dtype=torch.int32, device=device) + + +def _time_moe_forward_eager( + moe, + x: torch.Tensor, + router_logits: torch.Tensor, + all_rank_num_tokens: List[int], + *, + warmup: int, + iters: int, + flush_l2: bool = True, + collect_kernels: bool = True, +) -> Tuple[List[float], Dict[str, Any]]: + """Time eager ``ConfigurableMoE.forward``. + + Latency is ALWAYS measured with pure ``torch.cuda.Event`` records so the + reported ``score_ms`` is comparable across ``cuda_graph`` and eager paths + (the CUDA-Graph path also uses external CUDA events for its per-iter + window). When ``collect_kernels=True`` a separate, shorter profiler pass is + run only to gather the kernel breakdown; profiler-derived numbers are not + used to score the candidate. + """ + device = x.device if x.numel() > 0 else torch.device("cuda") + l2_buffer = _l2_flush_buffer(device) if flush_l2 else None + + def _do_forward(): + with torch.inference_mode(): + _ = moe.forward(x, router_logits, all_rank_num_tokens=all_rank_num_tokens) + + starts = [torch.cuda.Event(enable_timing=True) for _ in range(iters)] + ends = [torch.cuda.Event(enable_timing=True) for _ in range(iters)] + _sync() + for _ in range(warmup): + if l2_buffer is not None: + l2_buffer.zero_() + _do_forward() + for i in range(iters): + if l2_buffer is not None: + l2_buffer.zero_() + starts[i].record() + _do_forward() + ends[i].record() + _sync() + forward_times_ms = [starts[i].elapsed_time(ends[i]) for i in range(iters)] + + detailed_stats: Dict[str, Any] = { + "moe_forward_kernels": [], + "other_kernels": [], + } + if not collect_kernels: + return forward_times_ms, detailed_stats + + # Separate profiler-only pass for kernel breakdown. Use a small fixed iter + # count (capped by ``iters``) so the profiler overhead does not dominate + # the case wall-clock budget. The latencies produced here are intentionally + # discarded; only the kernel categorisation is kept. + breakdown_iters = max(1, min(iters, 3)) + try: + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA, torch.profiler.ProfilerActivity.CPU], + record_shapes=False, + with_stack=False, + ) as prof: + _sync() + if l2_buffer is not None: + l2_buffer.zero_() + _do_forward() # one warmup under profiler + for _ in range(breakdown_iters): + if l2_buffer is not None: + l2_buffer.zero_() + with torch.profiler.record_function("moe_forward"): + _do_forward() + _sync() + _, detailed_stats = _parse_profiler_events_moe(list(prof.events())) + except Exception as exc: + # Breakdown is best-effort; do not fail the case if Kineto misbehaves. + _maybe_print_rank0(f"[bench_moe] kernel breakdown skipped: {type(exc).__name__}: {exc}") + return forward_times_ms, detailed_stats + + +def _parse_profiler_events_moe(events_list: list) -> Tuple[List[float], Dict[str, Any]]: + """Parse Kineto events with ``moe_forward`` ranges. + + Returns ``(moe_forward_times_ms, detailed_stats)`` where ``detailed_stats`` + contains ``moe_forward_kernels`` (within the range) and ``other_kernels``. + """ + + def _is_gpu_event(evt) -> bool: + return getattr(evt, "device_type", None) == DeviceType.CUDA + + gpu_moe_intervals: List[Tuple[int, int]] = [] + for evt in events_list: + if not _is_gpu_event(evt) or evt.name != "moe_forward": + continue + tr = getattr(evt, "time_range", None) + if tr is None or tr.end <= tr.start: + continue + gpu_moe_intervals.append((tr.start, tr.end)) + gpu_moe_intervals.sort() + + def _scope(evt) -> Optional[str]: + tr = getattr(evt, "time_range", None) + if tr is None: + return None + for s, e in gpu_moe_intervals: + if s <= tr.start and tr.end <= e: + return "moe_forward" + return None + + moe_kernel_times: Dict[str, List[float]] = {} + other_kernel_times: Dict[str, List[float]] = {} + for evt in events_list: + if not _is_gpu_event(evt): + continue + if evt.device_time <= 0 or evt.name == "moe_forward": + continue + scope = _scope(evt) + bucket = moe_kernel_times if scope == "moe_forward" else other_kernel_times + # PyTorch profiler reports device_time in microseconds; convert to ms. + bucket.setdefault(evt.name, []).append(evt.device_time / 1e3) + + forward_times_ms: List[float] = [] + for evt in events_list: + if _is_gpu_event(evt) and evt.name == "moe_forward": + forward_times_ms.append(evt.device_time / 1e3) + + detailed_stats = { + "moe_forward_kernels": _kernel_times_to_summary_list(moe_kernel_times), + "other_kernels": _kernel_times_to_summary_list(other_kernel_times), + } + return forward_times_ms, detailed_stats diff --git a/tests/microbenchmarks/bench_moe/utils.py b/tests/microbenchmarks/bench_moe/utils.py new file mode 100644 index 000000000000..f580f7571aca --- /dev/null +++ b/tests/microbenchmarks/bench_moe/utils.py @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small process-local helpers shared across the bench_moe pipeline.""" + +from __future__ import annotations + +import os +import socket +from typing import Dict, Iterable, List, Optional, Tuple + +import torch +import torch.distributed as dist + +from tensorrt_llm._utils import local_mpi_rank, mpi_barrier, mpi_rank + +from .backend import MoeBackendType + +_InputCacheKey = Tuple[int, int, int, str, str, str, Optional[int]] +_InputCache = Dict[_InputCacheKey, Tuple[torch.Tensor, torch.Tensor]] + + +def _maybe_print_rank0(msg: str) -> None: + if mpi_rank() == 0: + print(msg, flush=True) + + +def _sync() -> None: + torch.cuda.synchronize() + mpi_barrier() + + +def _set_device_from_local_rank() -> int: + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required for this benchmark") + local_rank = local_mpi_rank() + device_count = torch.cuda.device_count() + if local_rank >= device_count: + raise RuntimeError( + "Detected GPU oversubscription: " + f"local_mpi_rank={local_rank} >= cuda_device_count={device_count}." + ) + dev = local_rank % device_count + torch.cuda.set_device(dev) + return dev + + +def _get_free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _ensure_dist_for_megamoe(moe_backend: str, rank: int, world_size: int) -> None: + """Initialize the torch.distributed NCCL ProcessGroup for MegaMoE.""" + if moe_backend.upper() != MoeBackendType.MEGAMOE.value: + return + if not torch.cuda.is_available(): + raise RuntimeError("CUDA required for MegaMoE backend") + if dist.is_initialized(): + return + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", str(_get_free_tcp_port())) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + os.environ["LOCAL_RANK"] = str(local_mpi_rank()) + dist.init_process_group(backend="nccl", rank=rank, world_size=world_size) + + +def _compute_stats(values: List[float]) -> Dict[str, float]: + if not values: + return { + "mean": 0.0, + "median": 0.0, + "stdev": 0.0, + "min": 0.0, + "max": 0.0, + "p90": 0.0, + } + s = sorted(values) + n = len(s) + mean = sum(s) / n + variance = sum((x - mean) ** 2 for x in s) / n + median = s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0 + p90_idx = max(0, min(n - 1, int(round(0.9 * (n - 1))))) + return { + "mean": mean, + "median": median, + "stdev": variance**0.5, + "min": s[0], + "max": s[-1], + "p90": s[p90_idx], + } + + +def _distribute_tokens(total: int, world_size: int) -> List[int]: + """Distribute ``total`` global tokens evenly across ``world_size`` ranks.""" + if world_size <= 0 or total < 0: + raise ValueError(f"invalid args: total={total}, world_size={world_size}") + if world_size == 1: + return [total] + base = total // world_size + out = [base] * world_size + out[0] += total - base * world_size + return out + + +def _validate_per_rank_token_list( + per_rank: Iterable[int], + *, + world_size: int, + expected_total: int, +) -> List[int]: + """Validate and normalize an explicit per-rank token list.""" + out = [int(v) for v in per_rank] + if len(out) != world_size: + raise ValueError( + f"per_rank_num_tokens has length {len(out)}, expected world_size={world_size}" + ) + if any(v < 0 for v in out): + raise ValueError("per_rank_num_tokens entries must be >= 0") + if sum(out) != int(expected_total): + raise ValueError( + f"sum(per_rank_num_tokens)={sum(out)} must equal num_tokens={expected_total}" + ) + return out + + +def _make_inputs( + local_num_tokens: int, + hidden_size: int, + num_experts: int, + act_dtype: torch.dtype, + routing_logits_dtype: torch.dtype, + device: torch.device, + seed: Optional[int] = None, + cache: Optional[_InputCache] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Create synthetic hidden states + router logits, optionally from a local seed.""" + cache_key = ( + int(local_num_tokens), + int(hidden_size), + int(num_experts), + str(act_dtype), + str(routing_logits_dtype), + str(device), + int(seed) if seed is not None else None, + ) + if cache is not None and cache_key in cache: + return cache[cache_key] + + if local_num_tokens == 0: + x = torch.empty((0, hidden_size), dtype=act_dtype, device=device) + logits = torch.empty((0, num_experts), dtype=routing_logits_dtype, device=device) + if cache is not None: + cache[cache_key] = (x, logits) + return x, logits + generator = None + if seed is not None: + generator_device = device.type if isinstance(device, torch.device) else device + generator = torch.Generator(device=generator_device) + generator.manual_seed(int(seed)) + x = torch.randn( + (local_num_tokens, hidden_size), dtype=act_dtype, device=device, generator=generator + ) + logits = torch.randn( + (local_num_tokens, num_experts), + dtype=routing_logits_dtype, + device=device, + generator=generator, + ) + if cache is not None: + cache[cache_key] = (x, logits) + return x, logits diff --git a/tests/microbenchmarks/bench_moe/worker.py b/tests/microbenchmarks/bench_moe/worker.py new file mode 100644 index 000000000000..1019b6bfc9a5 --- /dev/null +++ b/tests/microbenchmarks/bench_moe/worker.py @@ -0,0 +1,687 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MPI worker loop, checkpointing, and launcher entrypoints for bench_moe.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import pickle +import signal +import sys +import threading +import time +import traceback +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import torch +from mpi4py import MPI + +import tensorrt_llm as tllm +from tensorrt_llm._utils import local_mpi_rank, mpi_allgather, mpi_rank, mpi_world_size + +from . import reporting as _reporting +from .case_runner import _run_one_candidate +from .cli import ( + _BenchmarkContext, + _build_worker_header, + _maybe_load_config_file, + _resolve_benchmark_context, + parse_args, +) +from .results import _make_skipped_run_result, _make_upstream_skipped_row, _runresult_to_row +from .search import expand_and_prune +from .specs import ConfigSpec, WorkloadSpec +from .timing import _CuptiContext, _try_init_cupti +from .utils import _InputCache, _maybe_print_rank0, _set_device_from_local_rank + +_MICROBENCH_DIR = Path(__file__).resolve().parent.parent +_REPO_ROOT = _MICROBENCH_DIR.parent.parent + + +def _try_import(module_path: str, attr: Optional[str] = None, default: Any = None) -> Any: + """Import module_path; if attr is given, return getattr(m, attr).""" + try: + m = importlib.import_module(module_path) + except Exception: + return default + return m if attr is None else getattr(m, attr, default) + + +_cloudpickle = _try_import("cloudpickle") +_MPIPoolExecutor = _try_import("mpi4py.futures", "MPIPoolExecutor") + +POISON_HERE_PREFIX = "cuda_context_poisoned_after_success" +POISON_UPSTREAM_PREFIX = "cuda_context_poisoned_upstream" +WATCHDOG_UPSTREAM_PREFIX = "watchdog_timeout_upstream" +BENCH_MOE_POISON_EXIT_CODE = 75 + + +def is_completed_for_resume(row: dict[str, Any]) -> bool: + """Decide whether a previously-recorded row blocks a fresh attempt on resume.""" + status = (row.get("status") or "").lower() + reason = (row.get("skip_reason") or "").lower() + if status in {"success", "failed"}: + return True + if status == "skipped": + if reason.endswith("_upstream") or reason.startswith(POISON_UPSTREAM_PREFIX): + return False + return True + return False + + +def candidate_resume_key(*, workload: Any, config: Any) -> str: + """Stable string key identifying one ``(workload, config)`` candidate.""" + + def _get(obj: Any, key: str, default: Any = None) -> Any: + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + num_tokens = _get(workload, "num_tokens", 0) or 0 + backend = _get(config, "backend") or "" + parallel_mode = _get(config, "parallel_mode") or "" + comm_method = _get(config, "comm_method") or "AUTO" + cuda_graph = _get(config, "cuda_graph", True) + lpmc = _get(config, "use_low_precision_moe_combine", False) + moe_ep_size = _get(config, "moe_ep_size") + moe_tp_size = _get(config, "moe_tp_size") + enable_attention_dp = _get(config, "enable_attention_dp") + return "|".join( + [ + f"nt={int(num_tokens)}", + f"backend={str(backend).upper()}", + f"pmode={str(parallel_mode).upper()}", + f"comm={str(comm_method).upper()}", + f"cg={int(bool(cuda_graph))}", + f"lpmc={int(bool(lpmc))}", + f"ep={moe_ep_size if moe_ep_size is not None else '-'}", + f"tp={moe_tp_size if moe_tp_size is not None else '-'}", + f"adp={'-' if enable_attention_dp is None else int(bool(enable_attention_dp))}", + ] + ) + + +def load_resume_payload( + path: Optional[str], +) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + """Load an existing JSON for resume. + + Returns ``(completed_by_key, rows_to_carry_forward)``. Only terminal rows + are indexed and carried; upstream-skipped placeholders are dropped so they + get re-attempted. + """ + if not path or not os.path.exists(path): + return {}, [] + try: + with open(path) as f: + payload = json.load(f) + except Exception as exc: + if mpi_rank() == 0: + print( + f"[bench_moe] --resume_from={path}: failed to read existing JSON " + f"({type(exc).__name__}: {exc}); starting from scratch.", + flush=True, + ) + return {}, [] + rows = payload.get("results") or [] + completed: Dict[str, Dict[str, Any]] = {} + keep: List[Dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + if is_completed_for_resume(row): + key = candidate_resume_key( + workload=row.get("workload") or {}, + config=row.get("requested_config") or {}, + ) + completed[key] = row + keep.append(row) + return completed, keep + + +def cuda_poison_self_check() -> Optional[str]: + """Return a non-empty reason if the current CUDA context has a sticky error.""" + if not torch.cuda.is_available(): + return None + try: + torch.cuda.synchronize() + except RuntimeError as exc: + return f"{type(exc).__name__}: {exc}" + except Exception as exc: # pragma: no cover - belt and braces + return f"{type(exc).__name__}: {exc}" + return None + + +def allreduce_poison_reason(local_reason: Optional[str]) -> Optional[str]: + """Gather poison reasons across MPI ranks and return a shared summary.""" + try: + gathered: List[Optional[str]] = mpi_allgather(local_reason) + except Exception as exc: + return local_reason or f"mpi_allgather failed: {type(exc).__name__}: {exc}" + bad: List[Tuple[int, str]] = [(idx, reason) for idx, reason in enumerate(gathered) if reason] + if not bad: + return None + return "; ".join(f"rank{rank}={reason}" for rank, reason in bad) + + +class CandidateWatchdog: + """Hard wall-clock guard around one candidate; SIGKILLs the process on timeout.""" + + def __init__(self, budget_s: float, label: str): + self._budget_s = float(budget_s) + self._label = label + self._cancelled = threading.Event() + self._thread: Optional[threading.Thread] = None + + def __enter__(self) -> "CandidateWatchdog": + if self._budget_s <= 0: + return self + self._cancelled.clear() + self._thread = threading.Thread( + target=self._guard, + name="bench_moe-watchdog", + daemon=True, + ) + self._thread.start() + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + self._cancelled.set() + # Do not join: joining here could deadlock if the watchdog already + # fired while CUDA/NCCL was still draining. + return False + + def _guard(self) -> None: + if self._cancelled.wait(self._budget_s): + return + try: + sys.stderr.write( + f"[bench_moe watchdog] candidate '{self._label}' exceeded " + f"{self._budget_s:.1f}s budget on pid={os.getpid()} " + f"rank={mpi_rank()}; sending SIGKILL to break suspected " + f"NCCL deadlock or CUDA hang.\n" + ) + sys.stderr.flush() + except Exception: + pass + os.kill(os.getpid(), signal.SIGKILL) + + +def _emit_checkpoint_report( + *, + args: argparse.Namespace, + ctx: "_BenchmarkContext", + rows: List[Dict[str, Any]], + world_size: int, +) -> None: + """Persist a JSON snapshot of all completed rows after a candidate finishes. + + Only rank 0 writes; other ranks no-op. Uses tmp + ``os.replace`` so a + crash mid-dump cannot leave a half-written JSON. Called from the main + candidate loop so a sticky-error / watchdog crash cannot lose prior work. + """ + if mpi_rank() != 0: + return + out_path = getattr(args, "output_file", None) + if not out_path: + return + payload = _reporting.build_report_payload( + ctx=ctx, + rows=rows, + world_size=world_size, + cuda_graph_default=bool(args.cuda_graph), + repo_root=_REPO_ROOT, + ) + out_dir = os.path.dirname(out_path) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + tmp = out_path + ".checkpoint.tmp" + with open(tmp, "w") as f: + json.dump(payload, f, indent=2) + os.replace(tmp, out_path) + + +def _load_and_broadcast_resume_state( + rank: int, resume_path: Optional[str] +) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + """Load ``--resume_from`` payload on rank 0 and broadcast to all ranks. + + Returns ``(resumed_by_key, resumed_rows)``. Returns empty containers when + ``resume_path`` is unset or the bcast fails (with a logged warning so the + operator can decide whether divergence matters). + """ + resumed_by_key: Dict[str, Dict[str, Any]] = {} + resumed_rows: List[Dict[str, Any]] = [] + if not resume_path: + return resumed_by_key, resumed_rows + if rank == 0: + resumed_by_key, resumed_rows = load_resume_payload(resume_path) + if resumed_rows: + print( + f"[bench_moe] --resume_from={resume_path}: loaded " + f"{len(resumed_rows)} terminal row(s); they will be skipped.", + flush=True, + ) + try: + resumed_by_key = MPI.COMM_WORLD.bcast(resumed_by_key, root=0) + resumed_rows = MPI.COMM_WORLD.bcast(resumed_rows, root=0) + except Exception as exc: + _maybe_print_rank0( + f"[bench_moe] resume bcast failed ({type(exc).__name__}: {exc}); " + "ranks may diverge on which candidates to skip." + ) + return resumed_by_key, resumed_rows + + +def _build_flat_plan_for_sweep( + ctx: Any, + world_size: int, + max_configs: Optional[int], +) -> List[Tuple[WorkloadSpec, ConfigSpec, str]]: + """Flatten the per-workload search expansion into a single linear plan. + + Items are ``(workload, config, kind)`` where ``kind`` is ``"run"`` for + candidates to execute or ``"prune:"`` for compile-time rejects + (e.g. ``backend.can_implement`` returned False). The flattened form lets + the poison handler emit upstream-skipped placeholders for every + not-yet-attempted candidate. + """ + flat_plan: List[Tuple[WorkloadSpec, ConfigSpec, str]] = [] + for workload in ctx.workloads: + candidates, skipped = expand_and_prune( + base_config=ctx.base_config, + search=ctx.search, + model=ctx.model, + world_size=world_size, + act_dtype=ctx.act_dtype, + max_configs=max_configs, + ) + for cand, reason in skipped.items(): + flat_plan.append((workload, cand, f"prune:{reason}")) + for cand in candidates: + flat_plan.append((workload, cand, "run")) + return flat_plan + + +def _handle_cuda_poison_and_exit( + *, + any_poison: str, + args: argparse.Namespace, + ctx: Any, + rank: int, + world_size: int, + accumulated_rows: List[Dict[str, Any]], + flat_plan: List[Tuple[WorkloadSpec, ConfigSpec, str]], + resumed_by_key: Dict[str, Dict[str, Any]], + next_idx: int, +) -> None: + """Handle a poisoned CUDA context by checkpointing and exiting cleanly. + + Promotes the offending row to ``failed``, fills upstream placeholders for + every not-yet-attempted candidate, writes a checkpoint, then exits with + ``BENCH_MOE_POISON_EXIT_CODE``. + + Once a CUDA context is poisoned (sticky error from a prior candidate) + every later kernel launch on the same context will fail. The outer + driver wraps this worker and is responsible for restarting with + ``--resume_from`` after we exit. + + NOTE: This function calls ``os._exit`` and never returns. Atexit hooks + are skipped on purpose so NCCL / CUDA do not re-enter on a poisoned + context and deadlock. + """ + # Promote the just-finished row to "failed" so a future --resume_from + # never picks it again. Preserve original instrumentation; record + # the poison reason for the dashboard. + if accumulated_rows: + last = accumulated_rows[-1] + instr = last.setdefault("instrumentation", {}) + instr["post_run_cuda_poison_reason"] = any_poison + if (last.get("status") or "").lower() == "success": + last["status"] = "failed" + last["skip_reason"] = f"{POISON_HERE_PREFIX}: {any_poison}" + + # Emit upstream-skipped placeholders for every not-yet-attempted + # candidate so dashboards see the full search space and a fresh + # --resume_from run knows to re-attempt them. + placeholder_reason = ( + f"{POISON_UPSTREAM_PREFIX}: prior candidate poisoned the CUDA context ({any_poison})" + ) + for w2, c2, k2 in flat_plan[next_idx:]: + key2 = candidate_resume_key(workload=w2, config=c2) + if key2 in resumed_by_key: + continue + if k2.startswith("prune:"): + # Pruned candidates do not touch CUDA; record their true + # reason rather than the upstream placeholder. + pruned = _make_skipped_run_result( + model=ctx.model, + workload=w2, + config=c2, + world_size=world_size, + analysis=ctx.analysis, + reason=k2[len("prune:") :], + ) + accumulated_rows.append(_runresult_to_row(pruned)) + continue + accumulated_rows.append( + _make_upstream_skipped_row( + model=ctx.model, + workload=w2, + config=c2, + world_size=world_size, + analysis=ctx.analysis, + reason=placeholder_reason, + ) + ) + + _emit_checkpoint_report(args=args, ctx=ctx, rows=accumulated_rows, world_size=world_size) + if rank == 0: + sys.stderr.write( + f"[bench_moe] CUDA context poisoned ({any_poison}); " + f"checkpointed {len(accumulated_rows)} row(s) to " + f"{args.output_file!r}; exiting with code " + f"{BENCH_MOE_POISON_EXIT_CODE} so the outer driver can " + "restart with --resume_from.\n" + ) + sys.stderr.flush() + # ``os._exit`` (not ``sys.exit``) so Python atexit hooks do not + # re-enter NCCL / CUDA on a poisoned context and deadlock. + os._exit(BENCH_MOE_POISON_EXIT_CODE) + + +def _write_final_report( + *, + args: argparse.Namespace, + ctx: Any, + rows: List[Dict[str, Any]], + world_size: int, +) -> None: + """Produce the final report payload and write it to disk + workbook. + + Two paths: + - ``args.output_file`` set: atomic write (tmp + rename), plus an analysis + workbook next to it. + - No ``output_file``: echo a rankings-only summary on stdout for headless + invocations, optionally writing a workbook if explicitly requested. + + Caller must ensure this only runs on rank 0 -- all other ranks should + skip the final report step entirely. + """ + _reporting.write_final_report( + args=args, + ctx=ctx, + rows=rows, + world_size=world_size, + repo_root=_REPO_ROOT, + ) + + +def _run_benchmark_worker_under_current_mpi(args: argparse.Namespace, launcher: str) -> None: + args = _maybe_load_config_file(args) + ctx = _resolve_benchmark_context(args) + + # CUPTI MUST be initialized before the CUDA context is created. + _early_cupti_ctx: Optional[_CuptiContext] = None + if args.cuda_graph and "kernels" in ctx.analysis: + _cupti_init = _try_init_cupti() + if _cupti_init.ok: + _early_cupti_ctx = _cupti_init + + tllm.logger.set_level("error") + + world_size = mpi_world_size() + rank = mpi_rank() + _set_device_from_local_rank() + device = torch.device("cuda") + seed = int(args.random_seed) + rank + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + # Early header (rank 0) for stdout consumers. + if rank == 0: + print(json.dumps(_build_worker_header(ctx, launcher, world_size), indent=2), flush=True) + + resumed_by_key, resumed_rows = _load_and_broadcast_resume_state( + rank=rank, resume_path=getattr(args, "resume_from", None) + ) + flat_plan = _build_flat_plan_for_sweep(ctx, world_size, args.max_configs) + + # Accumulated rows include resumed rows (preserved as-is) plus rows + # produced this run. ``_build_report_payload`` consumes this directly. + accumulated_rows: List[Dict[str, Any]] = list(resumed_rows) + input_cache: _InputCache = {} + + checkpoint_every = max(0, int(getattr(args, "checkpoint_every", 1) or 0)) + candidates_since_checkpoint = 0 + watchdog_budget_s = float(getattr(args, "per_candidate_timeout_s", 0.0) or 0.0) + + deadline: Optional[float] = None + if args.time_budget_minutes is not None and args.time_budget_minutes > 0: + deadline = time.monotonic() + args.time_budget_minutes * 60.0 + + for idx, (workload, cand, kind) in enumerate(flat_plan): + key = candidate_resume_key(workload=workload, config=cand) + if key in resumed_by_key: + # E: short-circuit. The resumed row is already in accumulated_rows. + continue + + if kind.startswith("prune:"): + reason = kind[len("prune:") :] + r = _make_skipped_run_result( + model=ctx.model, + workload=workload, + config=cand, + world_size=world_size, + analysis=ctx.analysis, + reason=reason, + ) + accumulated_rows.append(_runresult_to_row(r)) + continue + + if deadline is not None and time.monotonic() > deadline: + _maybe_print_rank0( + "[bench_moe] --time_budget_minutes exceeded; remaining candidates " + "will be reported as skipped." + ) + r = _make_skipped_run_result( + model=ctx.model, + workload=workload, + config=cand, + world_size=world_size, + analysis=ctx.analysis, + reason="time_budget_exceeded", + ) + accumulated_rows.append(_runresult_to_row(r)) + continue + + case_label = ( + f"backend={cand.backend} parallel_mode={cand.parallel_mode} " + f"comm={cand.comm_method} num_tokens={workload.num_tokens}" + ) + _maybe_print_rank0(f"[bench_moe] running {case_label}") + + # Hard wall-clock guard around the actual candidate execution. + with CandidateWatchdog(watchdog_budget_s, case_label): + with torch.device(device): + r = _run_one_candidate( + model=ctx.model, + workload=workload, + config=cand, + world_size=world_size, + rank=rank, + device=device, + act_dtype=ctx.act_dtype, + routing_logits_dtype=ctx.routing_logits_dtype, + warmup=int(args.warmup), + iters=int(args.iters), + fast_autotune=bool(args.fast_autotune), + analysis=ctx.analysis, + cupti_ctx=_early_cupti_ctx, + random_seed=int(args.random_seed), + input_cache=input_cache, + enable_perfect_router_requested=bool(args.enable_perfect_router), + ) + row = _runresult_to_row(r) + accumulated_rows.append(row) + if rank == 0: + print(json.dumps(row, indent=2), flush=True) + + # Sticky CUDA error detection + lockstep exit across all ranks. + local_poison = cuda_poison_self_check() + any_poison = allreduce_poison_reason(local_poison) + if any_poison is not None: + _handle_cuda_poison_and_exit( + any_poison=any_poison, + args=args, + ctx=ctx, + rank=rank, + world_size=world_size, + accumulated_rows=accumulated_rows, + flat_plan=flat_plan, + resumed_by_key=resumed_by_key, + next_idx=idx + 1, + ) + # _handle_cuda_poison_and_exit calls os._exit; control never returns. + + # Incremental checkpoint so a future watchdog SIGKILL never loses + # already-completed rows. + candidates_since_checkpoint += 1 + if checkpoint_every > 0 and candidates_since_checkpoint >= checkpoint_every: + _emit_checkpoint_report( + args=args, ctx=ctx, rows=accumulated_rows, world_size=world_size + ) + candidates_since_checkpoint = 0 + + if rank == 0: + _write_final_report(args=args, ctx=ctx, rows=accumulated_rows, world_size=world_size) + + +# --------------------------------------------------------------------------- +# MPI launchers +# --------------------------------------------------------------------------- + + +_WORKER_ENV = { + "TRTLLM_CAN_USE_DEEP_EP": "1", + "TRTLLM_ENABLE_PDL": "0", +} + + +def _spawn_worker_main(args_blob: bytes) -> List[Dict[str, Any]]: + args = pickle.loads(args_blob) + try: + _run_benchmark_worker_under_current_mpi(args, launcher="spawn") + except Exception as exc: + rank = mpi_rank() + size = mpi_world_size() + msg = ( + "[bench_moe worker] uncaught exception:\n" + f"rank={rank}/{size} local_rank={local_mpi_rank()} pid={os.getpid()}\n" + f"{traceback.format_exc()}" + ) + try: + sys.stderr.write(msg + "\n") + sys.stderr.flush() + except Exception: + pass + raise RuntimeError(msg) from exc + return [] + + +def main() -> None: + args = parse_args() + + external_world_size = mpi_world_size() + if external_world_size > 1: + if args.world_size is not None and int(args.world_size) != external_world_size: + raise ValueError( + f"--world_size ({args.world_size}) must match external MPI world size " + f"({external_world_size}) under external mpirun/srun." + ) + os.environ.update(_WORKER_ENV) + _run_benchmark_worker_under_current_mpi(args, launcher="external_mpi") + return + + world_size = int(args.world_size or 1) + if world_size <= 0: + raise ValueError("--world_size must be > 0") + + if world_size == 1: + if mpi_rank() == 0: + print( + json.dumps( + { + "bench": "bench_moe", + "launcher": "inline_single_rank", + "world_size": 1, + "model": args.model, + "backend": args.backend, + "search": args.search, + }, + indent=2, + ), + flush=True, + ) + os.environ.update(_WORKER_ENV) + args.world_size = 1 + _run_benchmark_worker_under_current_mpi(args, launcher="inline_single_rank") + return + + if _cloudpickle is None or _MPIPoolExecutor is None: + missing = [ + name + for name, mod in (("cloudpickle", _cloudpickle), ("mpi4py.futures", _MPIPoolExecutor)) + if mod is None + ] + raise RuntimeError( + f"--world_size > 1 self-spawn launcher requires {', '.join(missing)}; " + "either install the missing package(s) or run the benchmark under mpirun/srun." + ) + + _cloudpickle.register_pickle_by_value(sys.modules[__name__]) + MPI.pickle.__init__( # type: ignore[attr-defined] + _cloudpickle.dumps, + _cloudpickle.loads, + pickle.HIGHEST_PROTOCOL, + ) + + if mpi_rank() == 0: + print( + json.dumps( + { + "bench": "bench_moe", + "launcher": "spawn", + "world_size": world_size, + "model": args.model, + "backend": args.backend, + "search": args.search, + }, + indent=2, + ), + flush=True, + ) + + args_blob = _cloudpickle.dumps(args) + executor = _MPIPoolExecutor(max_workers=world_size, env=_WORKER_ENV) + try: + _ = list(executor.map(_spawn_worker_main, [args_blob] * world_size)) + finally: + executor.shutdown(wait=False, cancel_futures=True) From 5cb6d2d4ad9b992e078fcb0d8d0ec19585ea6da0 Mon Sep 17 00:00:00 2001 From: Guoming Zhang <137257613+nv-guomingz@users.noreply.github.com> Date: Mon, 25 May 2026 15:36:20 +0800 Subject: [PATCH 010/308] [None][fix] Unwaive Qwen3.5 bf16 mtp on case (#14511) Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 27b22bb1b4f1..e5f56ead338c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -269,7 +269,6 @@ full:B200/perf/test_perf.py::test_perf[t5_3b] SKIP (bert_attention_plugin does n full:B200/perf/test_perf.py::test_perf[t5_base] SKIP (bert_attention_plugin does not support SM >= 100) full:B200/perf/test_perf.py::test_perf[t5_large] SKIP (bert_attention_plugin does not support SM >= 100) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) -full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp[mtp_on] SKIP (https://nvbugs/6205312) full:DGX_H100/kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked] SKIP (https://nvbugs/6136737) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) full:GB200/perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6194788) From 140d24bcf193b23655b90e1d211b8f4fc1c615ed Mon Sep 17 00:00:00 2001 From: Bo Deng Date: Mon, 25 May 2026 15:55:01 +0800 Subject: [PATCH 011/308] [None][fix] fix typo (#14510) Signed-off-by: Bo Deng --- tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py | 2 +- tests/integration/defs/accuracy/test_disaggregated_serving.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py index 8fbf40599d07..5aeeeadfe861 100644 --- a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @@ -1547,7 +1547,7 @@ def mamba_layer_cache( def free_resources(self, request: LlmRequest, pin_on_release: bool = False): if request in self.requests: self.requests.remove(request) - # self._request_id_to_state_index.pop(request.py_request_id, None) + self._request_id_to_state_index.pop(request.py_request_id, None) super().free_resources(request, pin_on_release) def _setup_state_indices(self) -> None: diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index bf9592920626..c786c80f3d19 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1978,7 +1978,7 @@ def test_nvfp4(self): @pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) -# @skip_pre_blackwell +@skip_pre_blackwell @pytest.mark.skip_less_device_memory(80000) class TestNemotron3Super120B(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Nemotron-Super-V3" From 01c9a34d53f568a07d3ebdaa5a7f01061c0d2da1 Mon Sep 17 00:00:00 2001 From: Guoming Zhang <137257613+nv-guomingz@users.noreply.github.com> Date: Mon, 25 May 2026 16:41:58 +0800 Subject: [PATCH 012/308] [https://nvbugs/6215684][fix] Fix invalid links in deployment guide (#14522) Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- .../deployment-guide-for-deepseek-r1-on-trtllm.md | 2 +- .../deployment-guide/deployment-guide-for-glm-5-on-trtllm.md | 2 +- .../deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md | 2 +- .../deployment-guide-for-llama3.3-70b-on-trtllm.md | 2 +- .../deployment-guide-for-llama4-scout-on-trtllm.md | 2 +- .../deployment-guide-for-nemotron-3-super-on-trtllm.md | 2 +- tests/integration/test_lists/waives.txt | 1 - 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 6ee9c8300d93..1a948213246e 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -58,7 +58,7 @@ Note: * The command also maps port `8000` from the container to your host so you can access the LLM API endpoint from your host * See the for all the available containers. The containers published in the main branch weekly have `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. -If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html) +If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) ### Recommended Performance Settings diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5-on-trtllm.md index cf83aee1ad1f..30d9b9eb13ee 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5-on-trtllm.md @@ -63,7 +63,7 @@ Note: * The command maps port `8000` from the container to your host so you can access the LLM API endpoint from your host. * See for all available containers. Containers published in the main branch weekly have an `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. -If you want to use the latest main branch, you can build from source: [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html) +If you want to use the latest main branch, you can build from source: [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) > **All commands below should be run inside the Docker container.** diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index fb784f846bd1..933dcd333474 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -54,7 +54,7 @@ Note: * The command also maps port `8000` from the container to your host so you can access the LLM API endpoint from your host * See the for all the available containers. The containers published in the main branch weekly have `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. -If you want to use latest main branch, you can choose to build from source to install TensorRT-LLM, the steps refer to . +If you want to use latest main branch, you can choose to build from source to install TensorRT-LLM, the steps refer to . ### Recommended Performance Settings diff --git a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md index 6f334efdd251..ce4cd54455de 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md @@ -50,7 +50,7 @@ Note: * The command also maps port **8000** from the container to your host so you can access the LLM API endpoint from your host * See the [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags) for all the available containers. The containers published in the main branch weekly have “rcN” suffix, while the monthly release with QA tests has no “rcN” suffix. Use the rc release to get the latest model and feature support. -If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html) +If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) ### Recommended Performance Settings diff --git a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md index 9fe91bd45473..157bab3bab86 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md @@ -49,7 +49,7 @@ Note: * The command also maps port `8000` from the container to your host so you can access the LLM API endpoint from your host * See the for all the available containers. The containers published in the main branch weekly have `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. -If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html) +If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) ### Recommended Performance Settings diff --git a/docs/source/deployment-guide/deployment-guide-for-nemotron-3-super-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-nemotron-3-super-on-trtllm.md index b84449a6de27..ac0ddcf279d5 100644 --- a/docs/source/deployment-guide/deployment-guide-for-nemotron-3-super-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-nemotron-3-super-on-trtllm.md @@ -53,7 +53,7 @@ Note: * The command also maps port `8000` from the container to your host so you can access the LLM API endpoint from your host. * See the for all the available containers. The containers published in the main branch weekly have `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. -If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html) +If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) ### Recommended Performance Settings diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e5f56ead338c..802cc68a7637 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -347,7 +347,6 @@ stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_360 stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) -test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6114608) From 998f41855dc17bd6e8ac5cb612d92e6ad173d8ce Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Mon, 25 May 2026 16:45:02 +0800 Subject: [PATCH 013/308] [https://nvbugs/6120981][fix] Switch to cu_seqlens_to_chunk_indices_offsets_triton with total_seqlens/extra_ch (#13566) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../custom_ops/mamba/mamba_backend_common.py | 26 ++++++++++++++++--- .../_torch/modules/mamba/mamba2_metadata.py | 25 +++++++++++------- tests/integration/test_lists/waives.txt | 1 - 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py index a91095fe7aa1..45feb2d4ea4b 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py @@ -19,7 +19,10 @@ from torch._ops import OpOverloadPacket from torch.fx import Node -from tensorrt_llm._torch.modules.mamba.mamba2_metadata import cu_seqlens_to_chunk_indices_offsets +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import ( + compute_extra_chunks_cpu, + cu_seqlens_to_chunk_indices_offsets_triton, +) from tensorrt_llm._torch.modules.mamba.ssd_combined import mamba_chunk_scan_combined from ..._compat import KvCacheConfig @@ -41,6 +44,7 @@ def _mamba_ssm_prepare_metadata( position_ids: torch.Tensor, batch_info_host: torch.Tensor, seq_len: torch.Tensor, + seq_len_host: torch.Tensor, cu_seqlen: torch.Tensor, # EXTRA METADATA PROVIDED BY THE DESCRIPTOR chunk_size: int, @@ -48,6 +52,10 @@ def _mamba_ssm_prepare_metadata( """Prepare metadata for cached SSM transform. Returns a tuple of (chunk_indices, chunk_offsets, seq_idx_prefill). + + Uses seq_len_host (CPU tensor) and batch_info_host to derive total_seqlens + and extra_chunks without GPU->CPU synchronization, preventing deadlocks + when NCCL collectives from MoE expert-parallel layers are pending. """ device = cu_seqlen.device batch_info = BatchInfo(batch_info_host) @@ -55,11 +63,20 @@ def _mamba_ssm_prepare_metadata( num_prefill, _, _ = batch_info.get_num_sequences() if num_prefill > 0: - chunk_indices, chunk_offsets = cu_seqlens_to_chunk_indices_offsets( - cu_seqlen[: num_prefill + 1], chunk_size + num_prefill_tokens, _, _ = batch_info.get_num_tokens() + + _extra = compute_extra_chunks_cpu(seq_len_host, num_prefill, chunk_size) + + chunk_indices, chunk_offsets = cu_seqlens_to_chunk_indices_offsets_triton( + cu_seqlen[: num_prefill + 1], + chunk_size, + total_seqlens=num_prefill_tokens, + extra_chunks=_extra, ) seq_idx_prefill = torch.repeat_interleave( - torch.arange(num_prefill, device=device, dtype=torch.int32), seq_len[:num_prefill] + torch.arange(num_prefill, device=device, dtype=torch.int32), + seq_len[:num_prefill], + output_size=num_prefill_tokens, ).view(1, -1) else: chunk_indices = torch.empty(0, dtype=torch.int32, device=device) @@ -75,6 +92,7 @@ def _mamba_ssm_prepare_metadata_fake( position_ids: torch.Tensor, batch_info_host: torch.Tensor, seq_len: torch.Tensor, + seq_len_host: torch.Tensor, cu_seqlen: torch.Tensor, # EXTRA METADATA PROVIDED BY THE DESCRIPTOR chunk_size: int, diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py index 3d1316e40519..4ddd67707bb2 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py @@ -63,6 +63,20 @@ def _cu_seqlens_triton_kernel( tl.store(chunk_offsets_ptr + offsets, chunk_offsets.to(tl.int32), mask=mask) +def compute_extra_chunks_cpu(seq_lens, num_seqs: int, chunk_size: int) -> int: + """Count extra chunks caused by misaligned sequence boundaries. + + Computes from CPU seq_lens to avoid GPU->CPU synchronization. + """ + cumsum = 0 + extra = 0 + for i in range(num_seqs - 1): + cumsum += int(seq_lens[i]) + if cumsum % chunk_size != 0: + extra += 1 + return extra + + def cu_seqlens_to_chunk_indices_offsets_triton( cu_seqlens: torch.Tensor, chunk_size: int, @@ -327,15 +341,8 @@ def prepare(self, attn_metadata: AttentionMetadata): self.has_initial_states_cpu[:num_contexts].any()) if self.use_initial_states: - # Compute extra_chunks using pure Python arithmetic on CPU - # seq_lens to avoid any GPU->CPU sync point. - _cs = self.chunk_size - _cumsum = 0 - _extra = 0 - for i in range(num_contexts - 1): - _cumsum += int(attn_metadata.seq_lens[i]) - if _cumsum % _cs != 0: - _extra += 1 + _extra = compute_extra_chunks_cpu(attn_metadata.seq_lens, + num_contexts, self.chunk_size) self.chunk_indices, self.chunk_offsets = cu_seqlens_to_chunk_indices_offsets_triton( self.cu_seqlens[:num_contexts + 1], diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 802cc68a7637..e465b1bc49e3 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -15,7 +15,6 @@ accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbu accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) -accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-4-trtllm] SKIP (https://nvbugs/6120981) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) From 2e3a75c223bf302e54b303018b623cac74e7661e Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 25 May 2026 16:57:30 +0800 Subject: [PATCH 014/308] [None][infra] Waive 5 failed cases for main in post-merge 2733 (#14516) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e465b1bc49e3..cfb251cbc87c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -22,6 +22,7 @@ accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SK accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6018046) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4] SKIP (https://nvbugs/6215793) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus SKIP (https://nvbugs/6193778) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp1] SKIP (https://nvbugs/6185196) @@ -323,18 +324,22 @@ perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20-gpus:2] SK perf/test_perf.py::test_perf[t5-bench-float16-maxbs:1-input_output_len:128,20-gpus:2] SKIP perf/test_perf.py::test_perf[t5_base-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20] SKIP +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215810) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k] SKIP (https://nvbugs/6167060) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k] SKIP (https://nvbugs/6190071) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6189928) perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6192201) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6179661) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6016528) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6200257) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) From 92c5030ff4563effdde57853803b7977326a453d Mon Sep 17 00:00:00 2001 From: Grzegorz Kwasniewski <213329731+greg-kwasniewski1@users.noreply.github.com> Date: Mon, 25 May 2026 12:13:45 +0200 Subject: [PATCH 015/308] [TRTLLM-13429][feat] Switch DeepSeek/NemotronH/Qwen3/Qwen3.5-MoE to sharding-IR canonical models (#13478) Signed-off-by: greg-kwasniewski1 <213329731+greg-kwasniewski1@users.noreply.github.com> Co-authored-by: Suyog Gupta <41447211+suyoggupta@users.noreply.github.com> --- .claude/skills/ad-model-onboard/SKILL.md | 137 +- .claude/skills/ad-sharding-ir-port/SKILL.md | 222 ++ examples/auto_deploy/.gitignore | 1 - .../{ => configs}/enable_sharder_ir.yaml | 0 .../auto_deploy/model_registry/models.yaml | 40 +- .../auto_deploy/models/custom/__init__.py | 22 +- .../models/custom/modeling_deepseek.py | 217 +- .../models/custom/modeling_deepseek_ir.py | 776 ----- .../models/custom/modeling_nemotron_h.py | 237 +- .../models/custom/modeling_nemotron_h_ir.py | 827 ----- ...modeling_qwen3_ir.py => modeling_qwen3.py} | 90 +- .../models/custom/modeling_qwen3_5_moe.py | 167 +- .../models/custom/modeling_qwen3_5_moe_ir.py | 3055 ----------------- .../transform/library/hidden_states.py | 18 +- .../library/test_tp_sharding.py | 12 + 15 files changed, 878 insertions(+), 4943 deletions(-) create mode 100644 .claude/skills/ad-sharding-ir-port/SKILL.md rename examples/auto_deploy/model_registry/{ => configs}/enable_sharder_ir.yaml (100%) delete mode 100644 tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek_ir.py delete mode 100644 tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h_ir.py rename tensorrt_llm/_torch/auto_deploy/models/custom/{modeling_qwen3_ir.py => modeling_qwen3.py} (80%) delete mode 100644 tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe_ir.py diff --git a/.claude/skills/ad-model-onboard/SKILL.md b/.claude/skills/ad-model-onboard/SKILL.md index e48c56875677..2e7de8db5a65 100644 --- a/.claude/skills/ad-model-onboard/SKILL.md +++ b/.claude/skills/ad-model-onboard/SKILL.md @@ -340,142 +340,9 @@ GH_CONFIG_DIR= gh pr view --json reviews,state **Do NOT stop polling prematurely.** The loop must continue until the PR is approved or a clear termination signal is received. If polling has been running for an extended period (e.g., >2 hours) with no new activity, inform the user that you are still monitoring and ask if they want you to continue or stop. -## Sharding-aware IR model porting (`modeling_*_ir.py`) +## Sharding-aware IR model porting -Use this when porting an existing AutoDeploy custom model (`tensorrt_llm/_torch/auto_deploy/models/custom/modeling_*.py`) to explicit sharding hint ops in `modeling_*_ir.py` **in the same directory** (no separate `new_sharding/` tree). The exported FX graph must fully specify how the model should be sharded: the `apply_sharding_hints` transform combines hints with a runtime `DistConfig` for deterministic, node-local sharding. - -**Argument reference:** Do not duplicate operator tables here. Refer to the custom op docstrings in `tensorrt_llm/_torch/auto_deploy/custom_ops/` for the complete argument reference (including sharding hints, `tp_mode`, `layer_type`, and which ops accept hints). - -### Reference examples (study before porting) - -| Original | IR / sharding-aware | Layer types | -|----------|---------------------|-------------| -| `modeling_nemotron_h.py` | `modeling_nemotron_h_ir.py` | Mamba SSM, MHA, SwiGLU MLP, MoE | -| `modeling_qwen3_5_moe.py` | `modeling_qwen3_5_moe_ir.py` | GatedDeltaNet, Gated MHA, SwiGLU MLP, MoE | -| `modeling_mistral.py` | `modeling_mistral_ir.py` | MHA, SwiGLU MLP (simplest) | -| `modeling_deepseek_v2.py` | `modeling_deepseek_v2_ir.py` | MLA, SwiGLU MLP, MoE | - -### Step-by-step porting procedure - -#### Step 1: Copy the source file - -```bash -cp tensorrt_llm/_torch/auto_deploy/models/custom/modeling_foo.py \ - tensorrt_llm/_torch/auto_deploy/models/custom/modeling_foo_ir.py -``` - -#### Step 2: Update the module docstring and add imports - -At the top of the IR file: - -```python -import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401 -- register all ops -``` - -Do **not** add global `SHARD_*` flags. Layer-level control uses the `layer_type` hint on each op and `shard_layers` in YAML. - -#### Step 3: Replace linear projections - -For every `self.proj(x)` or `nn.Linear` call, use `torch.ops.auto_deploy.torch_linear_simple` with explicit `tp_mode` and `layer_type`. Always set `tp_mode` unconditionally (no `if _s else "none"`). **Rules:** opening projections (Q/K/V/gate/up/in_proj) → `"colwise"`; closing (O/down/out_proj) → `"rowwise"`; tiny outputs (e.g. `shared_expert_gate` dim 1) → `"none"`; MLA latent projections (q_a, kv_a) → `"none"`. For fused weights split later, pass `output_sizes=[...]`. For GQA, use `tp_min_local_shape=self.head_dim` on K/V colwise lines. - -#### Step 4: Replace split / chunk after fused colwise projections - -Use `torch.ops.auto_deploy.split_with_sizes` with `shardable` / `layer_type` where sizes scale with TP. - -#### Step 5: Replace view / reshape with concrete head counts - -During `torch.export`, `-1` becomes concrete; after TP, wrong values break. Any reshape whose dimension is a head count that scales with TP must use `torch.ops.auto_deploy.view` with `tp_scaled_dim` set appropriately. Safe cases: flat-to-2D, or `[B,S,-1]` when the input is already correctly sharded. - -#### Step 6: Insert `all_reduce` - -After every rowwise projection, add `torch.ops.auto_deploy.all_reduce(..., layer_type=...)`. **Parallel branch rule:** when branches merge by addition, use a **single** `all_reduce` after the sum (e.g. MoE routed + shared expert; parallel attention + MLP residual branches). - -#### Step 7: Special ops (Conv1d, SSM, GatedDeltaNet, gated RMSNorm) - -Add sharding hints on `torch_causal_conv1d`, `torch_ssm`, `torch_gated_delta_rule`, `torch_rmsnorm_gated` per docstrings—typically `shardable` / `output_sizes` / `tp_mode` as required. - -#### Step 8: MoE - -Pass `layer_type="moe"` into `torch_moe`; `apply_sharding_hints` handles EP/TP. - -#### Step 9: Register the IR model - -1. Bottom of the IR file: `AutoModelForCausalLMFactory.register_custom_model_cls("ConfigClassName", ForCausalLM)` (same pattern as Phase 4). -2. Add a **side-effect import** in `tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py` (e.g. `from . import modeling_foo_ir # noqa: F401`) and extend `__all__` if you export symbols. Without this import, worker processes may not load your class and `apply_sharding_hints` can report **0 nodes processed**. Do **not** use a separate `register_sharded_models.py` indirection. - -#### Step 10: YAML — composable registry pattern - -Prefer the model registry (`examples/auto_deploy/model_registry/models.yaml`) and **compose** shared fragments under `examples/auto_deploy/model_registry/configs/`, same as other models: list `dashboard_default.yaml`, the right `world_size_N.yaml`, then a dedicated fragment (e.g. `enable_sharder_ir.yaml`) that holds IR sharding transforms. That fragment should disable legacy sharding passes and enable hint-driven sharding. Registry fragments are deep-merged in `yaml_extra` order (see `DynamicYamlMixInForSettings` in `tensorrt_llm/_torch/auto_deploy/utils/_config.py`); place transform keys under `transforms:` so they merge with `dashboard_default.yaml`. Standalone experiment YAMLs for `build_and_run_ad` may wrap the same fields under a top-level `args:` block matching `LlmArgs`. - -Example transform block: - -```yaml -# Typical contents for enable_sharder_ir.yaml (registry composable fragment) -transforms: - export_to_gm: - num_moe_experts_for_export: 2 # often required when expert count is large (>64) - detect_sharding: - stage: sharding - enabled: false - sharding_transform_executor: - stage: sharding - enabled: false - apply_sharding_hints: - stage: sharding - enabled: true - run_shape_prop: true - allreduce_strategy: NCCL - # shard_layers: ['mha', 'mlp'] # optional selective sharding - gather_logits_before_lm_head: - enabled: true -``` - -Use `world_size: 8` when validating TP head-divisibility. Optional `shard_layers` limits which `layer_type` hints are processed; unset means shard all shardable nodes. - -#### Step 11: Validate - -Do not report success until a run completes successfully. - -1. Prefer `python examples/auto_deploy/build_and_run_ad.py --model --use-registry` after adding/updating the registry entry and composable YAMLs (Phase 8–9 style). -2. `apply_sharding_hints` logs should show **`N nodes processed` with N > 0**. -3. If validation fails with infrastructure limits (e.g. head count not divisible by `world_size`), document the assert and compatible sizes; do not "fix" core `sharding.py` / custom op schemas without owner review. -4. If blocked by missing infrastructure support, rename artifacts to `broken_modeling_*_ir.py` / broken YAML and file a short error report for humans (do not silently patch core transforms). - -**Layer type strings** (for `layer_type` / `shard_layers`): use `"mha"`, `"mla"`, `"mlp"`, `"moe"`, `"ssm"`, `"delta"`, or `"unknown"` (default; skipped when `shard_layers` is set). Match the conventions used in `apply_sharding_hints` and project enums. - -### Layer-specific sharding patterns - -**MHA (standard or gated):** `layer_type="mha"`: q/k/v colwise (GQA: `tp_min_local_shape`), `view` with `tp_scaled_dim` for head dim, o rowwise + `all_reduce`. Fused Q+gate interleaved per head: colwise without `output_sizes`; contiguous Q|K|V fused blocks need `output_sizes`. - -**SwiGLU MLP:** `layer_type="mlp"`: gate/up colwise, down rowwise + `all_reduce`. - -**Mamba / SSM:** `layer_type="ssm"`: in_proj colwise + `output_sizes`, splits shardable, conv1d shardable + `output_sizes`, views, `torch_ssm` shardable, norm gated colwise if weight scales, out rowwise + `all_reduce`. - -**GatedDeltaNet:** `layer_type="delta"`: in_proj_qkv with `output_sizes`, other in_projs colwise, conv1d/splits/views as above, `torch_gated_delta_rule` shardable, out rowwise + `all_reduce`. - -**MoE + shared expert:** `layer_type="moe"`: router replicated; one `all_reduce` after `routed + shared`, not two. - -**MLA (DeepSeek):** `layer_type="mla"`: keep `torch_mla` intact with `shardable=True`—do **not** decompose into separate linears + `torch_attention` (introduces bad `expand`/`view` with concrete head counts). q_a/kv_a latent: `tp_mode="none"`; q_b colwise; `o_proj` rowwise + `all_reduce`. - -### Common pitfalls (sharding IR) - -1. **Missing `auto_deploy::view` for head reshapes** — concrete shapes from export break after sharding. -2. **Sharding tiny projections** — dim-1 gates: `tp_mode="none"`. -3. **Double `all_reduce` in MoE** — one merge-point reduction for routed + shared. -4. **Cross-layer parameter contamination** — in `_apply_hint_*` handlers using `get_source_nodes()`, restrict with `allowed_ops` so residual links do not pull weights from other layers. -5. **Missing `num_moe_experts_for_export`** for very large expert counts — export can hang. -6. **Decomposing ops that absorb weights** (e.g. `torch_mla`) — use `shardable` + handler instead of splitting into plain linears. -7. **Interleaved vs contiguous fused weights** — interleaved per-head groups: colwise only; contiguous Q|K|V blocks: require `output_sizes`. -8. **Omitting `layer_type` when using `shard_layers`** — `"unknown"` nodes are skipped; set hints explicitly on sharding-aware ops. -9. **`layer_type` on non-hint ops** — do **not** pass `layer_type` to ops that are not designed for sharding hints (e.g. `torch_attention`, `torch_l2norm`, `torch_rope_*`); extra positional args break calls. Confirm in `custom_ops/` docstrings which ops accept hints. -10. **Conditional hint values** — no `if _s else "none"`; use unconditional hints and rely on `shard_layers` / transform config. - -### Sharding IR validation checklist (human review) - -- `world_size=1`: unsharded path; hints should not break correctness. -- `world_size=2` and `8`: shape checks and coherent output. -- `apply_sharding_hints` node count vs expectation. -- Optional: `shard_layers: ['moe']` to verify selective sharding. +For porting an existing custom model to a sharding-aware `_ir.py` variant, see the `ad-sharding-ir-port` skill. ## Key Gotchas - **Canonical ops first:** Always use `torch.ops.auto_deploy.torch_*` canonical ops whenever one exists for the operation. This is how AD knows what to optimize. Writing manual attention, MoE, RoPE, or normalization in plain PyTorch instead of using the canonical op will prevent AD transforms from working. diff --git a/.claude/skills/ad-sharding-ir-port/SKILL.md b/.claude/skills/ad-sharding-ir-port/SKILL.md new file mode 100644 index 000000000000..b619d580452a --- /dev/null +++ b/.claude/skills/ad-sharding-ir-port/SKILL.md @@ -0,0 +1,222 @@ +--- +name: ad-sharding-ir-port +description: > + Adds sharding-aware IR hints (op substitutions, sharding kwargs, all_reduce + insertions) directly into an existing AutoDeploy custom model + (modeling_*.py). Edits the file in place — no separate _ir.py copy. + Validates with apply_sharding_hints and end-to-end multi-GPU runs. +license: Apache-2.0 +metadata: + author: NVIDIA Corporation +--- + +# Adding Sharding IR Hints to an AutoDeploy Custom Model + +**Input:** An existing AutoDeploy custom model at `tensorrt_llm/_torch/auto_deploy/models/custom/modeling_*.py`. +**Output:** The same file, updated in place with sharding hints, plus YAML config and validation. + +**No separate `_ir.py` file.** Sharding IR is the default path — hints are added directly to the canonical `modeling_*.py`. The legacy pattern of maintaining parallel `modeling_*_ir.py` files is deprecated. + +**Prerequisites:** Familiarity with AD canonical ops (see `ad-model-onboard` skill, Phase 3) and op registration patterns (Phase 4). Refer to the custom op docstrings in `tensorrt_llm/_torch/auto_deploy/custom_ops/` for the complete argument reference (including sharding hints, `tp_mode`, `layer_type`, and which ops accept hints). + +The exported FX graph must fully specify how the model should be sharded: the `apply_sharding_hints` transform combines hints with a runtime `DistConfig` for deterministic, node-local sharding. + +## Step 0 — Sharding-hint delta contract (READ FIRST) + +Adding sharding hints is a **mechanical, structural transform** of the existing `modeling_.py`, NOT a rewrite. The file at the target branch HEAD before your changes is the AUTHORITATIVE source of model logic. + +You MAY introduce ONLY the following changes: + +**ALLOWED:** + +- **A1. Op substitutions:** + - `nn.Linear(...)` / `F.linear(...)` → `torch.ops.auto_deploy.torch_linear_simple(...)` + - `tensor.view(...)` / `tensor.reshape(...)` → `torch.ops.auto_deploy.view(...)` (only when the shape contains a TP-scaled dim) + - `torch.split(...)` / `torch.split_with_sizes(...)` → `torch.ops.auto_deploy.split_with_sizes(...)` +- **A2. Sharding-hint kwargs added** to call sites of: `torch_moe`, `torch_ssm`, `torch_gated_delta_rule`, `torch_causal_conv1d`, `torch_rmsnorm_gated`, `torch_mla`, `torch_linear_simple`, `auto_deploy.split_with_sizes`, `auto_deploy.view`. Allowed kwargs: `tp_mode`, `layer_type`, `output_sizes`, `tp_min_local_shape`, `tp_scaled_dim`, `shardable`, `enable_sharding`. +- **A3. Inserting `torch.ops.auto_deploy.all_reduce(..., layer_type=...)`** after rowwise projections / at MoE merge points (single all_reduce after routed + shared sums). +- **A4. Adding `import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401`** side-effect import at the top if not already present. +- **A5. The module docstring update** describing the sharding strategy. + +**FORBIDDEN (everything else, including but not limited to):** + +- **F1. Replacing ANY `torch.ops.trtllm.*` op with vanilla PyTorch** (e.g. `noaux_tc_op`, `dsv3_router_gemm_op`, fused norm/MLP kernels). The router gate is TP-replicated; there is nothing to shard. AD has no fusion pass that recovers these kernels from a vanilla rewrite — keep the call site verbatim. +- **F2. Changing the input contract** of `forward()` — adding/removing/changing `assert` or `if` statements that change what the caller must pass. +- **F3. Adding/removing/renaming `nn.Module` subclasses, parameters, buffers**, or `register_load_state_dict_pre_hook` registrations. Module hierarchy and state_dict keys must remain identical. +- **F4. Changing dtype handling, scaling factors, normalization order, mask fill values** (e.g. `0.0` vs `-inf` in `masked_fill`), or any other numerical-semantics detail. +- **F5. Renaming methods, changing return types, changing forward signatures**, or reordering operations. +- **F6. "Cleanup" of allegedly unused code paths.** If it is in the file, it stays. +- **F7. Adding code that does not appear in the original** "because a legacy `_ir.py` reference had it" — legacy IR files may be stale or wrong. + +If a change is required that falls outside the allowlist, **STOP and report it to the parent** for explicit human approval BEFORE writing it. Never silently rewrite logic. + +## Reference examples (study before porting) + +The models below already have sharding hints integrated directly into their `modeling_*.py` files. Study them to see how `tp_mode`, `layer_type`, `output_sizes`, `tp_scaled_dim`, `shardable`, `all_reduce`, etc. are placed for different layer types. + +| Model file | Layer types | +|----------|-------------| +| `modeling_nemotron_h.py` | Mamba SSM, MHA, SwiGLU MLP, MoE | +| `modeling_qwen3_5_moe.py` | GatedDeltaNet, Gated MHA, SwiGLU MLP, MoE | +| `modeling_deepseek.py` | MLA, SwiGLU MLP, MoE | +| `modeling_qwen3.py` | MHA, SwiGLU MLP (simplest MHA example) | + +## Step-by-step procedure + +### Step 1: Create a git checkpoint + +Before editing, ensure the file is committed so you can diff against the original: + +```bash +git stash # or commit — ensure a clean baseline to diff against +``` + +### Step 2: Add the custom_ops side-effect import + +If not already present at the top of the file: + +```python +import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401 -- register all ops +``` + +Do **not** add global `SHARD_*` flags. Layer-level control uses the `layer_type` hint on each op and `shard_layers` in YAML. + +### Step 3: Replace linear projections + +For every `self.proj(x)` or `nn.Linear` call, use `torch.ops.auto_deploy.torch_linear_simple` with explicit `tp_mode` and `layer_type`. Always set `tp_mode` unconditionally (no `if _s else "none"`). **Rules:** opening projections (Q/K/V/gate/up/in_proj) → `"colwise"`; closing (O/down/out_proj) → `"rowwise"`; tiny outputs (e.g. `shared_expert_gate` dim 1) → `"none"`; MLA latent projections (q_a, kv_a) → `"none"`. For fused weights split later, pass `output_sizes=[...]`. For GQA, use `tp_min_local_shape=self.head_dim` on K/V colwise lines. + +### Step 4: Replace split / chunk after fused colwise projections + +Use `torch.ops.auto_deploy.split_with_sizes` with `shardable` / `layer_type` where sizes scale with TP. + +### Step 5: Replace view / reshape with concrete head counts + +During `torch.export`, `-1` becomes concrete; after TP, wrong values break. Any reshape whose dimension is a head count that scales with TP must use `torch.ops.auto_deploy.view` with `tp_scaled_dim` set appropriately. Safe cases: flat-to-2D, or `[B,S,-1]` when the input is already correctly sharded. + +### Step 6: Insert `all_reduce` + +After every rowwise projection, add `torch.ops.auto_deploy.all_reduce(..., layer_type=...)`. **Parallel branch rule:** when branches merge by addition, use a **single** `all_reduce` after the sum (e.g. MoE routed + shared expert; parallel attention + MLP residual branches). + +### Step 7: Special ops (Conv1d, SSM, GatedDeltaNet, gated RMSNorm) + +Add sharding hints on `torch_causal_conv1d`, `torch_ssm`, `torch_gated_delta_rule`, `torch_rmsnorm_gated` per docstrings—typically `shardable` / `output_sizes` / `tp_mode` as required. + +### Step 8: MoE + +Pass `layer_type="moe"` into `torch_moe`; `apply_sharding_hints` handles EP/TP. + +### Step 9: Verify registration + +The model's existing registration (`AutoModelForCausalLMFactory.register_custom_model_cls` at the bottom of the file and its import in `__init__.py`) stays unchanged. No new registration is needed — sharding hints do not change the model identity. + +### Step 10: YAML — enable hint-driven sharding + +Add `enable_sharder_ir.yaml` to the model's `yaml_extra` list in `examples/auto_deploy/model_registry/models.yaml` (if not already present). This composable fragment disables legacy sharding passes and enables `apply_sharding_hints`. Registry fragments are deep-merged in `yaml_extra` order (see `DynamicYamlMixInForSettings` in `tensorrt_llm/_torch/auto_deploy/utils/_config.py`). + +Example transform block: + +```yaml +# Typical contents for enable_sharder_ir.yaml (registry composable fragment) +transforms: + export_to_gm: + num_moe_experts_for_export: 2 # often required when expert count is large (>64) + detect_sharding: + stage: sharding + enabled: false + sharding_transform_executor: + stage: sharding + enabled: false + apply_sharding_hints: + stage: sharding + enabled: true + run_shape_prop: true + allreduce_strategy: NCCL + # shard_layers: ['mha', 'mlp'] # optional selective sharding + gather_logits_before_lm_head: + enabled: true +``` + +Use `world_size: 8` when validating TP head-divisibility. Optional `shard_layers` limits which `layer_type` hints are processed; unset means shard all shardable nodes. + +### Step 11: Validate + +Do not report success until a run completes successfully. + +1. Prefer `python examples/auto_deploy/build_and_run_ad.py --model --use-registry` after updating the registry entry. +2. `apply_sharding_hints` logs should show **`N nodes processed` with N > 0**. +3. If validation fails with infrastructure limits (e.g. head count not divisible by `world_size`), document the assert and compatible sizes; do not "fix" core `sharding.py` / custom op schemas without owner review. +4. If blocked by missing infrastructure support, revert the sharding-hint changes and file a short error report for humans (do not silently patch core transforms). + +**Layer type strings** (for `layer_type` / `shard_layers`): use `"mha"`, `"mla"`, `"mlp"`, `"moe"`, `"ssm"`, `"delta"`, or `"unknown"` (default; skipped when `shard_layers` is set). Match the conventions used in `apply_sharding_hints` and project enums. + +### Step 12 — Pre-finalization self-audit (MANDATORY) + +Before reporting the file as done, you MUST diff your changes against the git baseline: + +```bash +git diff tensorrt_llm/_torch/auto_deploy/models/custom/modeling_.py +``` + +Then classify every hunk into one of the following categories (defined in Step 0): + +| Category | Allowed? | Description | +|---|---|---| +| **A1** | yes | Op substitution (`linear` / `view` / `split`) | +| **A2** | yes | Sharding-hint kwarg added (`tp_mode`, `layer_type`, `output_sizes`, `tp_min_local_shape`, `tp_scaled_dim`, `shardable`, `enable_sharding`) | +| **A3** | yes | `auto_deploy.all_reduce` insertion | +| **A4** | yes | `custom_ops` side-effect import added | +| **A5** | yes | Module docstring update describing sharding strategy | +| **F1** | NO | `torch.ops.trtllm.*` replaced with vanilla PyTorch | +| **F2** | NO | Input contract change (asserts, fallbacks added/removed) | +| **F3** | NO | Module hierarchy / parameter / buffer / load-hook change | +| **F4** | NO | Numerical-semantics change (dtype, scale, mask fill, order) | +| **F5** | NO | Method rename / signature change / op reorder | +| **F6** | NO | Removal of allegedly unused base code | +| **F7** | NO | Code added because a legacy `_ir.py` reference had it (and the base did not) | + +**If you find any F# hunk, REVERT it before reporting done.** Report the full diff classification table back to the parent agent in your final message, with one row per hunk: + +``` +| Hunk lines | Summary of change | Category | Verdict | +|---|---|---|---| +| 234-240 | F.linear → torch_linear_simple, tp_mode="colwise" | A1 + A2 | OK | +| 264-340 | noaux_tc_op replaced with vanilla PyTorch | F1 | REVERTED | +| ... | ... | ... | ... | +``` + +You are NOT done until every row in the table is a yes-allowed category. + +## Layer-specific sharding patterns + +**MHA (standard or gated):** `layer_type="mha"`: q/k/v colwise (GQA: `tp_min_local_shape`), `view` with `tp_scaled_dim` for head dim, o rowwise + `all_reduce`. Fused Q+gate interleaved per head: colwise without `output_sizes`; contiguous Q|K|V fused blocks need `output_sizes`. + +**SwiGLU MLP:** `layer_type="mlp"`: gate/up colwise, down rowwise + `all_reduce`. + +**Mamba / SSM:** `layer_type="ssm"`: in_proj colwise + `output_sizes`, splits shardable, conv1d shardable + `output_sizes`, views, `torch_ssm` shardable, norm gated colwise if weight scales, out rowwise + `all_reduce`. + +**GatedDeltaNet:** `layer_type="delta"`: in_proj_qkv with `output_sizes`, other in_projs colwise, conv1d/splits/views as above, `torch_gated_delta_rule` shardable, out rowwise + `all_reduce`. + +**MoE + shared expert:** `layer_type="moe"`: router replicated; one `all_reduce` after `routed + shared`, not two. + +**MLA (DeepSeek):** `layer_type="mla"`: keep `torch_mla` intact with `shardable=True`—do **not** decompose into separate linears + `torch_attention` (introduces bad `expand`/`view` with concrete head counts). q_a/kv_a latent: `tp_mode="none"`; q_b colwise; `o_proj` rowwise + `all_reduce`. + +## Common pitfalls + +1. **Missing `auto_deploy::view` for head reshapes** — concrete shapes from export break after sharding. +2. **Sharding tiny projections** — dim-1 gates: `tp_mode="none"`. +3. **Double `all_reduce` in MoE** — one merge-point reduction for routed + shared. +4. **Cross-layer parameter contamination** — in `_apply_hint_*` handlers using `get_source_nodes()`, restrict with `allowed_ops` so residual links do not pull weights from other layers. +5. **Missing `num_moe_experts_for_export`** for very large expert counts — export can hang. +6. **Decomposing ops that absorb weights** (e.g. `torch_mla`) — use `shardable` + handler instead of splitting into plain linears. +7. **Interleaved vs contiguous fused weights** — interleaved per-head groups: colwise only; contiguous Q|K|V blocks: require `output_sizes`. +8. **Omitting `layer_type` when using `shard_layers`** — `"unknown"` nodes are skipped; set hints explicitly on sharding-aware ops. +9. **`layer_type` on non-hint ops** — do **not** pass `layer_type` to ops that are not designed for sharding hints (e.g. `torch_attention`, `torch_l2norm`, `torch_rope_*`); extra positional args break calls. Confirm in `custom_ops/` docstrings which ops accept hints. +10. **Conditional hint values** — no `if _s else "none"`; use unconditional hints and rely on `shard_layers` / transform config. +11. **Replacing `torch.ops.trtllm.*` ops** — `noaux_tc_op`, `dsv3_router_gemm_op`, fused norm/MLP kernels are TP-replicated and must be kept verbatim (rule F1). AD has no fusion pass to recover them from vanilla PyTorch. + +## Validation checklist (human review) + +- `world_size=1`: unsharded path; hints should not break correctness. +- `world_size=2` and `8`: shape checks and coherent output. +- `apply_sharding_hints` node count vs expectation. +- Optional: `shard_layers: ['moe']` to verify selective sharding. diff --git a/examples/auto_deploy/.gitignore b/examples/auto_deploy/.gitignore index bb5a36a5efc3..0999a4ed7619 100644 --- a/examples/auto_deploy/.gitignore +++ b/examples/auto_deploy/.gitignore @@ -7,4 +7,3 @@ benchmark_results.json !nano_v3.yaml !nemotron_flash.yaml !model_registry/configs/*.yaml -!model_registry/enable_sharder_ir.yaml diff --git a/examples/auto_deploy/model_registry/enable_sharder_ir.yaml b/examples/auto_deploy/model_registry/configs/enable_sharder_ir.yaml similarity index 100% rename from examples/auto_deploy/model_registry/enable_sharder_ir.yaml rename to examples/auto_deploy/model_registry/configs/enable_sharder_ir.yaml diff --git a/examples/auto_deploy/model_registry/models.yaml b/examples/auto_deploy/model_registry/models.yaml index 454d11a43fff..704a2d9845b2 100644 --- a/examples/auto_deploy/model_registry/models.yaml +++ b/examples/auto_deploy/model_registry/models.yaml @@ -12,7 +12,7 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] - name: Qwen/Qwen3-0.6B config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] - name: apple/OpenELM-270M-Instruct config_id: openelm yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] @@ -60,10 +60,10 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: Qwen/Qwen3-4B config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/Qwen3-8B config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] - name: microsoft/phi-4 config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] @@ -168,10 +168,10 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8 config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] - name: nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4 config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] - name: nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8 config_id: multimodal yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] @@ -228,7 +228,7 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 config_id: nano_v3 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/QwQ-32B-Preview config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] @@ -282,10 +282,10 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] - name: deepseek-ai/DeepSeek-R1 config_id: deepseek_r1 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] - name: deepseek-ai/DeepSeek-R1-0528 config_id: deepseek_r1 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] - name: deepseek-ai/DeepSeek-Coder-V2-Instruct config_id: deepseek_v2_ep yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek_v2_ep.yaml'] @@ -315,7 +315,7 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml','super_v3.yaml'] - name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 config_id: super_v3_mtp - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'super_v3_mtp.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'super_v3_mtp.yaml', 'enable_sharder_ir.yaml'] - name: zai-org/GLM-4.7-Flash config_id: glm_4_7_flash yaml_extra: ['glm-4.7-flash.yaml'] @@ -335,10 +335,10 @@ models: # --- Qwen3.5 MoE (Feb 2026) --- - name: Qwen/Qwen3.5-35B-A3B config_id: qwen3_5_moe_35b - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'qwen3.5_moe_35b.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'qwen3.5_moe_35b.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/Qwen3.5-397B-A17B config_id: qwen3_5_moe_400b - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'qwen3.5_moe_400b.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'qwen3.5_moe_400b.yaml', 'enable_sharder_ir.yaml'] # --- GLM-5 (Feb 2026) --- - name: zai-org/GLM-5 config_id: glm_5 @@ -452,7 +452,7 @@ models: # --- Qwen3 Instruct 2507 update --- - name: Qwen/Qwen3-4B-Instruct-2507 config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] # --- SmolLM3 (Jul 2025) --- - name: HuggingFaceTB/SmolLM3-3B config_id: default_ws_1 @@ -488,14 +488,14 @@ models: # --- Qwen3 missing sizes (May 2025) --- - name: Qwen/Qwen3-1.7B config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/Qwen3-32B config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'enable_sharder_ir.yaml'] # --- DeepSeek R1-0528 (May 2025) --- - name: deepseek-ai/DeepSeek-R1-0528-Qwen3-8B config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] # --- Llama 4 base models (Apr 2025) --- - name: meta-llama/Llama-4-Scout-17B-16E config_id: multimodal__llama4_scout @@ -566,7 +566,7 @@ models: # --- DeepSeek Prover V2 671B (2025) --- - name: deepseek-ai/DeepSeek-Prover-V2-671B config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] # --- OLMo 2 (Mar 2025) --- - name: allenai/OLMo-2-0325-32B-Instruct config_id: default_ws_1 @@ -616,11 +616,11 @@ models: # --- Qwen3-14B (May 2025) --- - name: Qwen/Qwen3-14B config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'enable_sharder_ir.yaml'] # --- DeepSeek V3 (Jan 2025) --- - name: deepseek-ai/DeepSeek-V3 config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] # --- Qwen2.5 larger sizes --- - name: Qwen/Qwen2.5-14B-Instruct config_id: default_ws_4 @@ -635,7 +635,7 @@ models: # --- Nvidia Nemotron 3 Nano (2025) --- - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 config_id: nano_v3 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] - name: nvidia/NVIDIA-Nemotron-Nano-12B-v2 config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] @@ -653,7 +653,7 @@ models: yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] - name: Qwen/Qwen3-0.6B-FP8 config_id: qwen3_fp8_bf16 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'qwen3_fp8_bf16.yaml'] + yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'qwen3_fp8_bf16.yaml', 'enable_sharder_ir.yaml'] # --- Mistral updates (2025) --- - name: mistralai/Codestral-25.01 config_id: default_ws_4 diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py b/tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py index 5c93a260d96b..e90767977c06 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py @@ -21,6 +21,11 @@ # Import each custom model individually so that models with transitive TRT-LLM # dependencies (e.g., NemotronH needing mamba layernorm_gated) don't prevent # other models from loading in standalone mode. +# +# NOTE: deepseek, nemotron_h, qwen3, qwen3_5_moe entries are the sharding-IR +# variants (see PR for #13429). The legacy non-IR versions of these four files +# were removed in the same change; the canonical name now points to the +# IR-aware implementation that uses ``apply_sharding_hints`` for TP/EP/BMM. _MODEL_MODULES = { "modeling_cohere": ["CohereForCausalLM"], "modeling_decilm": ["DeciLMForCausalLM"], @@ -50,6 +55,7 @@ "modeling_nemotron_h": ["NemotronHForCausalLM"], "modeling_olmo3": ["Olmo3ForCausalLM"], "modeling_openelm": ["OpenELMForCausalLM"], + "modeling_qwen3": ["Qwen3ForCausalLM"], "modeling_qwen3_5_moe": ["Qwen3_5MoeForCausalLM", "Qwen3_5MoeForConditionalGeneration"], "modeling_qwen3_moe": ["Qwen3MoeForCausalLM"], "modeling_qwen3_next": ["Qwen3NextForCausalLM"], @@ -59,15 +65,15 @@ "modeling_starcoder2": ["Starcoder2ForCausalLM"], } +# AD_USE_IR_MODELS: opt-in flag for staging additional ``_ir.py`` modeling +# variants alongside the canonical models above. PR #13478 promoted the +# deepseek/nemotron_h/qwen3/qwen3_5_moe ``_ir.py`` variants to canonical +# names, so the gate currently has no entries to load. Keep the hook in +# place: future model onboardings may stage an ``_ir.py`` variant here +# for side-by-side comparison while the legacy ``detect_sharding`` path +# is still supported and the canonical rename hasn't happened yet. if os.environ.get("AD_USE_IR_MODELS"): - _MODEL_MODULES["modeling_deepseek_ir"] = ["DeepSeekV3ForCausalLM"] - _MODEL_MODULES["modeling_llama3_ir"] = ["Llama3ForCausalLM"] - _MODEL_MODULES["modeling_nemotron_h_ir"] = ["NemotronHForCausalLM"] - _MODEL_MODULES["modeling_qwen3_5_moe_ir"] = [ - "Qwen3_5MoeForCausalLM", - "Qwen3_5MoeForConditionalGeneration", - ] - _MODEL_MODULES["modeling_qwen3_ir"] = ["Qwen3ForCausalLM"] + pass # no staged _ir.py variants at the moment __all__ = [] for _module_name, _names in _MODEL_MODULES.items(): diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py index eb5977c083b9..1bf0c7d5ca78 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py @@ -5,19 +5,53 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Slimmed down PyTorch DeepSeekV3 model implementation for auto_deploy export. - -Source: -https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/main/modeling_deepseek.py - -This implementation differs from the original in the following ways: -* Simplified for prefill-only inference (no KV caching) -* Uses auto_deploy custom ops for export compatibility -* Removed flash attention variants (uses torch_mla custom op) -* Removed gradient checkpointing and training code paths -* Removed attention dropout (inference only) - -This allows us to have a clean export-ready implementation with auto_deploy custom ops. +"""DeepSeekV3 model with explicit sharding-hint custom ops for AutoDeploy IR sharding. + +Sharding-aware variant of ``modeling_deepseek.py``. The non-IR file is the +authoritative source of model logic; this file applies a strictly mechanical +structural transform that encodes TP sharding intent in the exported FX graph +via ``auto_deploy`` custom ops with sharding-hint kwargs. The +``apply_sharding_hints`` transform reads these hints together with a runtime +``DistConfig`` and applies deterministic, node-local sharding. + +Mechanical deltas vs ``modeling_deepseek.py``: +* ``nn.Linear`` / ``F.linear`` projections become + ``torch.ops.auto_deploy.torch_linear_simple`` calls carrying ``tp_mode`` and + ``layer_type`` hints. +* TP-scaled head reshapes use ``torch.ops.auto_deploy.view`` with + ``tp_scaled_dim``. +* Rowwise projections / MoE merge points are followed by + ``torch.ops.auto_deploy.all_reduce(..., layer_type=...)``. +* ``torch.ops.auto_deploy.torch_mla`` carries ``enable_sharding=True`` and + ``layer_type="mla"`` so ``_apply_hint_mla`` shards ``kv_b_proj.weight`` + column-wise per head without decomposing the MLA op. +* ``torch.ops.auto_deploy.torch_moe`` carries ``layer_type="moe"``. + +Sharding strategy: +* **MLA** (``DeepSeekV3Attention``): ``q_a_proj`` and ``kv_a_proj_with_mqa`` + stay replicated (``tp_mode="none"``); ``q_b_proj`` is colwise (sharded by + ``num_heads``); ``o_proj`` is rowwise + ``all_reduce(layer_type="mla")``; + ``torch_mla`` carries ``enable_sharding=True, layer_type="mla"``. +* **MoE** (``DeepSeekV3MoE``): the ``noaux_tc`` router gate is TP-replicated + and keeps its ``torch.ops.trtllm.{dsv3_router_gemm_op, noaux_tc_op}`` calls + verbatim from the non-IR base (no sharding hints; AD has no fusion that + recovers these kernels from a vanilla rewrite). ``torch_moe`` carries + ``layer_type="moe"``. The shared expert MLP is constructed with + ``add_all_reduce=False, layer_type="moe"`` so its closing all_reduce is + deferred. A single ``all_reduce(layer_type="moe")`` is emitted after the + routed + shared partial sums. +* **MLP** (``DeepSeekV3MLP``): SwiGLU gate/up are colwise, down is rowwise + + ``all_reduce(layer_type="mlp")``. + +Both ``load_state_dict`` pre-hooks from the non-IR file are preserved verbatim: +* ``mla_rope_utils._rope_deinterleave_load_hook`` -- permutes RoPE columns of + ``q_b_proj`` and ``kv_a_proj_with_mqa`` from interleaved to NeoX layout so + the forward can use ``torch_rope_with_explicit_cos_sin`` + (-> ``flashinfer_rope``). +* ``mla_rope_utils._kv_b_proj_dequant_load_hook`` -- dequantizes FP8 + ``kv_b_proj.weight`` using its block-wise ``weight_scale_inv``; + ``kv_b_proj`` is consumed directly by ``torch_mla`` (not via a quantized + linear op), so the FineGrainedFP8 transform skips it. """ import math @@ -33,6 +67,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput +from ... import custom_ops # noqa: F401 -- register all ops from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory from . import mla_rope_utils @@ -178,10 +213,20 @@ def _yarn_linear_ramp_mask(min_val: float, max_val: float, dim: int) -> torch.Te class DeepSeekV3MLP(nn.Module): - """MLP layer for DeepSeekV3 (SwiGLU activation).""" + """MLP layer for DeepSeekV3 (SwiGLU activation). + + When used as a shared expert inside MoE, ``add_all_reduce=False`` and + ``layer_type="moe"`` so the closing all_reduce is deferred to the merge + point and combined with the routed expert output. + """ def __init__( - self, config, hidden_size: Optional[int] = None, intermediate_size: Optional[int] = None + self, + config, + hidden_size: Optional[int] = None, + intermediate_size: Optional[int] = None, + add_all_reduce: bool = True, + layer_type: str = "mlp", ): super().__init__() self.config = config @@ -192,13 +237,46 @@ def __init__( self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) self.act_fn = ACT2FN[config.hidden_act] + self.add_all_reduce = add_all_reduce + self.layer_type = layer_type def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + gate = torch.ops.auto_deploy.torch_linear_simple( + x, + self.gate_proj.weight, + self.gate_proj.bias, + tp_mode="colwise", + layer_type=self.layer_type, + ) + up = torch.ops.auto_deploy.torch_linear_simple( + x, + self.up_proj.weight, + self.up_proj.bias, + tp_mode="colwise", + layer_type=self.layer_type, + ) + down = torch.ops.auto_deploy.torch_linear_simple( + self.act_fn(gate) * up, + self.down_proj.weight, + self.down_proj.bias, + tp_mode="rowwise", + layer_type=self.layer_type, + ) + if self.add_all_reduce: + down = torch.ops.auto_deploy.all_reduce(down, layer_type=self.layer_type) + return down class DeepSeekV3MoEGate(nn.Module): - """MoE Gating for DeepSeekV3 with noaux_tc top-k selection.""" + """MoE Gating for DeepSeekV3 with noaux_tc top-k selection. + + The router gate is TP-replicated; weight and outputs are identical on every + rank, so no sharding hints are applied here. The fused + ``torch.ops.trtllm.dsv3_router_gemm_op`` and + ``torch.ops.trtllm.noaux_tc_op`` calls are kept verbatim from the non-IR + base -- AD has no transform that recovers these kernels from a vanilla + PyTorch rewrite. + """ def __init__(self, config): super().__init__() @@ -249,7 +327,14 @@ def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tens class DeepSeekV3MoE(nn.Module): - """Mixture of Experts layer for DeepSeekV3.""" + """Mixture of Experts layer for DeepSeekV3. + + Routed experts are dispatched via ``torch_moe`` (sharded by + ``apply_sharding_hints`` using ``layer_type="moe"``). The shared expert is a + TP-sharded MLP whose closing all_reduce is deferred so the routed and + shared partial sums can be combined with a single + ``all_reduce(layer_type="moe")`` at the merge point. + """ def __init__(self, config): super().__init__() @@ -270,7 +355,12 @@ def __init__(self, config): # Shared experts (if configured) if config.n_shared_experts is not None: intermediate_size = config.moe_intermediate_size * config.n_shared_experts - self.shared_experts = DeepSeekV3MLP(config, intermediate_size=intermediate_size) + self.shared_experts = DeepSeekV3MLP( + config, + intermediate_size=intermediate_size, + add_all_reduce=False, + layer_type="moe", + ) else: self.shared_experts = None @@ -296,6 +386,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: w3_weight=[expert.up_proj.weight for expert in self.experts], is_gated_mlp=True, act_fn=int(ActivationType.Silu), + layer_type="moe", ) final_hidden_states = final_hidden_states.view(*orig_shape) @@ -303,6 +394,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: if self.shared_experts is not None: final_hidden_states = final_hidden_states + shared_expert_output + # Single merge-point all_reduce for routed + shared partial sums. + final_hidden_states = torch.ops.auto_deploy.all_reduce( + final_hidden_states, layer_type="moe" + ) + return final_hidden_states.to(hidden_states.dtype) @@ -310,6 +406,17 @@ class DeepSeekV3Attention(nn.Module): """Multi-head Latent Attention (MLA) for DeepSeekV3. Uses compressed KV representation with latent projections. + + Sharding strategy: + ``q_a_proj`` / ``kv_a_proj_with_mqa`` -> ``tp_mode="none"`` (replicated + latent projections). + ``q_b_proj`` -> ``tp_mode="colwise"`` (sharded by ``num_heads``). + Q reshape -> ``auto_deploy.view`` with ``tp_scaled_dim=2``. + ``torch_mla`` -> ``enable_sharding=True, layer_type="mla"``. Do NOT + decompose ``torch_mla`` into separate linears + ``torch_attention`` -- + ``_apply_hint_mla`` shards ``kv_b_proj.weight`` column-wise per head. + Post-attention reshape -> ``auto_deploy.view`` with ``tp_scaled_dim=2``. + ``o_proj`` -> ``tp_mode="rowwise"`` + ``all_reduce(layer_type="mla")``. """ def __init__(self, config, layer_idx: Optional[int] = None): @@ -423,18 +530,49 @@ def forward( ) -> torch.Tensor: bsz, q_len, _ = hidden_states.size() - # Q projection + # Q projection: latent projections replicated, q_b_proj colwise. if self.q_lora_rank is None: - q = self.q_proj(hidden_states) + q = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.q_proj.weight, + self.q_proj.bias, + tp_mode="colwise", + layer_type="mla", + ) else: - q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states))) + q = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.q_a_proj.weight, + self.q_a_proj.bias, + tp_mode="none", + layer_type="mla", + ) + q = self.q_a_layernorm(q) + q = torch.ops.auto_deploy.torch_linear_simple( + q, + self.q_b_proj.weight, + self.q_b_proj.bias, + tp_mode="colwise", + layer_type="mla", + ) - # Shape: [B, S, N, q_head_dim] (BSND layout) - q = q.view(bsz, q_len, self.num_heads, self.q_head_dim) + # Shape: [B, S, N, q_head_dim] (BSND layout); num_heads (dim 2) scales with TP. + q = torch.ops.auto_deploy.view( + q, + [bsz, q_len, self.num_heads, self.q_head_dim], + tp_scaled_dim=2, + layer_type="mla", + ) q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - # KV projection - keep compressed form - kv_a_output = self.kv_a_proj_with_mqa(hidden_states) + # KV projection - keep compressed form. Latent compression is replicated. + kv_a_output = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.kv_a_proj_with_mqa.weight, + self.kv_a_proj_with_mqa.bias, + tp_mode="none", + layer_type="mla", + ) compressed_kv, k_pe = torch.split( kv_a_output, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 ) @@ -443,6 +581,7 @@ def forward( compressed_kv = self.kv_a_layernorm(compressed_kv) # k_pe: [B, S, 1, qk_rope_head_dim] (BSND layout, shared across heads) + # dim 2 is fixed at 1 and never scales with TP, so plain `.view` is correct. k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim) kv_seq_len = q_len @@ -460,7 +599,10 @@ def forward( 2, # unsqueeze_dim=2 for BSND layout ) - # Call MLA with compressed KV + # Call MLA with compressed KV. enable_sharding=True lets _apply_hint_mla + # shard kv_b_proj_weight column-wise along the head dimension. Do NOT + # decompose torch_mla into separate linears + torch_attention -- that + # introduces concrete-shape view/expand that break under TP. attn_output = torch.ops.auto_deploy.torch_mla( q_nope, # [B, S, N, qk_nope_head_dim] q_pe_rotated, # [B, S, N, qk_rope_head_dim] @@ -470,11 +612,26 @@ def forward( True, # is_causal self.softmax_scale, "bsnd", # layout + enable_sharding=True, + layer_type="mla", ) - # Output: [B, S, N, v_head_dim] -> [B, S, N * v_head_dim] - attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.v_head_dim) - attn_output = self.o_proj(attn_output) + # Output: [B, S, N, v_head_dim] -> [B, S, N * v_head_dim]. + # Collapsed dim scales with TP via num_heads. + attn_output = torch.ops.auto_deploy.view( + attn_output, + [bsz, q_len, self.num_heads * self.v_head_dim], + tp_scaled_dim=2, + layer_type="mla", + ) + attn_output = torch.ops.auto_deploy.torch_linear_simple( + attn_output, + self.o_proj.weight, + self.o_proj.bias, + tp_mode="rowwise", + layer_type="mla", + ) + attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mla") return attn_output diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek_ir.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek_ir.py deleted file mode 100644 index 34d7137abf63..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek_ir.py +++ /dev/null @@ -1,776 +0,0 @@ -# Copyright 2018 The HuggingFace Team -# Licensed under the Apache License, Version 2.0. -# Original source: https://github.com/huggingface/transformers -# -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""DeepSeekV3 model with explicit sharding hint ops. - -WARNING: tested only on up to 8 layers (8xH100 machine can't fit the full model, -the output is not fully verified). The sharding pipeline processes all nodes -correctly (41 nodes at 4 layers, 59+ at 8 layers) but coherent end-to-end output -has not been validated due to memory constraints. Additionally, the DeepSeek-V3/R1 -FP8 checkpoints use dsv3_router_gemm_op which crashes on H100 (pre-existing bug). - -Based on the original modeling_deepseek.py. All enable_sharding operations use -AutoDeploy custom ops with sharding hint kwargs. The graph produced by this -model is a complete, self-contained specification of "how this model should be -sharded." The ``apply_sharding_hints`` transform reads the hints together with -a runtime ``DistConfig`` to apply deterministic, node-local sharding. - -Shardable custom ops used: - - torch.ops.auto_deploy.torch_linear_simple (tp_mode) - - torch.ops.auto_deploy.view (tp_scaled_dim) - - torch.ops.auto_deploy.all_reduce (identity / dist.all_reduce) - - torch.ops.auto_deploy.torch_mla (enable_sharding) - - torch.ops.auto_deploy.torch_moe (sharded by apply_sharding_hints) -""" - -import math -from dataclasses import dataclass -from functools import partial -from typing import Optional, Tuple - -import torch -import torch.nn.functional as F -from torch import nn -from transformers.activations import ACT2FN -from transformers.generation import GenerationMixin -from transformers.modeling_utils import PreTrainedModel -from transformers.utils import ModelOutput - -from tensorrt_llm._torch.utils import ActivationType - -from ... import custom_ops # noqa: F401 -- register all ops -from ..hf import AutoModelForCausalLMFactory -from . import mla_rope_utils - - -class DeepSeekV3RMSNorm(nn.Module): - """RMS Normalization for DeepSeekV3.""" - - def __init__(self, hidden_size: int, eps: float = 1e-6): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - return torch.ops.auto_deploy.triton_rms_norm( - hidden_states, self.weight, self.variance_epsilon - ).to(hidden_states.dtype) - - -class DeepSeekV3RotaryEmbedding(nn.Module): - """Rotary Position Embedding for DeepSeekV3. - - Simplified version that precomputes and caches cos/sin values. - Returns full cached values (not sliced by seq_len) to enable export. - """ - - def __init__(self, dim: int, max_position_embeddings: int = 2048, base: float = 10000.0): - super().__init__() - self.dim = dim - self.max_position_embeddings = max_position_embeddings - self.base = base - - inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - # Build cos/sin cache - self._set_cos_sin_cache(max_position_embeddings) - - def _set_cos_sin_cache(self, seq_len: int): - self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos(), persistent=False) - self.register_buffer("sin_cached", emb.sin(), persistent=False) - - def forward( - self, x: torch.Tensor, seq_len: Optional[int] = None - ) -> Tuple[torch.Tensor, torch.Tensor]: - return ( - self.cos_cached.to(dtype=x.dtype, device=x.device), - self.sin_cached.to(dtype=x.dtype, device=x.device), - ) - - -class DeepSeekV3YarnRotaryEmbedding(DeepSeekV3RotaryEmbedding): - """YaRN-extended rotary embedding for DeepSeekV3.""" - - def __init__( - self, - dim: int, - max_position_embeddings: int = 2048, - base: float = 10000.0, - scaling_factor: float = 1.0, - original_max_position_embeddings: int = 4096, - beta_fast: int = 32, - beta_slow: int = 1, - mscale: float = 1.0, - mscale_all_dim: float = 0.0, - ): - self.scaling_factor = scaling_factor - self.original_max_position_embeddings = original_max_position_embeddings - self.beta_fast = beta_fast - self.beta_slow = beta_slow - self.mscale = mscale - self.mscale_all_dim = mscale_all_dim - super().__init__(dim, max_position_embeddings, base) - - def _set_cos_sin_cache(self, seq_len: int): - self.max_seq_len_cached = seq_len - dim = self.dim - - freq_extra = 1.0 / (self.base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) - freq_inter = 1.0 / ( - self.scaling_factor * self.base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim) - ) - - low, high = self._yarn_find_correction_range( - self.beta_fast, - self.beta_slow, - dim, - self.base, - self.original_max_position_embeddings, - ) - inv_freq_mask = 1.0 - self._yarn_linear_ramp_mask(low, high, dim // 2) - inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask - self.register_buffer("inv_freq", inv_freq, persistent=False) - - t = torch.arange(seq_len, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - - _mscale = float( - self._yarn_get_mscale(self.scaling_factor, self.mscale) - / self._yarn_get_mscale(self.scaling_factor, self.mscale_all_dim) - ) - - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", (emb.cos() * _mscale), persistent=False) - self.register_buffer("sin_cached", (emb.sin() * _mscale), persistent=False) - - @staticmethod - def _yarn_find_correction_dim( - num_rotations: float, dim: int, base: float = 10000, max_position_embeddings: int = 2048 - ) -> float: - return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / ( - 2 * math.log(base) - ) - - def _yarn_find_correction_range( - self, low_rot: int, high_rot: int, dim: int, base: float, max_position_embeddings: int - ) -> Tuple[int, int]: - low = math.floor( - self._yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings) - ) - high = math.ceil( - self._yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings) - ) - return max(low, 0), min(high, dim - 1) - - @staticmethod - def _yarn_get_mscale(scale: float = 1.0, mscale: float = 1.0) -> float: - if scale <= 1: - return 1.0 - return 0.1 * mscale * math.log(scale) + 1.0 - - @staticmethod - def _yarn_linear_ramp_mask(min_val: float, max_val: float, dim: int) -> torch.Tensor: - if min_val == max_val: - max_val += 0.001 - linear_func = (torch.arange(dim, dtype=torch.float32) - min_val) / (max_val - min_val) - return torch.clamp(linear_func, 0, 1) - - -class DeepSeekV3MLP(nn.Module): - """MLP layer for DeepSeekV3 (SwiGLU activation) with sharding hints. - - When used as a shared expert inside MoE, ``add_all_reduce`` is set to False - and the all_reduce is deferred to the MoE merge point. - """ - - def __init__( - self, - config, - hidden_size: Optional[int] = None, - intermediate_size: Optional[int] = None, - add_all_reduce: bool = True, - layer_type: str = "mlp", - ): - super().__init__() - self.config = config - self.hidden_size = hidden_size or config.hidden_size - self.intermediate_size = intermediate_size or config.intermediate_size - - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_act] - self.add_all_reduce = add_all_reduce - self.layer_type = layer_type - - def forward(self, x: torch.Tensor) -> torch.Tensor: - gate = torch.ops.auto_deploy.torch_linear_simple( - x, - self.gate_proj.weight, - self.gate_proj.bias, - tp_mode="colwise", - layer_type=self.layer_type, - ) - up = torch.ops.auto_deploy.torch_linear_simple( - x, - self.up_proj.weight, - self.up_proj.bias, - tp_mode="colwise", - layer_type=self.layer_type, - ) - down = torch.ops.auto_deploy.torch_linear_simple( - self.act_fn(gate) * up, - self.down_proj.weight, - self.down_proj.bias, - tp_mode="rowwise", - layer_type=self.layer_type, - ) - if self.add_all_reduce: - down = torch.ops.auto_deploy.all_reduce(down, layer_type=self.layer_type) - return down - - -class DeepSeekV3MoEGate(nn.Module): - """MoE Gating for DeepSeekV3 with noaux_tc top-k selection.""" - - def __init__(self, config): - super().__init__() - self.config = config - self.top_k = config.num_experts_per_tok - self.n_routed_experts = config.n_routed_experts - self.routed_scaling_factor = config.routed_scaling_factor - self.n_group = config.n_group - self.topk_group = config.topk_group - - self.weight = nn.Parameter( - torch.empty((self.n_routed_experts, config.hidden_size), dtype=torch.float32) - ) - self.register_buffer( - "e_score_correction_bias", - torch.zeros(self.n_routed_experts, dtype=torch.float32), - ) - self.reset_parameters() - - def reset_parameters(self) -> None: - """Initialize gate weights using kaiming uniform (matches original DeepSeek implementation).""" - torch.nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) - - def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward pass returning (selected_experts, routing_weights).""" - bsz, seq_len, hidden_dim = hidden_states.shape - hidden_states_flat = hidden_states.view(-1, hidden_dim) - - if self.weight.dtype == torch.float32: - router_logits = F.linear(hidden_states_flat.float(), self.weight) - else: - router_logits = torch.ops.trtllm.dsv3_router_gemm_op( - hidden_states_flat, self.weight.t(), bias=None, out_dtype=torch.float32 - ) - - topk_weights, topk_indices = torch.ops.trtllm.noaux_tc_op( - router_logits, - self.e_score_correction_bias, - self.n_group, - self.topk_group, - self.top_k, - self.routed_scaling_factor, - ) - - return topk_indices, topk_weights - - -class DeepSeekV3MoE(nn.Module): - """Mixture of Experts layer for DeepSeekV3 with sharding hints. - - Routed experts are handled by torch_moe (sharded by apply_sharding_hints). - Shared expert uses TP-sharded MLP with deferred all_reduce. - Single all_reduce at the merge point (routed + shared). - """ - - def __init__(self, config): - super().__init__() - self.config = config - self.num_experts_per_tok = config.num_experts_per_tok - - self.experts = nn.ModuleList( - [ - DeepSeekV3MLP(config, intermediate_size=config.moe_intermediate_size) - for _ in range(config.n_routed_experts) - ] - ) - - self.gate = DeepSeekV3MoEGate(config) - - if config.n_shared_experts is not None: - intermediate_size = config.moe_intermediate_size * config.n_shared_experts - self.shared_experts = DeepSeekV3MLP( - config, - intermediate_size=intermediate_size, - add_all_reduce=False, - layer_type="moe", - ) - else: - self.shared_experts = None - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - identity = hidden_states - orig_shape = hidden_states.shape - - selected_experts, routing_weights = self.gate(hidden_states) - - if self.shared_experts is not None: - shared_expert_output = self.shared_experts(identity) - - final_hidden_states = torch.ops.auto_deploy.torch_moe( - hidden_states.view(-1, hidden_states.shape[-1]), - selected_experts, - routing_weights, - w1_weight=[expert.gate_proj.weight for expert in self.experts], - w2_weight=[expert.down_proj.weight for expert in self.experts], - w3_weight=[expert.up_proj.weight for expert in self.experts], - is_gated_mlp=True, - act_fn=int(ActivationType.Silu), - layer_type="moe", - ) - - final_hidden_states = final_hidden_states.view(*orig_shape) - - if self.shared_experts is not None: - final_hidden_states = final_hidden_states + shared_expert_output - - final_hidden_states = torch.ops.auto_deploy.all_reduce( - final_hidden_states, layer_type="moe" - ) - - return final_hidden_states.to(hidden_states.dtype) - - -class DeepSeekV3Attention(nn.Module): - """Multi-head Latent Attention (MLA) for DeepSeekV3 with sharding hints. - - MLA sharding strategy (from porting instructions): - q_a_proj -> tp_mode="none" (replicated latent projection) - q_a_layernorm -> unchanged - q_b_proj -> tp_mode="colwise" (shard by num_heads) - kv_a_proj -> tp_mode="none" (replicated latent projection) - kv_a_layernorm -> unchanged - torch_mla -> enable_sharding=True (kv_b_proj_weight sharded by _apply_hint_mla) - view -> tp_scaled_dim=2 for num_heads (Q reshape only) - o_proj -> tp_mode="rowwise" + all_reduce - """ - - def __init__(self, config, layer_idx: Optional[int] = None): - super().__init__() - self.config = config - self.layer_idx = layer_idx - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.q_lora_rank = config.q_lora_rank - self.kv_lora_rank = config.kv_lora_rank - self.qk_nope_head_dim = config.qk_nope_head_dim - self.qk_rope_head_dim = config.qk_rope_head_dim - self.v_head_dim = config.v_head_dim - self.q_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim - - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - - if self.q_lora_rank is None: - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.q_head_dim, bias=False) - else: - self.q_a_proj = nn.Linear( - self.hidden_size, config.q_lora_rank, bias=config.attention_bias - ) - self.q_a_layernorm = DeepSeekV3RMSNorm(config.q_lora_rank) - self.q_b_proj = nn.Linear( - config.q_lora_rank, self.num_heads * self.q_head_dim, bias=False - ) - - self.kv_a_proj_with_mqa = nn.Linear( - self.hidden_size, - self.kv_lora_rank + self.qk_rope_head_dim, - bias=config.attention_bias, - ) - self.kv_a_layernorm = DeepSeekV3RMSNorm(self.kv_lora_rank) - self.kv_b_proj = nn.Linear( - self.kv_lora_rank, - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), - bias=False, - ) - - self.o_proj = nn.Linear( - self.num_heads * self.v_head_dim, self.hidden_size, bias=config.attention_bias - ) - - self._init_rope() - - self.softmax_scale = self.q_head_dim ** (-0.5) - if config.rope_scaling is not None: - mscale_all_dim = config.rope_scaling.get("mscale_all_dim", 0) - scaling_factor = config.rope_scaling["factor"] - if mscale_all_dim: - mscale = DeepSeekV3YarnRotaryEmbedding._yarn_get_mscale( - scaling_factor, mscale_all_dim - ) - self.softmax_scale = self.softmax_scale * mscale * mscale - - def _init_rope(self): - if self.config.rope_scaling is None: - self.rotary_emb = DeepSeekV3RotaryEmbedding( - self.qk_rope_head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - else: - scaling_type = self.config.rope_scaling["type"] - scaling_factor = self.config.rope_scaling["factor"] - - if scaling_type == "yarn": - kwargs = { - key: self.config.rope_scaling[key] - for key in [ - "original_max_position_embeddings", - "beta_fast", - "beta_slow", - "mscale", - "mscale_all_dim", - ] - if key in self.config.rope_scaling - } - self.rotary_emb = DeepSeekV3YarnRotaryEmbedding( - self.qk_rope_head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - **kwargs, - ) - else: - self.rotary_emb = DeepSeekV3RotaryEmbedding( - self.qk_rope_head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - - def forward( - self, - hidden_states: torch.Tensor, - position_ids: torch.Tensor, - ) -> torch.Tensor: - bsz, q_len, _ = hidden_states.size() - - # Q projection: latent projections are replicated, q_b_proj is colwise - if self.q_lora_rank is None: - q = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.q_proj.weight, - self.q_proj.bias, - tp_mode="colwise", - layer_type="mla", - ) - else: - q = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.q_a_proj.weight, - self.q_a_proj.bias, - tp_mode="none", - layer_type="mla", - ) - q = self.q_a_layernorm(q) - q = torch.ops.auto_deploy.torch_linear_simple( - q, - self.q_b_proj.weight, - self.q_b_proj.bias, - tp_mode="colwise", - layer_type="mla", - ) - - # Shape: [B, S, N, q_head_dim] -- num_heads at dim 2 scales with TP - q = torch.ops.auto_deploy.view( - q, - [bsz, q_len, self.num_heads, self.q_head_dim], - tp_scaled_dim=2, - layer_type="mla", - ) - # Split on last dim (head_dim) -- does NOT scale with TP, plain torch.split - q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - - # KV projection -- replicated (latent compression, not per-head) - kv_a_output = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.kv_a_proj_with_mqa.weight, - self.kv_a_proj_with_mqa.bias, - tp_mode="none", - layer_type="mla", - ) - compressed_kv, k_pe = torch.split( - kv_a_output, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 - ) - - compressed_kv = self.kv_a_layernorm(compressed_kv) - - # k_pe: [B, S, 1, qk_rope_head_dim] -- shared across heads, dim 2 is always 1 - k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim) - - kv_seq_len = q_len - - cos, sin = self.rotary_emb(hidden_states, seq_len=kv_seq_len) - cos = cos[position_ids] - sin = sin[position_ids] - - q_pe_rotated, kpe = torch.ops.auto_deploy.torch_rope_with_explicit_cos_sin( - q_pe, - k_pe, - cos, - sin, - 2, - ) - - # MLA: enable_sharding=True lets _apply_hint_mla shard kv_b_proj_weight colwise - attn_output = torch.ops.auto_deploy.torch_mla( - q_nope, - q_pe_rotated, - compressed_kv, - kpe, - self.kv_b_proj.weight, - True, - self.softmax_scale, - "bsnd", - enable_sharding=True, - layer_type="mla", - ) - - # Output: [B, S, N, v_head_dim] -> [B, S, N * v_head_dim] - attn_output = torch.ops.auto_deploy.view( - attn_output, - [bsz, q_len, self.num_heads * self.v_head_dim], - tp_scaled_dim=2, - layer_type="mla", - ) - attn_output = torch.ops.auto_deploy.torch_linear_simple( - attn_output, - self.o_proj.weight, - self.o_proj.bias, - tp_mode="rowwise", - layer_type="mla", - ) - attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mla") - - return attn_output - - -class DeepSeekV3DecoderLayer(nn.Module): - """Transformer decoder layer for DeepSeekV3.""" - - def __init__(self, config, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - self.layer_idx = layer_idx - - self.self_attn = DeepSeekV3Attention(config, layer_idx=layer_idx) - - use_moe = ( - config.n_routed_experts is not None - and layer_idx >= config.first_k_dense_replace - and layer_idx % config.moe_layer_freq == 0 - ) - if use_moe: - self.mlp = DeepSeekV3MoE(config) - else: - self.mlp = DeepSeekV3MLP(config) - - self.input_layernorm = DeepSeekV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = DeepSeekV3RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - hidden_states: torch.Tensor, - position_ids: torch.Tensor, - ) -> torch.Tensor: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - hidden_states = self.self_attn(hidden_states, position_ids) - hidden_states = residual + hidden_states - - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - - return hidden_states - - -@dataclass -class DeepSeekV3Output(ModelOutput): - """Output for DeepSeekV3Model.""" - - last_hidden_state: Optional[torch.FloatTensor] = None - - -@dataclass -class DeepSeekV3CausalLMOutput(ModelOutput): - """Output for DeepSeekV3ForCausalLM.""" - - logits: Optional[torch.FloatTensor] = None - - -class DeepSeekV3PreTrainedModel(PreTrainedModel): - """Base class for DeepSeekV3 models.""" - - base_model_prefix = "model" - _no_split_modules = ["DeepSeekV3DecoderLayer"] - supports_gradient_checkpointing = False - - def _init_weights(self, module): - std = self.config.initializer_range - if isinstance(module, nn.Linear): - module.weight.data.normal_(mean=0.0, std=std) - if module.bias is not None: - module.bias.data.zero_() - elif isinstance(module, nn.Embedding): - module.weight.data.normal_(mean=0.0, std=std) - if module.padding_idx is not None: - module.weight.data[module.padding_idx].zero_() - - -class DeepSeekV3Model(DeepSeekV3PreTrainedModel): - """DeepSeekV3 transformer decoder model.""" - - def __init__(self, config): - super().__init__(config) - self.config = config - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [ - DeepSeekV3DecoderLayer(config, layer_idx=idx) - for idx in range(config.num_hidden_layers) - ] - ) - self.norm = DeepSeekV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - self.post_init() - - def get_input_embeddings(self): - return self.embed_tokens - - def set_input_embeddings(self, value): - self.embed_tokens = value - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - **kwargs, - ) -> DeepSeekV3Output: - if input_ids is not None and inputs_embeds is not None: - raise ValueError("Cannot specify both input_ids and inputs_embeds") - elif input_ids is None and inputs_embeds is None: - raise ValueError("Must specify either input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - batch_size, seq_length = inputs_embeds.shape[:2] - - if position_ids is None: - device = input_ids.device if input_ids is not None else inputs_embeds.device - position_ids = torch.arange(seq_length, dtype=torch.long, device=device) - position_ids = position_ids.unsqueeze(0).expand(batch_size, -1) - - hidden_states = inputs_embeds - - for decoder_layer in self.layers: - hidden_states = decoder_layer(hidden_states, position_ids) - - hidden_states = self.norm(hidden_states) - - return DeepSeekV3Output(last_hidden_state=hidden_states) - - -class DeepSeekV3ForCausalLM(DeepSeekV3PreTrainedModel, GenerationMixin): - """DeepSeekV3 model with language modeling head.""" - - _tied_weights_keys = ["lm_head.weight"] - - def __init__(self, config): - super().__init__(config) - self.model = DeepSeekV3Model(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - self._register_load_state_dict_pre_hook( - partial( - mla_rope_utils._rope_deinterleave_load_hook, - qk_rope_head_dim=config.qk_rope_head_dim, - qk_nope_head_dim=config.qk_nope_head_dim, - num_heads=config.num_attention_heads, - kv_lora_rank=config.kv_lora_rank, - num_layers=config.num_hidden_layers, - ) - ) - - self.post_init() - - def get_input_embeddings(self): - return self.model.embed_tokens - - def set_input_embeddings(self, value): - self.model.embed_tokens = value - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - def get_decoder(self): - return self.model - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - **kwargs, - ) -> DeepSeekV3CausalLMOutput: - outputs = self.model( - input_ids=input_ids, - position_ids=position_ids, - inputs_embeds=inputs_embeds, - **kwargs, - ) - - hidden_states = outputs.last_hidden_state - logits = self.lm_head(hidden_states).float() - - return DeepSeekV3CausalLMOutput(logits=logits) - - -AutoModelForCausalLMFactory.register_custom_model_cls("DeepseekV3Config", DeepSeekV3ForCausalLM) diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h.py index 4a74f137c156..d03d3a5e1ad8 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h.py @@ -5,22 +5,30 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Slimmed down PyTorch NemotronH model implementation. - -Source: -https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/ -dbe2b5b379f25ec52223b39e2c097d3e9654b8db/modeling_nemotron_h.py - -This implementation differs from the original in the following ways: -* dependencies on custom kernel libraries (mamba_ssm, causal_conv1d, flash_attention_2) have been - removed. -* bugs in the original implementation have been fixed. -* cache-related code paths have been removed. -* training-related code paths have been removed. -* unnecessary fields in the output structure(s). - -This allows us to have a "pytorch" native reference implementation decoupled from bugs and -dependency issues in the source. +"""Sharding-aware NemotronH model for AutoDeploy IR sharding. + +Derived from ``modeling_nemotron_h.py`` with explicit sharding hints on the +shardable custom ops. The exported FX graph fully specifies how the model +should be sharded; ``apply_sharding_hints`` reads the hint kwargs together +with a runtime ``DistConfig`` to apply deterministic, node-local TP/EP +sharding. + +The router gate (``NemotronHTopkRouter``) is intentionally TP-replicated and +keeps the fused ``torch.ops.trtllm.{dsv3_router_gemm_op, noaux_tc_op}`` +kernel calls verbatim. AD has no transform that recovers the fused +``noaux_tc`` kernel from a vanilla rewrite, so the call site is preserved. + +Shardable custom ops used: + - ``torch.ops.auto_deploy.torch_linear_simple`` + (``tp_mode``, ``output_sizes``, ``tp_min_local_shape``, ``layer_type``) + - ``torch.ops.auto_deploy.view`` (``tp_scaled_dim``, ``layer_type``) + - ``torch.ops.auto_deploy.split_with_sizes`` (``enable_sharding``, ``layer_type``) + - ``torch.ops.auto_deploy.all_reduce`` (``layer_type``; identity until sharded) + - ``torch.ops.auto_deploy.torch_causal_conv1d`` (``enable_sharding``, ``output_sizes``, ``layer_type``) + - ``torch.ops.auto_deploy.torch_ssm`` (``enable_sharding``, ``layer_type``) + - ``torch.ops.auto_deploy.torch_rmsnorm_gated`` (``tp_mode``, ``layer_type``) + - ``torch.ops.auto_deploy.torch_moe`` (``layer_type``) + - ``torch.ops.auto_deploy.torch_attention`` (``layer_type`` metadata; sharding-invariant) """ import math @@ -36,8 +44,8 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput +from ... import custom_ops # noqa: F401 -- register all ops from ..._compat import ActivationType -from ...custom_ops.normalization.rms_norm import gated_rms_norm_ref from ..hf import AutoModelForCausalLMFactory @@ -49,14 +57,14 @@ def __init__(self, hidden_size, group_size, eps=1e-5): self.group_size = group_size def forward(self, hidden_states, gate=None): - return gated_rms_norm_ref( - x=hidden_states, - weight=self.weight, - bias=None, - z=gate, - eps=self.variance_epsilon, - group_size=self.group_size, - norm_before_gate=False, + return torch.ops.auto_deploy.torch_rmsnorm_gated( + hidden_states, + self.weight, + gate, + self.variance_epsilon, + self.group_size, + tp_mode="colwise", + layer_type="ssm", ) @@ -134,9 +142,29 @@ def torch_forward(self, input_states): dtype = input_states.dtype # 1. Gated MLP's linear projection - projected_states = self.in_proj(input_states) - gate, hidden_states_B_C, dt = projected_states.split( - [self.intermediate_size, self.conv_dim, self.num_heads], dim=-1 + # in_proj output = [gate | hidden | B | C | dt]; conv_dim = hidden + B + C. + # Pass head-aligned ``output_sizes`` so each rank ends up with its own + # head-aligned share of the fused weight. + projected_states = torch.ops.auto_deploy.torch_linear_simple( + input_states, + self.in_proj.weight, + self.in_proj.bias, + tp_mode="colwise", + output_sizes=[ + self.intermediate_size, + self.intermediate_size, + self.n_groups * self.ssm_state_size, + self.n_groups * self.ssm_state_size, + self.num_heads, + ], + layer_type="ssm", + ) + gate, hidden_states_B_C, dt = torch.ops.auto_deploy.split_with_sizes( + projected_states, + [self.intermediate_size, self.conv_dim, self.num_heads], + dim=-1, + enable_sharding=True, + layer_type="ssm", ) # 2. Convolution sequence transformation @@ -150,10 +178,17 @@ def torch_forward(self, input_states): self.conv1d.dilation[0], self.conv1d.groups, self.conv1d.padding_mode, + enable_sharding=True, + output_sizes=[ + self.intermediate_size, + self.n_groups * self.ssm_state_size, + self.n_groups * self.ssm_state_size, + ], + layer_type="ssm", ) ) - hidden_states, B, C = torch.split( + hidden_states, B, C = torch.ops.auto_deploy.split_with_sizes( hidden_states_B_C, [ self.intermediate_size, @@ -161,20 +196,39 @@ def torch_forward(self, input_states): self.n_groups * self.ssm_state_size, ], dim=-1, + enable_sharding=True, + layer_type="ssm", ) # 3. SSM transformation A = -torch.exp(self.A_log.float()) y = torch.ops.auto_deploy.torch_ssm( - hidden_states=hidden_states.view(batch_size, seq_len, -1, self.head_dim), + hidden_states=torch.ops.auto_deploy.view( + hidden_states, + [batch_size, seq_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="ssm", + ), A=A, - B=B.view(batch_size, seq_len, -1, self.ssm_state_size), - C=C.view(batch_size, seq_len, -1, self.ssm_state_size), + B=torch.ops.auto_deploy.view( + B, + [batch_size, seq_len, -1, self.ssm_state_size], + tp_scaled_dim=2, + layer_type="ssm", + ), + C=torch.ops.auto_deploy.view( + C, + [batch_size, seq_len, -1, self.ssm_state_size], + tp_scaled_dim=2, + layer_type="ssm", + ), D=self.D, dt=dt, dt_bias=self.dt_bias, time_step_limit=list(self.time_step_limit), chunk_size=self.chunk_size, + enable_sharding=True, + layer_type="ssm", ) y = y.reshape(batch_size, seq_len, -1) @@ -183,9 +237,16 @@ def torch_forward(self, input_states): # end ssd naive # 4. Final linear projection - contextualized_states = self.out_proj( - scan_output.to(dtype) + contextualized_states = torch.ops.auto_deploy.torch_linear_simple( + scan_output.to(dtype), + self.out_proj.weight, + self.out_proj.bias, + tp_mode="rowwise", + layer_type="ssm", ) # [batch, seq_len, hidden_size] + contextualized_states = torch.ops.auto_deploy.all_reduce( + contextualized_states, layer_type="ssm" + ) return contextualized_states def forward(self, hidden_states): @@ -250,6 +311,8 @@ def __init__( layer_idx: int, intermediate_size: Optional[int] = None, is_expert: bool = False, + add_all_reduce: bool = True, + sharding_layer_type: str = "mlp", ): super().__init__() self.config = config @@ -262,9 +325,32 @@ def __init__( self.up_proj = nn.Linear(input_size, self.intermediate_size, bias=config.mlp_bias) self.down_proj = nn.Linear(self.intermediate_size, input_size, bias=config.mlp_bias) self.act_fn = ACT2FN[config.mlp_hidden_act] + # When this MLP is used as a routed expert / shared-expert sub-module + # inside ``NemotronHMOE``, the merge-point ``all_reduce`` is emitted once + # after ``routed + shared`` instead of per sub-MLP. Stand-alone MLP + # blocks still emit their own ``all_reduce``. + self.add_all_reduce = add_all_reduce + self.sharding_layer_type = sharding_layer_type def forward(self, x): - return self.down_proj(self.act_fn(self.up_proj(x))) + _lt = self.sharding_layer_type + up = torch.ops.auto_deploy.torch_linear_simple( + x, + self.up_proj.weight, + self.up_proj.bias, + tp_mode="colwise", + layer_type=_lt, + ) + down = torch.ops.auto_deploy.torch_linear_simple( + self.act_fn(up), + self.down_proj.weight, + self.down_proj.bias, + tp_mode="rowwise", + layer_type=_lt, + ) + if self.add_all_reduce: + down = torch.ops.auto_deploy.all_reduce(down, layer_type=_lt) + return down class NemotronHMOE(nn.Module): @@ -278,6 +364,8 @@ def __init__(self, config, layer_idx: Optional[int] = None): layer_idx=layer_idx, intermediate_size=config.moe_intermediate_size, is_expert=True, + add_all_reduce=False, + sharding_layer_type="moe", ) for _ in range(config.n_routed_experts) ] @@ -288,6 +376,8 @@ def __init__(self, config, layer_idx: Optional[int] = None): intermediate_size=config.moe_shared_expert_intermediate_size, layer_idx=layer_idx, is_expert=False, + add_all_reduce=False, + sharding_layer_type="moe", ) # Add latent projections when using latent MoE (required for SuperV3) if getattr(config, "moe_latent_size", None) is not None: @@ -329,6 +419,7 @@ def forward(self, hidden_states: torch.Tensor): w3_weight=[], act_fn=ActivationType.Relu2, is_gated_mlp=False, + layer_type="moe", ) if has_latent_proj: @@ -337,6 +428,9 @@ def forward(self, hidden_states: torch.Tensor): routed_out = out_flat.view(*orig_shape) out = shared_out + routed_out + # Single merge-point all_reduce for routed + shared, per the IR sharding + # rule: one reduction at the merge of parallel branches. + out = torch.ops.auto_deploy.all_reduce(out, layer_type="moe") return out @@ -445,13 +539,54 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: bsz, q_len, _ = hidden_states.size() - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) + # Q/K/V are colwise linears with layer_type="mha". For GQA, K/V use + # ``tp_min_local_shape=self.head_dim`` so each rank keeps at least one + # full head when ``num_kv_heads < tp_size``. + query_states = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.q_proj.weight, + self.q_proj.bias, + tp_mode="colwise", + layer_type="mha", + ) + key_states = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.k_proj.weight, + self.k_proj.bias, + tp_mode="colwise", + tp_min_local_shape=self.head_dim, + layer_type="mha", + ) + value_states = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.v_proj.weight, + self.v_proj.bias, + tp_mode="colwise", + tp_min_local_shape=self.head_dim, + layer_type="mha", + ) - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) + # Use the sharding-aware view so the head-count dim is rescaled + # per-rank under TP. We do NOT manually repeat K/V heads for GQA -- + # ``torch_attention`` handles it. + query_states = torch.ops.auto_deploy.view( + query_states, + [bsz, q_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="mha", + ) + key_states = torch.ops.auto_deploy.view( + key_states, + [bsz, q_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="mha", + ) + value_states = torch.ops.auto_deploy.view( + value_states, + [bsz, q_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="mha", + ) attn_output = torch.ops.auto_deploy.torch_attention( query_states, @@ -462,10 +597,26 @@ def forward( dropout_p=0.0, is_causal=True, layout="bsnd", + layer_type="mha", + ) + # Reshape [bsz, q_len, n_heads, head_dim] -> [bsz, q_len, n_heads * head_dim]. + # Under TP, ``apply_sharding_hints`` divides shape[2] by tp_size so each + # rank reshapes its local ``n_heads_local * head_dim`` activations. + attn_output = torch.ops.auto_deploy.view( + attn_output, + [bsz, q_len, self.num_heads * self.head_dim], + tp_scaled_dim=2, + layer_type="mha", ) - attn_output = attn_output.view(bsz, q_len, self.num_heads * self.head_dim) - attn_output = self.o_proj(attn_output) + attn_output = torch.ops.auto_deploy.torch_linear_simple( + attn_output, + self.o_proj.weight, + self.o_proj.bias, + tp_mode="rowwise", + layer_type="mha", + ) + attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mha") return attn_output diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h_ir.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h_ir.py deleted file mode 100644 index d8a285ee5e95..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_h_ir.py +++ /dev/null @@ -1,827 +0,0 @@ -# Copyright 2018 The HuggingFace Team -# Licensed under the Apache License, Version 2.0. -# Original source: https://github.com/huggingface/transformers -# -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sharding-aware NemotronH model for AutoDeploy IR sharding. - -Derived from modeling_nemotron_h.py with explicit sharding hints on all -custom ops. ``apply_sharding_hints`` reads these hints to apply -deterministic, node-local TP/EP sharding. - -Shardable custom ops used: - - torch.ops.auto_deploy.torch_linear_simple (tp_mode, output_sizes, layer_type) - - torch.ops.auto_deploy.view (tp_scaled_dim, layer_type) - - torch.ops.auto_deploy.split_with_sizes (enable_sharding, layer_type) - - torch.ops.auto_deploy.all_reduce (layer_type) - - torch.ops.auto_deploy.torch_causal_conv1d (enable_sharding, layer_type) - - torch.ops.auto_deploy.torch_ssm (enable_sharding, layer_type) - - torch.ops.auto_deploy.torch_rmsnorm_gated (tp_mode, layer_type) - - torch.ops.auto_deploy.torch_moe (layer_type) - - torch.ops.auto_deploy.torch_attention (sharding-invariant, no hints needed) -""" - -import math -from dataclasses import dataclass -from typing import Optional, Tuple, Union - -import torch -import torch.nn.functional as F -import torch.utils.checkpoint -from torch import nn -from transformers.activations import ACT2FN -from transformers.generation import GenerationMixin -from transformers.modeling_utils import PreTrainedModel -from transformers.utils import ModelOutput - -from tensorrt_llm._torch.utils import ActivationType - -from ... import custom_ops # noqa: F401, I001 -- register all ops -from ..hf import AutoModelForCausalLMFactory - - -class MambaRMSNormGated(torch.nn.Module): - def __init__(self, hidden_size, group_size, eps=1e-5): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - self.group_size = group_size - - def forward(self, hidden_states, gate=None): - return torch.ops.auto_deploy.torch_rmsnorm_gated( - hidden_states, - self.weight, - gate, - self.variance_epsilon, - self.group_size, - tp_mode="colwise", - layer_type="ssm", - ) - - -class NemotronHMamba2Mixer(nn.Module): - """ - Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. - A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) - ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, - and is why Mamba is called **selective** state spaces) - """ - - def __init__(self, config, layer_idx: int): - super().__init__() - self.num_heads = config.mamba_num_heads - self.hidden_size = config.hidden_size - self.ssm_state_size = config.ssm_state_size - self.conv_kernel_size = config.conv_kernel - self.intermediate_size = config.mamba_num_heads * config.mamba_head_dim - self.layer_idx = layer_idx - self.use_conv_bias = config.use_conv_bias - self.activation = config.mamba_hidden_act - self.act = ACT2FN[config.mamba_hidden_act] - - self.layer_norm_epsilon = config.layer_norm_epsilon - - self.n_groups = config.n_groups - self.head_dim = config.mamba_head_dim - self.chunk_size = config.chunk_size - - self.time_step_limit = config.time_step_limit - self.time_step_min = config.time_step_min - self.time_step_max = config.time_step_max - - self.conv_dim = self.intermediate_size + 2 * self.n_groups * self.ssm_state_size - self.conv1d = nn.Conv1d( - in_channels=self.conv_dim, - out_channels=self.conv_dim, - bias=config.use_conv_bias, - kernel_size=config.conv_kernel, - groups=self.conv_dim, - padding=config.conv_kernel - 1, - ) - - projection_size = self.intermediate_size + self.conv_dim + self.num_heads - self.in_proj = nn.Linear( - self.hidden_size, - projection_size, - bias=config.use_bias, - ) - - self.dt_bias = nn.Parameter(torch.ones(self.num_heads)) - - A = torch.arange(1, self.num_heads + 1) - self.A_log = nn.Parameter(torch.log(A)) - self.A_log._no_weight_decay = True - self.norm = MambaRMSNormGated( - self.intermediate_size, - eps=self.layer_norm_epsilon, - group_size=self.intermediate_size // self.n_groups, - ) - self.D = nn.Parameter(torch.ones(self.num_heads)) - self.D._no_weight_decay = True - - self.out_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.use_bias) - self.use_bias = config.use_bias - - # Fused output sizes for in_proj sharding: - # in_proj output = [gate | hidden | B | C | dt] - self._in_proj_output_sizes = [ - self.intermediate_size, - self.intermediate_size, - self.n_groups * self.ssm_state_size, - self.n_groups * self.ssm_state_size, - self.num_heads, - ] - - # Fused output sizes for conv1d sharding: - # conv_dim = [hidden | B | C] - self._conv1d_output_sizes = [ - self.intermediate_size, - self.n_groups * self.ssm_state_size, - self.n_groups * self.ssm_state_size, - ] - - def torch_forward(self, input_states): - batch_size, seq_len, _ = input_states.shape - dtype = input_states.dtype - - # 1. Gated MLP's linear projection (colwise with fused output sizes) - projected_states = torch.ops.auto_deploy.torch_linear_simple( - input_states, - self.in_proj.weight, - self.in_proj.bias, - tp_mode="colwise", - output_sizes=self._in_proj_output_sizes, - layer_type="ssm", - ) - gate, hidden_states_B_C, dt = torch.ops.auto_deploy.split_with_sizes( - projected_states, - [self.intermediate_size, self.conv_dim, self.num_heads], - dim=-1, - enable_sharding=True, - layer_type="ssm", - ) - - # 2. Convolution sequence transformation - hidden_states_B_C = self.act( - torch.ops.auto_deploy.torch_causal_conv1d( - hidden_states_B_C, - self.conv1d.weight, - self.conv1d.bias, - self.conv1d.stride[0], - self.conv1d.padding[0], - self.conv1d.dilation[0], - self.conv1d.groups, - self.conv1d.padding_mode, - enable_sharding=True, - output_sizes=self._conv1d_output_sizes, - layer_type="ssm", - ) - ) - - hidden_states, B, C = torch.ops.auto_deploy.split_with_sizes( - hidden_states_B_C, - [ - self.intermediate_size, - self.n_groups * self.ssm_state_size, - self.n_groups * self.ssm_state_size, - ], - dim=-1, - enable_sharding=True, - layer_type="ssm", - ) - - # 3. SSM transformation - A = -torch.exp(self.A_log.float()) - y = torch.ops.auto_deploy.torch_ssm( - hidden_states=torch.ops.auto_deploy.view( - hidden_states, - [batch_size, seq_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="ssm", - ), - A=A, - B=torch.ops.auto_deploy.view( - B, - [batch_size, seq_len, -1, self.ssm_state_size], - tp_scaled_dim=2, - layer_type="ssm", - ), - C=torch.ops.auto_deploy.view( - C, - [batch_size, seq_len, -1, self.ssm_state_size], - tp_scaled_dim=2, - layer_type="ssm", - ), - D=self.D, - dt=dt, - dt_bias=self.dt_bias, - time_step_limit=list(self.time_step_limit), - chunk_size=self.chunk_size, - enable_sharding=True, - layer_type="ssm", - ) - y = y.reshape(batch_size, seq_len, -1) - - scan_output = self.norm(y, gate) - - # 4. Final linear projection (rowwise) + all_reduce - contextualized_states = torch.ops.auto_deploy.torch_linear_simple( - scan_output.to(dtype), - self.out_proj.weight, - self.out_proj.bias, - tp_mode="rowwise", - layer_type="ssm", - ) - contextualized_states = torch.ops.auto_deploy.all_reduce( - contextualized_states, layer_type="ssm" - ) - return contextualized_states - - def forward(self, hidden_states): - return self.torch_forward(hidden_states) - - -class NemotronHRMSNorm(nn.Module): - def __init__(self, hidden_size, eps=1e-6): - """ - NemotronHRMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states): - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - # Weights are in float32 - return (self.weight.to(torch.float32) * hidden_states).to(input_dtype) - - -class NemotronHBlock(nn.Module): - def __init__(self, config, layer_idx): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.residual_in_fp32 = config.residual_in_fp32 - self.norm = NemotronHRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - - # M: Mamba2, *: Attention, -: MLP - self.block_type = config.layers_block_type[layer_idx] - if self.block_type == "mamba": - self.mixer = NemotronHMamba2Mixer(config, layer_idx=layer_idx) - elif self.block_type == "attention": - self.mixer = NemotronHAttention(config, layer_idx=layer_idx) - elif self.block_type == "mlp": - self.mixer = NemotronHMLP(config, layer_idx=layer_idx) - elif self.block_type == "moe": - self.mixer = NemotronHMOE(config, layer_idx=layer_idx) - else: - raise ValueError(f"Invalid layer pattern {config.hybrid_override_pattern[layer_idx]}") - - def forward(self, hidden_states): - residual = hidden_states - hidden_states = self.norm(hidden_states.to(dtype=self.norm.weight.dtype)) - if self.residual_in_fp32: - residual = residual.to(torch.float32) - - hidden_states = self.mixer(hidden_states) - hidden_states = residual + hidden_states - return hidden_states - - -class NemotronHMLP(nn.Module): - def __init__( - self, - config, - layer_idx: int, - intermediate_size: Optional[int] = None, - is_expert: bool = False, - add_all_reduce: bool = True, - sharding_layer_type: str = "mlp", - ): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.hidden_size = config.hidden_size - self.intermediate_size = intermediate_size or config.intermediate_size - use_latent_size = (getattr(self.config, "moe_latent_size", None) is not None) and is_expert - input_size = self.config.moe_latent_size if use_latent_size else self.hidden_size - self.up_proj = nn.Linear(input_size, self.intermediate_size, bias=config.mlp_bias) - self.down_proj = nn.Linear(self.intermediate_size, input_size, bias=config.mlp_bias) - self.act_fn = ACT2FN[config.mlp_hidden_act] - self.add_all_reduce = add_all_reduce - self.sharding_layer_type = sharding_layer_type - - def forward(self, x): - _lt = self.sharding_layer_type - up = torch.ops.auto_deploy.torch_linear_simple( - x, - self.up_proj.weight, - self.up_proj.bias, - tp_mode="colwise", - layer_type=_lt, - ) - down = torch.ops.auto_deploy.torch_linear_simple( - self.act_fn(up), - self.down_proj.weight, - self.down_proj.bias, - tp_mode="rowwise", - layer_type=_lt, - ) - if self.add_all_reduce: - down = torch.ops.auto_deploy.all_reduce(down, layer_type=_lt) - return down - - -class NemotronHMOE(nn.Module): - def __init__(self, config, layer_idx: Optional[int] = None): - super().__init__() - self.config = config - self.experts = nn.ModuleList( - [ - NemotronHMLP( - config, - layer_idx=layer_idx, - intermediate_size=config.moe_intermediate_size, - is_expert=True, - add_all_reduce=False, - sharding_layer_type="moe", - ) - for _ in range(config.n_routed_experts) - ] - ) - self.gate = NemotronHTopkRouter(config) - self.shared_experts = NemotronHMLP( - config=config, - intermediate_size=config.moe_shared_expert_intermediate_size, - layer_idx=layer_idx, - is_expert=False, - add_all_reduce=False, - sharding_layer_type="moe", - ) - if getattr(config, "moe_latent_size", None) is not None: - self.fc1_latent_proj = nn.Linear( - config.hidden_size, config.moe_latent_size, bias=config.mlp_bias - ) - self.fc2_latent_proj = nn.Linear( - config.moe_latent_size, config.hidden_size, bias=config.mlp_bias - ) - else: - self.fc1_latent_proj = nn.Identity() - self.fc2_latent_proj = nn.Identity() - - def forward(self, hidden_states: torch.Tensor): - residuals = hidden_states - orig_shape = hidden_states.shape - topk_indices, topk_weights = self.gate(hidden_states) - x_flat = hidden_states.view(-1, hidden_states.shape[-1]) - - # Shared expert first (dispatch order matches exported graph node order) - shared_out = self.shared_experts(residuals) - - has_latent_proj = hasattr(self, "fc1_latent_proj") and hasattr(self, "fc2_latent_proj") - - if has_latent_proj: - x_flat = self.fc1_latent_proj(x_flat) - - out_flat = torch.ops.auto_deploy.torch_moe( - x_flat, - topk_indices, - topk_weights, - w1_weight=[e.up_proj.weight for e in self.experts], - w2_weight=[e.down_proj.weight for e in self.experts], - w3_weight=[], - act_fn=ActivationType.Relu2, - is_gated_mlp=False, - layer_type="moe", - ) - - if has_latent_proj: - out_flat = self.fc2_latent_proj(out_flat) - - routed_out = out_flat.view(*orig_shape) - out = shared_out + routed_out - out = torch.ops.auto_deploy.all_reduce(out, layer_type="moe") - return out - - -class NemotronHTopkRouter(nn.Module): - def __init__(self, config): - super().__init__() - self.config = config - self.top_k = config.num_experts_per_tok - self.n_routed_experts = config.n_routed_experts - self.routed_scaling_factor = config.routed_scaling_factor - self.n_group = config.n_group - self.topk_group = config.topk_group - self.norm_topk_prob = config.norm_topk_prob - - self.weight = nn.Parameter(torch.empty((self.n_routed_experts, config.hidden_size))) - self.register_buffer( - "e_score_correction_bias", torch.zeros(self.n_routed_experts, dtype=torch.float32) - ) - - def forward(self, hidden_states): - hidden_states = hidden_states.view(-1, self.config.hidden_size) - if self.weight.dtype == torch.float32: - router_logits = F.linear(hidden_states.type(torch.float32), self.weight) - else: - router_logits = torch.ops.trtllm.dsv3_router_gemm_op( - hidden_states, self.weight.t(), bias=None, out_dtype=torch.float32 - ) - - topk_weights, topk_indices = torch.ops.trtllm.noaux_tc_op( - router_logits, - self.e_score_correction_bias, - self.n_group, - self.topk_group, - self.top_k, - self.routed_scaling_factor, - ) - - return topk_indices, topk_weights - - -class NemotronHAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config, layer_idx: Optional[int] = None): - super().__init__() - self.config = config - self.layer_idx = layer_idx - if layer_idx is None: - raise ValueError("Please make sure to provide a `layer_idx` when creating this class.") - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - - if hasattr(config, "head_dim"): - head_dim = config.head_dim - elif hasattr(config, "attention_head_dim"): - head_dim = config.attention_head_dim - else: - raise AttributeError( - "Expected either `head_dim` or `attention_head_dim` to be present in the config " - "class, found neither." - ) - - if head_dim is not None: - self.head_dim = head_dim - else: - self.head_dim = config.hidden_size // config.num_attention_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.is_causal = True - - self.q_proj = nn.Linear( - self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias - ) - self.k_proj = nn.Linear( - self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.v_proj = nn.Linear( - self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.o_proj = nn.Linear( - self.head_dim * self.num_heads, self.hidden_size, bias=config.attention_bias - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - bsz, q_len, _ = hidden_states.size() - - query_states = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.q_proj.weight, - self.q_proj.bias, - tp_mode="colwise", - layer_type="mha", - ) - key_states = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.k_proj.weight, - self.k_proj.bias, - tp_mode="colwise", - tp_min_local_shape=self.head_dim, - layer_type="mha", - ) - value_states = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.v_proj.weight, - self.v_proj.bias, - tp_mode="colwise", - tp_min_local_shape=self.head_dim, - layer_type="mha", - ) - - query_states = torch.ops.auto_deploy.view( - query_states, - [bsz, q_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="mha", - ) - key_states = torch.ops.auto_deploy.view( - key_states, - [bsz, q_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="mha", - ) - value_states = torch.ops.auto_deploy.view( - value_states, - [bsz, q_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="mha", - ) - - attn_output = torch.ops.auto_deploy.torch_attention( - query_states, - key_states, - value_states, - attn_mask=None, - dropout_p=0.0, - is_causal=True, - layout="bsnd", - ) - attn_output = attn_output.view(bsz, q_len, -1) - - attn_output = torch.ops.auto_deploy.torch_linear_simple( - attn_output, - self.o_proj.weight, - self.o_proj.bias, - tp_mode="rowwise", - layer_type="mha", - ) - attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mha") - - return attn_output - - -class NemotronHPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - - base_model_prefix = "backbone" - _no_split_modules = ["NemotronHBlock"] - supports_gradient_checkpointing = True - _is_stateful = True - - def _init_weights(self, module): - """Initialize the weights.""" - if isinstance(module, NemotronHMamba2Mixer): - module.A_log._no_weight_decay = True - module.D._no_weight_decay = True - - dt = torch.exp( - torch.rand(self.config.mamba_num_heads) - * (math.log(self.config.time_step_max) - math.log(self.config.time_step_min)) - + math.log(self.config.time_step_min) - ).clamp(min=self.config.time_step_floor) - - inv_dt = dt + torch.log(-torch.expm1(-dt)) - with torch.no_grad(): - module.dt_bias.copy_(inv_dt) - module.dt_bias._no_reinit = True - - if isinstance(module, nn.Linear): - if module.bias is not None: - if not getattr(module.bias, "_no_reinit", False): - nn.init.zeros_(module.bias) - elif isinstance(module, nn.Embedding): - nn.init.normal_(module.weight, std=self.config.initializer_range) - - if self.config.rescale_prenorm_residual: - for name, p in module.named_parameters(): - if name in ["out_proj.weight"]: - nn.init.kaiming_uniform_(p, a=math.sqrt(5)) - with torch.no_grad(): - p /= math.sqrt(self.config.num_hidden_layers) - - -@dataclass -class NemotronHOutput(ModelOutput): - """ - Class for the NemotronH model outputs. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - - -@dataclass -class NemotronHCausalLMOutput(ModelOutput): - """ - Base class for causal language model (or autoregressive) outputs. - - Args: - logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): - Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). - """ - - logits: Optional[torch.FloatTensor] = None - - -class NemotronHModel(NemotronHPreTrainedModel): - def __init__(self, config): - super().__init__(config) - - self.embeddings = nn.Embedding(config.vocab_size, config.hidden_size) - self.layers = nn.ModuleList( - [NemotronHBlock(config, layer_idx=idx) for idx in range(config.num_hidden_layers)] - ) - - self.norm_f = NemotronHRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self._register_load_state_dict_pre_hook(self.load_hook) - self.post_init() - - def load_hook(self, state_dict, prefix, *args): - for k in state_dict: - if "embedding." in k: - state_dict[k.replace("embedding.", "embeddings.")] = state_dict.pop(k) - break - - def get_input_embeddings(self): - return self.embeddings - - def set_input_embeddings(self, new_embeddings): - self.embeddings = new_embeddings - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.LongTensor] = None, - **kwargs, - ) -> Union[Tuple, NemotronHOutput]: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.embeddings(input_ids) - - hidden_states = inputs_embeds - - for mixer_block in self.layers: - hidden_states = mixer_block(hidden_states) - - hidden_states = self.norm_f(hidden_states) - - return NemotronHOutput(last_hidden_state=hidden_states) - - -class NemotronHForCausalLM(NemotronHPreTrainedModel, GenerationMixin): - _tied_weights_keys = ["lm_head.weight"] - - def __init__(self, config): - super().__init__(config) - self.backbone = NemotronHModel(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - self.post_init() - - def get_input_embeddings(self): - return self.backbone.get_input_embeddings() - - def set_input_embeddings(self, new_embeddings): - return self.backbone.set_input_embeddings(new_embeddings) - - def get_final_normalization(self): - return self.backbone.norm_f - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - **kwargs, - ) -> Union[Tuple, NemotronHCausalLMOutput]: - nemotron_h_outputs = self.backbone(input_ids, inputs_embeds=inputs_embeds) - hidden_states = nemotron_h_outputs[0] - - logits = self.lm_head(hidden_states.to(self.lm_head.weight.dtype)).float() - - return NemotronHCausalLMOutput(logits) - - -AutoModelForCausalLMFactory.register_custom_model_cls("NemotronHConfig", NemotronHForCausalLM) - - -# ============================================================================= -# Eagle Layer Builder for NemotronH MTP (Multi-Token Prediction) -# ============================================================================= - - -class NemotronHEagleLayer(nn.Module): - """Eagle layer for NemotronH models. - - NemotronH does not use RoPE, so position_ids is accepted but ignored. - The layer implements the MTP (Multi-Token Prediction) architecture: - - First layer fuses embeds + hidden_states via start projections (enorm, hnorm, eh_proj) - - All layers have pre-norm residual block with mixer (Attention or MoE) - - Last layer applies final_layernorm - - Supported layers are * (Attention with start projections) and E (MoE with final_layernorm) - """ - - def __init__( - self, - config, - layer_idx: int, - layer_type: str, - has_start_projections: bool, - has_end_norm: bool, - ): - super().__init__() - eps = getattr(config, "layer_norm_epsilon") - if eps is None: - raise ValueError("layer_norm_epsilon is not set in the config") - self.residual_in_fp32 = config.residual_in_fp32 - self.has_start_projections = has_start_projections - self.has_end_norm = has_end_norm - - if has_start_projections: - self.enorm = NemotronHRMSNorm(config.hidden_size, eps=eps) - self.hnorm = NemotronHRMSNorm(config.hidden_size, eps=eps) - self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) - - self.norm = NemotronHRMSNorm(config.hidden_size, eps=eps) - - if layer_type == "*": - self.mixer = NemotronHAttention(config, layer_idx=layer_idx) - elif layer_type == "E": - self.mixer = NemotronHMOE(config, layer_idx=layer_idx) - else: - raise ValueError( - f"Unsupported MTP layer type in NemotronHEagleLayer. Only * and E are currently supported." - f"Layer type: {layer_type}" - ) - - if has_end_norm: - self.final_layernorm = NemotronHRMSNorm(config.hidden_size, eps=eps) - - def forward( - self, - hidden_states: torch.Tensor, - inputs_embeds: torch.Tensor, - position_ids: torch.LongTensor, - ) -> torch.Tensor: - if self.has_start_projections: - e_normed = self.enorm(inputs_embeds) - h_normed = self.hnorm(hidden_states) - hidden_states = self.eh_proj(torch.cat([e_normed, h_normed], dim=-1)) - - residual = hidden_states - hidden_states = self.norm(hidden_states.to(dtype=self.norm.weight.dtype)) - if self.residual_in_fp32: - residual = residual.to(torch.float32) - hidden_states = self.mixer(hidden_states) - hidden_states = residual + hidden_states - - if self.has_end_norm: - hidden_states = self.final_layernorm(hidden_states) - - return hidden_states - - -def build_nemotron_eagle_layers(config) -> list[nn.Module]: - """Build NemotronH MTP layers for Eagle drafter.""" - pattern = getattr(config, "mtp_hybrid_override_pattern", None) - if pattern is None: - raise ValueError("mtp_hybrid_override_pattern is not set in the config") - - return [ - NemotronHEagleLayer( - config, - layer_idx=i, - layer_type=char, - has_start_projections=(i == 0), - has_end_norm=(i == len(pattern) - 1), - ) - for i, char in enumerate(pattern) - ] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_ir.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py similarity index 80% rename from tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_ir.py rename to tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py index 5297b4e59323..37fda3c1bcd5 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_ir.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py @@ -17,13 +17,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Qwen3 model with explicit sharding hint ops. +"""Qwen3 (dense) model with explicit sharding hint ops. -This is a rewrite of modeling_qwen3.py where all sharding-enabled operations use -AutoDeploy custom ops with sharding hint kwargs. The graph produced by this -model is a complete, self-contained specification of how this model should be -sharded. The ``apply_sharding_hints`` transform reads the hints together with a -runtime ``DistConfig`` to apply deterministic, node-local sharding. +This is a sharding-aware rewrite of ``modeling_qwen3.py``: every shardable op +uses an AutoDeploy custom op with explicit sharding hint kwargs. The exported +FX graph is therefore a complete, self-contained specification of how the model +should be sharded under tensor parallelism. The ``apply_sharding_hints`` +transform reads the hints together with a runtime ``DistConfig`` to apply +deterministic, node-local sharding. + +Source of truth for model logic: +https://huggingface.co/Qwen/Qwen3-0.6B (and the rest of the Qwen3 dense family). + +Differences from the original HuggingFace version: +* Simplified for prefill-only inference (no KV caching) +* Uses auto_deploy custom ops for export compatibility +* Removed flash attention variants (uses torch_attention custom op) +* Removed gradient checkpointing and training code paths +* Removed attention dropout (inference only) +* Removed sliding window attention (not needed for prefill-only) + +The Qwen3 model uses Grouped Query Attention (GQA) with per-head Q/K normalization +(RMSNorm on head_dim), which distinguishes it from standard Llama-style models. Shardable custom ops used: - torch.ops.auto_deploy.torch_linear_simple (tp_mode, tp_min_local_shape, layer_type) @@ -83,6 +98,7 @@ def __init__( inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) + # Build cos/sin cache with AD-specific naming self._set_cos_sin_cache(max_position_embeddings) def _set_cos_sin_cache(self, seq_len: int): @@ -90,19 +106,21 @@ def _set_cos_sin_cache(self, seq_len: int): t = torch.arange(seq_len, dtype=self.inv_freq.dtype) freqs = torch.outer(t, self.inv_freq) emb = torch.cat((freqs, freqs), dim=-1) + # Use _ad_ prefix for AutoDeploy compatibility with lift_to_meta self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + # Slice cos/sin by position_ids here (once) instead of in every attention layer cos = self._ad_cos_cached.to(dtype=x.dtype, device=x.device) sin = self._ad_sin_cached.to(dtype=x.dtype, device=x.device) return cos[position_ids], sin[position_ids] class Qwen3MLP(nn.Module): - """MLP layer for Qwen3 (SwiGLU) with sharding hints. + """MLP layer for Qwen3 (SwiGLU activation). Sharding strategy: gate_proj -> colwise @@ -147,7 +165,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class Qwen3Attention(nn.Module): - """Grouped Query Attention for Qwen3 with per-head Q/K normalization and sharding hints. + """Grouped Query Attention for Qwen3 with per-head Q/K normalization. + + Qwen3 applies RMSNorm to query and key states after projection and reshaping, + but before RoPE application. This per-head normalization on head_dim is a key + architectural difference from standard Llama-style models. Sharding strategy: q_proj -> colwise (+ tp_min_local_shape for GQA) @@ -170,6 +192,7 @@ def __init__(self, config: Qwen3Config, layer_idx: Optional[int] = None): ) self.scaling = self.head_dim ** (-0.5) + # Q/K/V/O projections self.q_proj = nn.Linear( self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias ) @@ -183,6 +206,7 @@ def __init__(self, config: Qwen3Config, layer_idx: Optional[int] = None): self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias ) + # Per-head Q/K normalization (unique to Qwen3) self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) @@ -193,6 +217,7 @@ def forward( ) -> torch.Tensor: bsz, q_len, _ = hidden_states.size() + # Project Q/K/V and reshape to [B, S, N, head_dim] (BSND layout) q = torch.ops.auto_deploy.torch_linear_simple( hidden_states, self.q_proj.weight, @@ -217,7 +242,6 @@ def forward( tp_min_local_shape=self.head_dim, layer_type="mha", ) - q = torch.ops.auto_deploy.view( q, [bsz, q_len, self.num_heads, self.head_dim], @@ -237,40 +261,44 @@ def forward( layer_type="mha", ) + # Apply per-head Q/K normalization (Qwen3-specific, on head_dim dimension) q = self.q_norm(q) k = self.k_norm(k) - cos, sin = position_embeddings + # Get pre-sliced cos/sin from position_embeddings (already indexed by position_ids) + cos, sin = position_embeddings # [B, S, head_dim] + # Apply RoPE using custom op (BSND layout, unsqueeze_dim=2) q, k = torch.ops.auto_deploy.torch_rope_with_explicit_cos_sin( q, k, cos, sin, - 2, + 2, # unsqueeze_dim=2 for BSND layout ) + # Attention using custom op with GQA support (BSND layout) attn_output = torch.ops.auto_deploy.torch_attention( - q, - k, - v, - None, - 0.0, - True, - self.scaling, - None, - None, - None, - "bsnd", + q, # [B, S, N, head_dim] + k, # [B, S, N_kv, head_dim] + v, # [B, S, N_kv, head_dim] + None, # attn_mask + 0.0, # dropout_p + True, # is_causal + self.scaling, # scale + None, # sinks + None, # sliding_window + None, # logit_cap + "bsnd", # layout ) + # Reshape [B, S, N, head_dim] -> [B, S, N * head_dim] and project attn_output = torch.ops.auto_deploy.view( attn_output, [bsz, q_len, self.num_heads * self.head_dim], tp_scaled_dim=2, layer_type="mha", ) - attn_output = torch.ops.auto_deploy.torch_linear_simple( attn_output, self.o_proj.weight, @@ -300,11 +328,13 @@ def forward( hidden_states: torch.Tensor, position_embeddings: Tuple[torch.Tensor, torch.Tensor], ) -> torch.Tensor: + # Self attention residual = hidden_states hidden_states = self.input_layernorm(hidden_states) hidden_states = self.self_attn(hidden_states, position_embeddings) hidden_states = residual + hidden_states + # MLP residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) @@ -362,11 +392,17 @@ def __init__(self, config: Qwen3Config): ) self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + # Shared rotary embedding at model level. In transformers 4.x rope_theta + # sits on the config directly; in transformers 5.x it has been moved + # under ``rope_scaling`` (alongside ``rope_type``). Handle both layouts. head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + rope_theta = getattr(config, "rope_theta", None) + if rope_theta is None: + rope_theta = (config.rope_scaling or {}).get("rope_theta", 10000.0) self.rotary_emb = Qwen3RotaryEmbedding( head_dim, max_position_embeddings=config.max_position_embeddings, - base=config.rope_theta, + base=rope_theta, ) self.post_init() @@ -394,8 +430,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) + # Cast to compute dtype (e.g., bfloat16) for FP8 models where embedding + # output may be FP8 but downstream ops (RMSNorm, attention) require FP16/BF16 inputs_embeds = inputs_embeds.to(self.norm.weight.dtype) + # Compute position embeddings once (sliced by position_ids in RoPE) position_embeddings = self.rotary_emb(inputs_embeds, position_ids) hidden_states = inputs_embeds @@ -411,7 +450,7 @@ def forward( class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): """Qwen3 model with language modeling head.""" - _tied_weights_keys = ["lm_head.weight"] + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} def __init__(self, config, **kwargs): super().__init__(config) @@ -457,4 +496,5 @@ def forward( return Qwen3CausalLMOutput(logits=logits) +# Register with AutoModelForCausalLMFactory AutoModelForCausalLMFactory.register_custom_model_cls("Qwen3Config", Qwen3ForCausalLM) diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py index cf402157f315..b53170f46470 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Qwen3.5 MoE model for auto_deploy (text + vision). +"""Sharding-aware Qwen3.5 MoE model for AutoDeploy IR sharding (text + vision). Reference HF modeling file (not yet in a released transformers version): transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py @@ -45,6 +45,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput +from ... import custom_ops # noqa: F401 -- register all ops from ...custom_ops.attention_interface import BatchInfo from ..factory import ModelFactoryRegistry from ..hf import ( @@ -353,10 +354,35 @@ def __init__(self, config: Qwen3_5MoeTextConfig, layer_idx: int): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: batch_size, seq_len, _ = hidden_states.shape # 1. Projections (separate, unlike Qwen3Next which uses combined in_proj_qkvz) - mixed_qkv = self.in_proj_qkv(hidden_states) # [B, S, conv_dim] - z = self.in_proj_z(hidden_states) # [B, S, value_dim] - b = self.in_proj_b(hidden_states) # [B, S, num_v_heads] - a = self.in_proj_a(hidden_states) # [B, S, num_v_heads] + mixed_qkv = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.in_proj_qkv.weight, + self.in_proj_qkv.bias, + tp_mode="colwise", + output_sizes=[self.key_dim, self.key_dim, self.value_dim], + layer_type="delta", + ) # [B, S, conv_dim] + z = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.in_proj_z.weight, + self.in_proj_z.bias, + tp_mode="colwise", + layer_type="delta", + ) # [B, S, value_dim] + b = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.in_proj_b.weight, + self.in_proj_b.bias, + tp_mode="colwise", + layer_type="delta", + ) # [B, S, num_v_heads] + a = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.in_proj_a.weight, + self.in_proj_a.bias, + tp_mode="colwise", + layer_type="delta", + ) # [B, S, num_v_heads] # 2. Causal Conv1d via autodeploy op # torch_causal_conv1d expects [B, S, C] input, handles transpose internally @@ -369,34 +395,74 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: self.conv1d.dilation[0], self.conv1d.groups, self.conv1d.padding_mode, + enable_sharding=True, + output_sizes=[self.key_dim, self.key_dim, self.value_dim], + layer_type="delta", ) mixed_qkv = F.silu(mixed_qkv) # Split back into Q, K, V - query, key, value = torch.split( + query, key, value = torch.ops.auto_deploy.split_with_sizes( mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1, + enable_sharding=True, + layer_type="delta", ) # Reshape to per-head: [B, S, num_heads, head_dim] - query = query.reshape(batch_size, seq_len, -1, self.head_k_dim) - key = key.reshape(batch_size, seq_len, -1, self.head_k_dim) - value = value.reshape(batch_size, seq_len, -1, self.head_v_dim) + query = torch.ops.auto_deploy.view( + query, + [batch_size, seq_len, -1, self.head_k_dim], + tp_scaled_dim=2, + layer_type="delta", + ) + key = torch.ops.auto_deploy.view( + key, + [batch_size, seq_len, -1, self.head_k_dim], + tp_scaled_dim=2, + layer_type="delta", + ) + value = torch.ops.auto_deploy.view( + value, + [batch_size, seq_len, -1, self.head_v_dim], + tp_scaled_dim=2, + layer_type="delta", + ) # 3. Gated Delta Rule via autodeploy custom op # L2 norm, GQA repeat-interleave, and g/beta computation are handled inside the op. core_attn_out = torch.ops.auto_deploy.torch_gated_delta_rule( - query, key, value, a, b, self.A_log, self.dt_bias + query, + key, + value, + a, + b, + self.A_log, + self.dt_bias, + enable_sharding=True, + layer_type="delta", ) # 5. Gated RMSNorm + merge heads - z = z.reshape(batch_size, seq_len, -1, self.head_v_dim) # [B, S, num_v_heads, head_v_dim] + z = torch.ops.auto_deploy.view( + z, + [batch_size, seq_len, -1, self.head_v_dim], + tp_scaled_dim=2, + layer_type="delta", + ) # [B, S, num_v_heads, head_v_dim] core_attn_out = self.norm(core_attn_out, z) core_attn_out = core_attn_out.reshape(batch_size, seq_len, -1) # 6. Output projection - output = self.out_proj(core_attn_out) + output = torch.ops.auto_deploy.torch_linear_simple( + core_attn_out, + self.out_proj.weight, + self.out_proj.bias, + tp_mode="rowwise", + layer_type="delta", + ) + output = torch.ops.auto_deploy.all_reduce(output, layer_type="delta") return output @@ -458,16 +524,50 @@ def forward( bsz, q_len, _ = hidden_states.size() # Q projection with gate: output shape (B, S, N, 2*D) - qg = self.q_proj(hidden_states).view(bsz, q_len, -1, self.head_dim * 2) + qg = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.q_proj.weight, + self.q_proj.bias, + tp_mode="colwise", + layer_type="mha", + ) + qg = torch.ops.auto_deploy.view( + qg, + [bsz, q_len, -1, self.head_dim * 2], + tp_scaled_dim=2, + layer_type="mha", + ) query_states, gate = torch.chunk(qg, 2, dim=-1) # each (B, S, N, D) gate = gate.reshape(bsz, q_len, -1) # (B, S, N*D) # K, V projections in bsnd layout - key_states = self.k_proj(hidden_states).view( - bsz, q_len, self.num_key_value_heads, self.head_dim + key_states = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.k_proj.weight, + self.k_proj.bias, + tp_mode="colwise", + tp_min_local_shape=self.head_dim, + layer_type="mha", + ) + key_states = torch.ops.auto_deploy.view( + key_states, + [bsz, q_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="mha", + ) + value_states = torch.ops.auto_deploy.torch_linear_simple( + hidden_states, + self.v_proj.weight, + self.v_proj.bias, + tp_mode="colwise", + tp_min_local_shape=self.head_dim, + layer_type="mha", ) - value_states = self.v_proj(hidden_states).view( - bsz, q_len, self.num_key_value_heads, self.head_dim + value_states = torch.ops.auto_deploy.view( + value_states, + [bsz, q_len, -1, self.head_dim], + tp_scaled_dim=2, + layer_type="mha", ) # Per-head Q/K norms (norm operates on last dim = head_dim) @@ -496,7 +596,14 @@ def forward( attn_output = attn_output * torch.sigmoid(gate) # Output projection - attn_output = self.o_proj(attn_output) + attn_output = torch.ops.auto_deploy.torch_linear_simple( + attn_output, + self.o_proj.weight, + self.o_proj.bias, + tp_mode="rowwise", + layer_type="mha", + ) + attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mha") return attn_output @@ -518,7 +625,27 @@ def __init__(self, config: Qwen3_5MoeTextConfig, intermediate_size: int): self.act_fn = ACT2FN[config.hidden_act] def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + gate = torch.ops.auto_deploy.torch_linear_simple( + x, + self.gate_proj.weight, + self.gate_proj.bias, + tp_mode="colwise", + layer_type="moe", + ) + up = torch.ops.auto_deploy.torch_linear_simple( + x, + self.up_proj.weight, + self.up_proj.bias, + tp_mode="colwise", + layer_type="moe", + ) + return torch.ops.auto_deploy.torch_linear_simple( + self.act_fn(gate) * up, + self.down_proj.weight, + self.down_proj.bias, + tp_mode="rowwise", + layer_type="moe", + ) class Qwen3_5MoeExpert(nn.Module): @@ -635,9 +762,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: w2_weights, w3_weights, is_gated_mlp=True, + layer_type="moe", ) expert_output = expert_output + shared_expert_output + expert_output = torch.ops.auto_deploy.all_reduce(expert_output, layer_type="moe") expert_output = expert_output.reshape(batch_size, sequence_length, hidden_dim) return expert_output diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe_ir.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe_ir.py deleted file mode 100644 index 68f76a3f2cbb..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe_ir.py +++ /dev/null @@ -1,3055 +0,0 @@ -# Copyright 2018 The HuggingFace Team -# Licensed under the Apache License, Version 2.0. -# Original source: https://github.com/huggingface/transformers -# -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sharding-aware Qwen3.5 MoE model for AutoDeploy IR sharding (text + vision). - -Reference HF modeling file (not yet in a released transformers version): - transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py - -This implementation differs from the HuggingFace original in the following ways: - * External kernel dependencies (flash-linear-attention, causal_conv1d) are replaced with - autodeploy custom ops. - * Cache-related code paths have been removed (prefill-only). - * Training-related code paths have been removed. - * Unnecessary output fields have been removed. - * The GatedDeltaNet forward is adapted from the Qwen3Next GDN patch - (tensorrt_llm/_torch/auto_deploy/models/patches/qwen3_next.py). - * The MoE implementation uses expert lists (individual nn.Linear layers per expert) - that directly match the checkpoint structure, dispatched via torch_moe op. - * The VLM wrapper passes 3D ``position_ids (3, B, S)`` to the text model, - which computes mRoPE cos/sin internally via its own ``rotary_emb``. - For text-only inputs the wrapper expands the executor's 2D positions to 3D; - for multimodal inputs it computes spatial (T, H, W) positions via ``get_rope_index``. - -This allows us to have a "pytorch" native reference implementation decoupled from bugs and -dependency issues in the source, while remaining weight-compatible with HF checkpoints. -""" - -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union - -import torch -import torch.nn.functional as F -from PIL import Image -from torch import nn -from torch.export import Dim -from transformers import AutoConfig -from transformers.activations import ACT2FN -from transformers.configuration_utils import PretrainedConfig -from transformers.generation import GenerationMixin -from transformers.modeling_outputs import BaseModelOutputWithPooling -from transformers.modeling_utils import PreTrainedModel -from transformers.utils import ModelOutput - -from tensorrt_llm.inputs.multimodal import MultimodalInput, apply_mm_hashes, hexdigest_to_int32 -from tensorrt_llm.inputs.utils import VideoData - -from ... import custom_ops # noqa: F401, I001 -- register all ops -from ...custom_ops.attention_interface import BatchInfo -from ..factory import ModelFactoryRegistry -from ..hf import ( - AutoModelForCausalLMFactory, - AutoModelForImageTextToTextFactory, - TextModelExportInfo, -) - -# ============================================================================= -# Configuration -# ============================================================================= - - -class Qwen3_5MoeTextConfig(PretrainedConfig): - """Minimal config class for Qwen3.5 MoE text model. - - Mirrors the attributes of the upstream Qwen3_5MoeTextConfig. Only attributes - needed by the slimmed-down prefill model are included. - """ - - model_type = "qwen3_5_moe_text" - - def __init__( - self, - vocab_size=248320, - hidden_size=2048, - num_hidden_layers=40, - num_attention_heads=16, - num_key_value_heads=2, - hidden_act="silu", - max_position_embeddings=32768, - initializer_range=0.02, - rms_norm_eps=1e-6, - tie_word_embeddings=False, - rope_parameters=None, - attention_bias=False, - attention_dropout=0.0, - head_dim=256, - # linear attention (GatedDeltaNet) params - linear_conv_kernel_dim=4, - linear_key_head_dim=128, - linear_value_head_dim=128, - linear_num_key_heads=16, - linear_num_value_heads=32, - # MoE params - moe_intermediate_size=512, - shared_expert_intermediate_size=512, - num_experts_per_tok=8, - num_experts=256, - # layer types - layer_types=None, - pad_token_id=None, - **kwargs, - ): - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.max_position_embeddings = max_position_embeddings - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - self.head_dim = head_dim - - if rope_parameters is None: - rope_parameters = { - "rope_type": "default", - "rope_theta": 1000000.0, - "partial_rotary_factor": 0.25, - "mrope_section": [11, 11, 10], - } - self.rope_parameters = rope_parameters - - self.linear_conv_kernel_dim = linear_conv_kernel_dim - self.linear_key_head_dim = linear_key_head_dim - self.linear_value_head_dim = linear_value_head_dim - self.linear_num_key_heads = linear_num_key_heads - self.linear_num_value_heads = linear_num_value_heads - - self.moe_intermediate_size = moe_intermediate_size - self.shared_expert_intermediate_size = shared_expert_intermediate_size - self.num_experts_per_tok = num_experts_per_tok - self.num_experts = num_experts - - self.layer_types = layer_types - if self.layer_types is None: - # Default pattern: every 4th layer is full_attention, rest are linear_attention - interval_pattern = kwargs.pop("full_attention_interval", 4) - self.layer_types = [ - "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention" - for i in range(self.num_hidden_layers) - ] - - super().__init__( - pad_token_id=pad_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -# ============================================================================= -# Normalization -# ============================================================================= - - -class Qwen3_5MoeRMSNorm(nn.Module): - """RMSNorm with weight scaling. - - The HF checkpoint stores weights in ``(1 + w)`` parameterisation (zeros - init). A load-time pre-hook adds 1.0 so that the forward can use a plain - ``weight * x`` multiply, which matches the autodeploy RMSNorm pattern and - gets fused into a single ``flashinfer_rms_norm`` kernel. - """ - - def __init__(self, dim: int, eps: float = 1e-6): - super().__init__() - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim)) - self._register_load_state_dict_pre_hook(self._offset_weight) - - @staticmethod - def _offset_weight(state_dict, prefix, *args): - key = prefix + "weight" - assert key in state_dict, f"RMSNorm: Key {key} not found in state_dict" - state_dict[key] = state_dict[key] + 1.0 - - def forward(self, x: torch.Tensor) -> torch.Tensor: - input_dtype = x.dtype - output = x.to(torch.float32) - output = output * torch.rsqrt(output.pow(2).mean(-1, keepdim=True) + self.eps) - return (self.weight.to(torch.float32) * output).to(input_dtype) - - -class Qwen3_5MoeRMSNormGated(nn.Module): - """Gated RMSNorm: norm(x) * weight * silu(gate). Weight is initialized to ones.""" - - def __init__(self, hidden_size: int, eps: float = 1e-6): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - hidden_states = self.weight * hidden_states.to(input_dtype) - hidden_states = hidden_states * F.silu(gate.to(torch.float32)) - return hidden_states.to(input_dtype) - - -# ============================================================================= -# Rotary Position Embedding (mRoPE) -# ============================================================================= - - -def rotate_half(x: torch.Tensor) -> torch.Tensor: - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb( - q: torch.Tensor, - k: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, - unsqueeze_dim: int = 2, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply partial RoPE to query and key tensors. - - Supports partial rotary where only the first `rotary_dim` dimensions are rotated. - Default unsqueeze_dim=2 is for bsnd layout (B, S, N, D). - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - - rotary_dim = cos.shape[-1] - q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:] - k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:] - - q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin) - k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin) - - q_embed = torch.cat([q_embed, q_pass], dim=-1) - k_embed = torch.cat([k_embed, k_pass], dim=-1) - return q_embed, k_embed - - -class Qwen3_5MoeTextRotaryEmbedding(nn.Module): - """Simplified mRoPE for text-only prefill. Supports only the "default" rope type.""" - - def __init__(self, config: Qwen3_5MoeTextConfig): - super().__init__() - rope_params = config.rope_parameters - base = rope_params["rope_theta"] - partial_rotary_factor = rope_params.get("partial_rotary_factor", 1.0) - head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) - dim = int(head_dim * partial_rotary_factor) - - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float() / dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.mrope_section = rope_params.get("mrope_section", [11, 11, 10]) - - @torch.no_grad() - def forward( - self, x: torch.Tensor, position_ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute cos/sin embeddings. - - Args: - x: Hidden states tensor, used only for dtype/device. - position_ids: Shape (3, B, S) for mRoPE or (B, S) for plain. - """ - if position_ids.ndim == 2: - position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) - - # inv_freq: (dim/2,) -> (3, B, dim/2, 1) - inv_freq_expanded = ( - self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) - ) - # position_ids: (3, B, S) -> (3, B, 1, S) - position_ids_expanded = position_ids[:, :, None, :].float() - - # freqs: (3, B, S, dim/2) - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) - # Apply interleaved mRoPE: (3, B, S, dim/2) -> (B, S, dim/2) - freqs = self._apply_interleaved_mrope(freqs) - # Double for cos/sin: (B, S, dim) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() - sin = emb.sin() - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor: - """Apply interleaved mRoPE. Merges T/H/W frequency channels into one tensor.""" - freqs_t = freqs[0].clone() - for dim_idx, offset in enumerate((1, 2), start=1): - length = self.mrope_section[dim_idx] * 3 - idx = slice(offset, length, 3) - freqs_t[..., idx] = freqs[dim_idx, ..., idx] - return freqs_t - - -# ============================================================================= -# GatedDeltaNet (Linear Attention) -# ============================================================================= -# Adapted from the Qwen3Next GDN patch: -# tensorrt_llm/_torch/auto_deploy/models/patches/qwen3_next.py -# Uses autodeploy custom ops: torch_causal_conv1d, torch_gated_delta_rule - - -class Qwen3_5MoeGatedDeltaNet(nn.Module): - """Prefill-only GatedDeltaNet using autodeploy custom ops.""" - - def __init__(self, config: Qwen3_5MoeTextConfig, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - self.num_v_heads = config.linear_num_value_heads - self.num_k_heads = config.linear_num_key_heads - self.head_k_dim = config.linear_key_head_dim - self.head_v_dim = config.linear_value_head_dim - self.key_dim = self.head_k_dim * self.num_k_heads - self.value_dim = self.head_v_dim * self.num_v_heads - - self.conv_kernel_size = config.linear_conv_kernel_dim - self.layer_idx = layer_idx - - # QKV convolution - self.conv_dim = self.key_dim * 2 + self.value_dim - self.conv1d = nn.Conv1d( - in_channels=self.conv_dim, - out_channels=self.conv_dim, - bias=False, - kernel_size=self.conv_kernel_size, - groups=self.conv_dim, - padding=self.conv_kernel_size - 1, - ) - - # dt_bias and A_log for gated delta rule - self.dt_bias = nn.Parameter(torch.ones(self.num_v_heads)) - A = torch.empty(self.num_v_heads).uniform_(0, 16) - self.A_log = nn.Parameter(torch.log(A)) - - # Gated RMSNorm (per head_v_dim) - self.norm = Qwen3_5MoeRMSNormGated(self.head_v_dim, eps=config.rms_norm_eps) - - # Projections - self.in_proj_qkv = nn.Linear( - self.hidden_size, self.key_dim * 2 + self.value_dim, bias=False - ) - self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False) - self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False) - self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False) - self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False) - - self._conv_split_sizes = [self.key_dim, self.key_dim, self.value_dim] - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - batch_size, seq_len, _ = hidden_states.shape - - # 1. Projections with sharding hints - mixed_qkv = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.in_proj_qkv.weight, - self.in_proj_qkv.bias, - tp_mode="colwise", - output_sizes=self._conv_split_sizes, - layer_type="delta", - ) - z = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.in_proj_z.weight, - self.in_proj_z.bias, - tp_mode="colwise", - layer_type="delta", - ) - z = torch.ops.auto_deploy.view( - z, - [batch_size, seq_len, -1, self.head_v_dim], - tp_scaled_dim=2, - layer_type="delta", - ) - b = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.in_proj_b.weight, - self.in_proj_b.bias, - tp_mode="colwise", - layer_type="delta", - ) - a = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.in_proj_a.weight, - self.in_proj_a.bias, - tp_mode="colwise", - layer_type="delta", - ) - - # 2. Causal Conv1d with sharding hint - mixed_qkv = torch.ops.auto_deploy.torch_causal_conv1d( - mixed_qkv, - self.conv1d.weight, - self.conv1d.bias, - self.conv1d.stride[0], - self.conv1d.padding[0], - self.conv1d.dilation[0], - self.conv1d.groups, - self.conv1d.padding_mode, - enable_sharding=True, - output_sizes=self._conv_split_sizes, - layer_type="delta", - ) - mixed_qkv = F.silu(mixed_qkv) - - # Split into Q, K, V with enable_sharding hint - query, key, value = torch.ops.auto_deploy.split_with_sizes( - mixed_qkv, - [self.key_dim, self.key_dim, self.value_dim], - dim=-1, - enable_sharding=True, - layer_type="delta", - ) - - # Reshape to per-head: [B, S, num_heads, head_dim] with -1 at head dim - query = torch.ops.auto_deploy.view( - query, - [batch_size, seq_len, -1, self.head_k_dim], - tp_scaled_dim=2, - layer_type="delta", - ) - key = torch.ops.auto_deploy.view( - key, - [batch_size, seq_len, -1, self.head_k_dim], - tp_scaled_dim=2, - layer_type="delta", - ) - value = torch.ops.auto_deploy.view( - value, - [batch_size, seq_len, -1, self.head_v_dim], - tp_scaled_dim=2, - layer_type="delta", - ) - - # 3. Gated Delta Rule with enable_sharding hint - core_attn_out = torch.ops.auto_deploy.torch_gated_delta_rule( - query, - key, - value, - a, - b, - self.A_log, - self.dt_bias, - enable_sharding=True, - layer_type="delta", - ) - - # 4. Gated RMSNorm (norm weight is replicated -- constant head_v_dim) - core_attn_out = core_attn_out.reshape(-1, core_attn_out.shape[-1]) - z_flat = z.reshape(-1, z.shape[-1]) - core_attn_out = self.norm(core_attn_out, z_flat) - core_attn_out = torch.ops.auto_deploy.view( - core_attn_out, - [batch_size, seq_len, -1, self.head_v_dim], - tp_scaled_dim=2, - layer_type="delta", - ) - core_attn_out = core_attn_out.reshape(batch_size, seq_len, -1) - - # 5. Output projection (rowwise) + all_reduce - output = torch.ops.auto_deploy.torch_linear_simple( - core_attn_out, - self.out_proj.weight, - self.out_proj.bias, - tp_mode="rowwise", - layer_type="delta", - ) - output = torch.ops.auto_deploy.all_reduce(output, layer_type="delta") - return output - - -# ============================================================================= -# Attention -# ============================================================================= - - -class Qwen3_5MoeAttention(nn.Module): - """Multi-headed attention with gating, Q/K norms, and partial RoPE. - - Key differences from standard attention: - - q_proj outputs 2x (query + gate), gating applied to attention output. - - q_norm / k_norm applied per-head before RoPE. - - Partial RoPE: only first rotary_dim dimensions are rotated. - """ - - def __init__(self, config: Qwen3_5MoeTextConfig, layer_idx: int): - super().__init__() - self.layer_idx = layer_idx - self.hidden_size = config.hidden_size - self.head_dim = getattr( - config, "head_dim", config.hidden_size // config.num_attention_heads - ) - self.num_heads = config.num_attention_heads - self.num_key_value_heads = config.num_key_value_heads - - # q_proj outputs 2x for query + gate - self.q_proj = nn.Linear( - config.hidden_size, - config.num_attention_heads * self.head_dim * 2, - bias=config.attention_bias, - ) - self.k_proj = nn.Linear( - config.hidden_size, - config.num_key_value_heads * self.head_dim, - bias=config.attention_bias, - ) - self.v_proj = nn.Linear( - config.hidden_size, - config.num_key_value_heads * self.head_dim, - bias=config.attention_bias, - ) - self.o_proj = nn.Linear( - config.num_attention_heads * self.head_dim, - config.hidden_size, - bias=config.attention_bias, - ) - - # Per-head Q/K norms - self.q_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.k_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - ) -> torch.Tensor: - bsz, q_len, _ = hidden_states.size() - - # Q projection with gate (interleaved per-head [q_h0,g_h0,...] -- plain colwise is correct) - qg = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.q_proj.weight, - self.q_proj.bias, - tp_mode="colwise", - layer_type="mha", - ) - qg = torch.ops.auto_deploy.view( - qg, - [bsz, q_len, -1, self.head_dim * 2], - tp_scaled_dim=2, - layer_type="mha", - ) - query_states, gate = torch.chunk(qg, 2, dim=-1) - gate = gate.reshape(bsz, q_len, -1) - - # K, V projections with tp_min_local_shape for GQA - key_states = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.k_proj.weight, - self.k_proj.bias, - tp_mode="colwise", - tp_min_local_shape=self.head_dim, - layer_type="mha", - ) - key_states = torch.ops.auto_deploy.view( - key_states, - [bsz, q_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="mha", - ) - value_states = torch.ops.auto_deploy.torch_linear_simple( - hidden_states, - self.v_proj.weight, - self.v_proj.bias, - tp_mode="colwise", - tp_min_local_shape=self.head_dim, - layer_type="mha", - ) - value_states = torch.ops.auto_deploy.view( - value_states, - [bsz, q_len, -1, self.head_dim], - tp_scaled_dim=2, - layer_type="mha", - ) - - # Per-head Q/K norms (norm on last dim = head_dim, no sharding) - query_states = self.q_norm(query_states) - key_states = self.k_norm(key_states) - - # Partial RoPE in bsnd layout - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb( - query_states, key_states, cos, sin, unsqueeze_dim=2 - ) - - # Attention via autodeploy op (bsnd layout) - attn_output = torch.ops.auto_deploy.torch_attention( - query_states, - key_states, - value_states, - attn_mask=None, - dropout_p=0.0, - is_causal=True, - layout="bsnd", - ) - attn_output = attn_output.view(bsz, q_len, -1) - - # Gated output - attn_output = attn_output * torch.sigmoid(gate) - - # Output projection (rowwise) + all_reduce - attn_output = torch.ops.auto_deploy.torch_linear_simple( - attn_output, - self.o_proj.weight, - self.o_proj.bias, - tp_mode="rowwise", - layer_type="mha", - ) - attn_output = torch.ops.auto_deploy.all_reduce(attn_output, layer_type="mha") - return attn_output - - -# ============================================================================= -# MLP and MoE -# ============================================================================= - - -class Qwen3_5MoeMLP(nn.Module): - """SwiGLU MLP used for the shared expert and standalone MLP layers.""" - - def __init__( - self, - config: Qwen3_5MoeTextConfig, - intermediate_size: int, - add_all_reduce: bool = True, - sharding_layer_type: str = "mlp", - ): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = intermediate_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_act] - self.add_all_reduce = add_all_reduce - self.sharding_layer_type = sharding_layer_type - - def forward(self, x: torch.Tensor) -> torch.Tensor: - _lt = self.sharding_layer_type - gate = torch.ops.auto_deploy.torch_linear_simple( - x, - self.gate_proj.weight, - self.gate_proj.bias, - tp_mode="colwise", - layer_type=_lt, - ) - up = torch.ops.auto_deploy.torch_linear_simple( - x, - self.up_proj.weight, - self.up_proj.bias, - tp_mode="colwise", - layer_type=_lt, - ) - down = torch.ops.auto_deploy.torch_linear_simple( - self.act_fn(gate) * up, - self.down_proj.weight, - self.down_proj.bias, - tp_mode="rowwise", - layer_type=_lt, - ) - if self.add_all_reduce: - down = torch.ops.auto_deploy.all_reduce(down, layer_type=_lt) - return down - - -class Qwen3_5MoeExpert(nn.Module): - """Single expert with gate, up, and down projections.""" - - def __init__(self, hidden_dim: int, intermediate_dim: int): - super().__init__() - self.gate_proj = nn.Linear(hidden_dim, intermediate_dim, bias=False) - self.up_proj = nn.Linear(hidden_dim, intermediate_dim, bias=False) - self.down_proj = nn.Linear(intermediate_dim, hidden_dim, bias=False) - - -class Qwen3_5MoeTopKRouter(nn.Module): - """Top-K router with softmax normalization.""" - - def __init__(self, config: Qwen3_5MoeTextConfig): - super().__init__() - self.top_k = config.num_experts_per_tok - self.num_experts = config.num_experts - self.hidden_dim = config.hidden_size - self.weight = nn.Parameter(torch.zeros(self.num_experts, self.hidden_dim)) - - def forward(self, hidden_states: torch.Tensor): - hidden_states = hidden_states.reshape(-1, self.hidden_dim) - router_logits = F.linear(hidden_states, self.weight) # (T, E) - routing_weights = F.softmax(router_logits, dtype=torch.float, dim=-1) - routing_weights, selected_experts = torch.topk( - routing_weights, self.top_k, dim=-1 - ) # (T, top_k) - routing_weights = routing_weights / routing_weights.sum(dim=-1, keepdim=True) - routing_weights = routing_weights.to(hidden_states.dtype) - return routing_weights, selected_experts - - -class Qwen3_5MoeSparseMoeBlock(nn.Module): - """MoE block with expert list implementation. - - Implements routed experts by iterating over selected experts and dispatching - tokens accordingly. Each expert is a separate nn.Linear triplet (gate, up, down). - """ - - def __init__(self, config: Qwen3_5MoeTextConfig): - super().__init__() - self.gate = Qwen3_5MoeTopKRouter(config) - self.experts = nn.ModuleList( - [ - Qwen3_5MoeExpert(config.hidden_size, config.moe_intermediate_size) - for _ in range(config.num_experts) - ] - ) - self.shared_expert = Qwen3_5MoeMLP( - config, - intermediate_size=config.shared_expert_intermediate_size, - add_all_reduce=False, - sharding_layer_type="moe", - ) - self.shared_expert_gate = nn.Linear(config.hidden_size, 1, bias=False) - self._register_load_state_dict_pre_hook(self._load_experts_from_fused_checkpoint) - - @staticmethod - def _load_experts_from_fused_checkpoint(state_dict, prefix, *args): - """Load fused MoE expert checkpoint tensors into per-expert ModuleList params. - - Checkpoint format: - - experts.gate_up_proj: [E, 2*I, H] with [gate, up] stacking - - experts.down_proj: [E, H, I] - - Target format: - - experts.{expert_id}.gate_proj.weight: [I, H] - - experts.{expert_id}.up_proj.weight: [I, H] - - experts.{expert_id}.down_proj.weight: [H, I] - """ - gate_up_key = prefix + "experts.gate_up_proj" - down_key = prefix + "experts.down_proj" - - if gate_up_key in state_dict: - fused = state_dict.pop(gate_up_key) - num_experts = fused.shape[0] - intermediate_dim = fused.shape[1] // 2 - gate_weights = fused[:, :intermediate_dim, :] - up_weights = fused[:, intermediate_dim:, :] - - for i in range(num_experts): - state_dict[f"{prefix}experts.{i}.gate_proj.weight"] = gate_weights[i] - state_dict[f"{prefix}experts.{i}.up_proj.weight"] = up_weights[i] - - if down_key in state_dict: - fused = state_dict.pop(down_key) - num_experts = fused.shape[0] - for i in range(num_experts): - state_dict[f"{prefix}experts.{i}.down_proj.weight"] = fused[i] - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - batch_size, sequence_length, hidden_dim = hidden_states.shape - hidden_states_flat = hidden_states.view(-1, hidden_dim) - - # Router - routing_weights, selected_experts = self.gate(hidden_states_flat) - - # Routed experts via torch_moe (sharded by apply_sharding_hints) - w1_weights = [self.experts[i].gate_proj.weight for i in range(len(self.experts))] - w2_weights = [self.experts[i].down_proj.weight for i in range(len(self.experts))] - w3_weights = [self.experts[i].up_proj.weight for i in range(len(self.experts))] - - expert_output = torch.ops.auto_deploy.torch_moe( - hidden_states_flat, - selected_experts, - routing_weights, - w1_weights, - w2_weights, - w3_weights, - is_gated_mlp=True, - layer_type="moe", - ) - - # Shared expert with sigmoid gating (all_reduce deferred) - shared_expert_output = self.shared_expert(hidden_states_flat) - shared_expert_output = ( - F.sigmoid(self.shared_expert_gate(hidden_states_flat)) * shared_expert_output - ) - - # Merge routed + shared, then single all_reduce - expert_output = expert_output + shared_expert_output - expert_output = torch.ops.auto_deploy.all_reduce(expert_output, layer_type="moe") - - expert_output = expert_output.reshape(batch_size, sequence_length, hidden_dim) - return expert_output - - -# ============================================================================= -# Decoder Layer -# ============================================================================= - - -class Qwen3_5MoeDecoderLayer(nn.Module): - """Single decoder layer: token mixer (linear_attention or full_attention) + MoE.""" - - def __init__(self, config: Qwen3_5MoeTextConfig, layer_idx: int): - super().__init__() - self.layer_type = config.layer_types[layer_idx] - - if self.layer_type == "linear_attention": - self.linear_attn = Qwen3_5MoeGatedDeltaNet(config, layer_idx) - elif self.layer_type == "full_attention": - self.self_attn = Qwen3_5MoeAttention(config, layer_idx) - else: - raise ValueError(f"Unknown layer type: {self.layer_type}") - - self.mlp = Qwen3_5MoeSparseMoeBlock(config) - self.input_layernorm = Qwen3_5MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = Qwen3_5MoeRMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - ) -> torch.Tensor: - # Token mixer - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - - if self.layer_type == "linear_attention": - hidden_states = self.linear_attn(hidden_states) - elif self.layer_type == "full_attention": - hidden_states = self.self_attn(hidden_states, position_embeddings=position_embeddings) - - hidden_states = residual + hidden_states - - # Channel mixer (MoE) - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - - return hidden_states - - -# ============================================================================= -# Model -# ============================================================================= - - -class Qwen3_5MoePreTrainedModel(PreTrainedModel): - """Base class for Qwen3.5 MoE pretrained models.""" - - config_class = Qwen3_5MoeTextConfig - base_model_prefix = "model" - _no_split_modules = ["Qwen3_5MoeDecoderLayer"] - supports_gradient_checkpointing = True - _is_stateful = True - - def _init_weights(self, module): - # Delegate nn.Linear / nn.Embedding / nn.Conv* to the base class, which - # safely resolves initializer_range via hasattr + get_text_config() fallback. - super()._init_weights(module) - std = getattr(self.config, "initializer_range", 0.02) - if isinstance(module, Qwen3_5MoeRMSNorm): - module.weight.data.fill_(1.0) - elif isinstance(module, Qwen3_5MoeRMSNormGated): - module.weight.data.fill_(1.0) - elif isinstance(module, Qwen3_5MoeGatedDeltaNet): - module.dt_bias.data.fill_(1.0) - module.A_log.data.copy_(torch.empty_like(module.A_log).uniform_(0, 16).log_()) - elif isinstance(module, Qwen3_5MoeTopKRouter): - module.weight.data.normal_(mean=0.0, std=std) - - -@dataclass -class Qwen3_5MoeOutput(ModelOutput): - """Output of the Qwen3.5 MoE text model.""" - - last_hidden_state: Optional[torch.FloatTensor] = None - - -@dataclass -class Qwen3_5MoeCausalLMOutput(ModelOutput): - """Output of the Qwen3.5 MoE causal language model.""" - - logits: Optional[torch.FloatTensor] = None - last_hidden_state: Optional[torch.FloatTensor] = None - - -class Qwen3_5MoeTextModel(Qwen3_5MoePreTrainedModel): - """Qwen3.5 MoE text model (embed + decoder layers + final norm + lm_head). - - lm_head is included so that the exported GraphModule contains it directly, - allowing sharding and gather_logits_before_lm_head transforms to see it. - """ - - def __init__(self, config: Qwen3_5MoeTextConfig): - super().__init__(config) - pad_token_id = getattr(config, "pad_token_id", None) - self.embed_tokens = nn.Embedding( - config.vocab_size, config.hidden_size, padding_idx=pad_token_id - ) - self.layers = nn.ModuleList( - [ - Qwen3_5MoeDecoderLayer(config, layer_idx) - for layer_idx in range(config.num_hidden_layers) - ] - ) - self.norm = Qwen3_5MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = Qwen3_5MoeTextRotaryEmbedding(config=config) - self.lm_head = None - - # Initialize weights and apply final processing - self.post_init() - - def set_lm_head(self, lm_head: nn.Module): - """Set the lm_head from the parent model.""" - self.lm_head = lm_head - - def get_input_embeddings(self): - return self.embed_tokens - - def set_input_embeddings(self, new_embeddings): - self.embed_tokens = new_embeddings - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - rope_cos: Optional[torch.Tensor] = None, - rope_sin: Optional[torch.Tensor] = None, - **kwargs, - ) -> Union[Tuple, Qwen3_5MoeOutput]: - """Forward pass. - - There are three ways to provide position information (checked in order): - - 1. ``rope_cos`` + ``rope_sin``: separate tensors, each ``(B, S, rotary_dim)``. - Export-friendly -- each is a proper graph input with its own dynamic shape. - 2. ``position_embeddings``: pre-computed ``(cos, sin)`` tuple. Convenient for - the multimodal wrapper calling at the plain-PyTorch level. - 3. ``position_ids`` (or ``None``): standard 2D ``(B, S)`` or 3D ``(3, B, S)`` - position IDs. The internal ``rotary_emb`` computes cos/sin. This is the - default text-only path. - """ - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - # Resolve position embeddings from one of the three input modes. - if rope_cos is not None and rope_sin is not None: - position_embeddings = (rope_cos, rope_sin) - elif position_embeddings is None: - if position_ids is None: - seq_len = inputs_embeds.shape[1] - position_ids = torch.arange(seq_len, device=inputs_embeds.device) - position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) - elif position_ids.ndim == 2: - position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) - position_embeddings = self.rotary_emb(inputs_embeds, position_ids) - - hidden_states = inputs_embeds - for decoder_layer in self.layers: - hidden_states = decoder_layer(hidden_states, position_embeddings=position_embeddings) - - hidden_states = self.norm(hidden_states) - assert self.lm_head is not None, ( - "lm_head not set — call set_lm_head() from the parent model before forward()" - ) - logits = self.lm_head(hidden_states.to(self.lm_head.weight.dtype)).float() - return Qwen3_5MoeCausalLMOutput(logits=logits, last_hidden_state=hidden_states) - - -class Qwen3_5MoeForCausalLM(Qwen3_5MoePreTrainedModel, GenerationMixin): - """Qwen3.5 MoE causal language model (text model + lm_head).""" - - _tied_weights_keys = ["lm_head.weight"] - - def __init__(self, config: Qwen3_5MoeTextConfig, **kwargs): - super().__init__(config) - self.model = Qwen3_5MoeTextModel(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - self.model.set_lm_head(self.lm_head) - - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self): - return self.model.get_input_embeddings() - - def set_input_embeddings(self, new_embeddings): - return self.model.set_input_embeddings(new_embeddings) - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - self.model.set_lm_head(new_embeddings) - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - rope_cos: Optional[torch.Tensor] = None, - rope_sin: Optional[torch.Tensor] = None, - **kwargs, - ) -> Union[Tuple, Qwen3_5MoeCausalLMOutput]: - outputs = self.model( - input_ids, - inputs_embeds=inputs_embeds, - position_ids=position_ids, - position_embeddings=position_embeddings, - rope_cos=rope_cos, - rope_sin=rope_sin, - ) - logits = outputs.logits - return Qwen3_5MoeCausalLMOutput(logits=logits) - - -# ============================================================================= -# Vision Configuration -# ============================================================================= - - -class Qwen3_5MoeVisionConfig(PretrainedConfig): - """Config class for the Qwen3.5 MoE vision tower. - - Mirrors the upstream ``Qwen3_5MoeVisionConfig``. - """ - - model_type = "qwen3_5_moe_vision" - - def __init__( - self, - depth: int = 27, - hidden_size: int = 1152, - hidden_act: str = "gelu_pytorch_tanh", - intermediate_size: int = 4304, - num_heads: int = 16, - in_channels: int = 3, - patch_size: int = 16, - spatial_merge_size: int = 2, - temporal_patch_size: int = 2, - out_hidden_size: int = 3584, - num_position_embeddings: int = 2304, - initializer_range: float = 0.02, - **kwargs, - ): - super().__init__(**kwargs) - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.intermediate_size = intermediate_size - self.num_heads = num_heads - self.in_channels = in_channels - self.patch_size = patch_size - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.out_hidden_size = out_hidden_size - self.num_position_embeddings = num_position_embeddings - self.initializer_range = initializer_range - - -class Qwen3_5MoeConfig(PretrainedConfig): - """Composite config containing both text and vision configs. - - Mirrors the upstream ``Qwen3_5MoeConfig``. - """ - - model_type = "qwen3_5_moe" - - def __init__( - self, - text_config=None, - vision_config=None, - image_token_id: int = 248056, - video_token_id: int = 248057, - vision_start_token_id: int = 248053, - vision_end_token_id: int = 248054, - tie_word_embeddings: bool = False, - **kwargs, - ): - if isinstance(vision_config, dict): - self.vision_config = Qwen3_5MoeVisionConfig(**vision_config) - elif vision_config is None: - self.vision_config = Qwen3_5MoeVisionConfig() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - self.text_config = Qwen3_5MoeTextConfig(**text_config) - elif text_config is None: - self.text_config = Qwen3_5MoeTextConfig() - else: - self.text_config = text_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - -# ============================================================================= -# Vision Tower Components (plain PyTorch -- NOT exported) -# ============================================================================= - - -class Qwen3_5MoeVisionRotaryEmbedding(nn.Module): - """Simple rotary embedding for the vision tower (not mRoPE).""" - - def __init__(self, dim: int, theta: float = 10000.0): - super().__init__() - self.dim = dim - self.theta = theta - inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, seqlen: int) -> torch.Tensor: - seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) - freqs = torch.outer(seq, self.inv_freq) - return freqs - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply RoPE to vision Q/K tensors. Layout: (seq, heads, dim).""" - orig_q_dtype, orig_k_dtype = q.dtype, k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed.to(orig_q_dtype), k_embed.to(orig_k_dtype) - - -class Qwen3_5MoeVisionPatchEmbed(nn.Module): - """3D convolution patch embedding for images/videos.""" - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d( - self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - -class Qwen3_5MoeVisionMLP(nn.Module): - """Feed-forward network for vision blocks.""" - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True) - self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: - return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state))) - - -class Qwen3_5MoeVisionAttention(nn.Module): - """Bidirectional attention for vision tokens with cu_seqlens support. - - Uses either: - - Eager path: splits by sequence lengths, runs attention per chunk. - - (Future) Flash Attention: single call with cu_seqlens. - - Always non-causal (is_causal=False). - """ - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True) - self.proj = nn.Linear(self.dim, self.dim) - self.scaling = self.head_dim**-0.5 - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states) - .reshape(seq_length, 3, self.num_heads, -1) - .permute(1, 0, 2, 3) - .unbind(0) - ) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) - - # Layout: (1, num_heads, seq_len, head_dim) per chunk - query_states = query_states.transpose(0, 1).unsqueeze(0) # (1, H, S, D) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - - lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - q_splits = torch.split(query_states, lengths, dim=2) - k_splits = torch.split(key_states, lengths, dim=2) - v_splits = torch.split(value_states, lengths, dim=2) - - attn_outputs = [] - for q, k, v in zip(q_splits, k_splits, v_splits): - attn_outputs.append(F.scaled_dot_product_attention(q, k, v, is_causal=False)) - - attn_output = torch.cat(attn_outputs, dim=2) # (1, H, total_S, D) - attn_output = attn_output.squeeze(0).transpose(0, 1) # (S, H, D) - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - -class Qwen3_5MoeVisionBlock(nn.Module): - """Vision transformer block: LayerNorm -> Attention -> Residual -> LayerNorm -> MLP -> Residual.""" - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6) - self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6) - self.attn = Qwen3_5MoeVisionAttention(config=config) - self.mlp = Qwen3_5MoeVisionMLP(config=config) - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - cu_seqlens=cu_seqlens, - position_embeddings=position_embeddings, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - -class Qwen3_5MoeVisionPatchMerger(nn.Module): - """Merges spatial_merge_size^2 patches into one token and projects to LLM hidden size.""" - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.hidden_size = config.hidden_size * (config.spatial_merge_size**2) - self.norm = nn.LayerNorm(config.hidden_size, eps=1e-6) - self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size) - self.act_fn = nn.GELU() - self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = self.norm(x).view(-1, self.hidden_size) - x = self.linear_fc2(self.act_fn(self.linear_fc1(x))) - return x - - -class Qwen3_5MoeVisionModel(nn.Module): - """Complete vision tower: PatchEmbed + PositionEmbed + VisionBlocks + PatchMerger. - - This module is NOT exported -- it runs in plain PyTorch. - """ - - def __init__(self, config: Qwen3_5MoeVisionConfig): - super().__init__() - self.config = config - self.spatial_merge_size = config.spatial_merge_size - self.patch_size = config.patch_size - self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size - self.dtype = None # set after loading weights - - self.patch_embed = Qwen3_5MoeVisionPatchEmbed(config=config) - self.pos_embed = nn.Embedding(config.num_position_embeddings, config.hidden_size) - self.num_grid_per_side = int(config.num_position_embeddings**0.5) - - head_dim = config.hidden_size // config.num_heads - self.rotary_pos_emb = Qwen3_5MoeVisionRotaryEmbedding(head_dim // 2) - - self.blocks = nn.ModuleList([Qwen3_5MoeVisionBlock(config) for _ in range(config.depth)]) - self.merger = Qwen3_5MoeVisionPatchMerger(config=config) - - def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: - """Compute rotary position embeddings for vision tokens.""" - merge_size = self.spatial_merge_size - max_hw = int(grid_thw[:, 1:].max().item()) - freq_table = self.rotary_pos_emb(max_hw) # (max_hw, dim//2) - device = freq_table.device - - total_tokens = int(torch.prod(grid_thw, dim=1).sum().item()) - pos_ids = torch.empty((total_tokens, 2), dtype=torch.long, device=device) - - offset = 0 - for num_frames, height, width in grid_thw: - merged_h, merged_w = height // merge_size, width // merge_size - - block_rows = torch.arange(merged_h, device=device) - block_cols = torch.arange(merged_w, device=device) - intra_row = torch.arange(merge_size, device=device) - intra_col = torch.arange(merge_size, device=device) - - row_idx = block_rows[:, None, None, None] * merge_size + intra_row[None, None, :, None] - col_idx = block_cols[None, :, None, None] * merge_size + intra_col[None, None, None, :] - - row_idx = row_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1) - col_idx = col_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1) - - coords = torch.stack((row_idx, col_idx), dim=-1) - if num_frames > 1: - coords = coords.repeat(num_frames, 1) - - num_tokens = coords.shape[0] - pos_ids[offset : offset + num_tokens] = coords - offset += num_tokens - - embeddings = freq_table[pos_ids] - embeddings = embeddings.flatten(1) - return embeddings - - def fast_pos_embed_interpolate(self, grid_thw: torch.Tensor) -> torch.Tensor: - """Bilinear interpolation of learned positional embeddings for variable image sizes.""" - grid_ts, grid_hs, grid_ws = grid_thw[:, 0], grid_thw[:, 1], grid_thw[:, 2] - device = self.pos_embed.weight.device - - idx_list: List[List[int]] = [[] for _ in range(4)] - weight_list: List[List[float]] = [[] for _ in range(4)] - - for t, h, w in zip(grid_ts, grid_hs, grid_ws): - h_idxs = torch.linspace(0, self.num_grid_per_side - 1, int(h.item())) - w_idxs = torch.linspace(0, self.num_grid_per_side - 1, int(w.item())) - - h_idxs_floor = h_idxs.int() - w_idxs_floor = w_idxs.int() - h_idxs_ceil = (h_idxs.int() + 1).clip(max=self.num_grid_per_side - 1) - w_idxs_ceil = (w_idxs.int() + 1).clip(max=self.num_grid_per_side - 1) - - dh = h_idxs - h_idxs_floor - dw = w_idxs - w_idxs_floor - - base_h = h_idxs_floor * self.num_grid_per_side - base_h_ceil = h_idxs_ceil * self.num_grid_per_side - - indices = [ - (base_h[None].T + w_idxs_floor[None]).flatten(), - (base_h[None].T + w_idxs_ceil[None]).flatten(), - (base_h_ceil[None].T + w_idxs_floor[None]).flatten(), - (base_h_ceil[None].T + w_idxs_ceil[None]).flatten(), - ] - weights = [ - ((1 - dh)[None].T * (1 - dw)[None]).flatten(), - ((1 - dh)[None].T * dw[None]).flatten(), - (dh[None].T * (1 - dw)[None]).flatten(), - (dh[None].T * dw[None]).flatten(), - ] - for i in range(4): - idx_list[i].extend(indices[i].tolist()) - weight_list[i].extend(weights[i].tolist()) - - idx_tensor = torch.tensor(idx_list, dtype=torch.long, device=device) - weight_tensor = torch.tensor(weight_list, dtype=self.pos_embed.weight.dtype, device=device) - pos_embeds = self.pos_embed(idx_tensor).to(device) * weight_tensor[:, :, None] - patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3] - - patch_pos_embeds = patch_pos_embeds.split( - [int(h.item()) * int(w.item()) for h, w in zip(grid_hs, grid_ws)] - ) - - merge_size = self.config.spatial_merge_size - patch_pos_embeds_permute = [] - for pos_embed, t, h, w in zip(patch_pos_embeds, grid_ts, grid_hs, grid_ws): - t, h, w = int(t.item()), int(h.item()), int(w.item()) - pos_embed = pos_embed.repeat(t, 1) - pos_embed = ( - pos_embed.view(t, h // merge_size, merge_size, w // merge_size, merge_size, -1) - .permute(0, 1, 3, 2, 4, 5) - .flatten(0, 4) - ) - patch_pos_embeds_permute.append(pos_embed) - - return torch.cat(patch_pos_embeds_permute) - - def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor - ) -> BaseModelOutputWithPooling: - """Run the vision tower. - - Args: - hidden_states: Raw pixel values reshaped for patch embedding. - grid_thw: Shape ``(num_images_or_videos, 3)`` -- temporal, height, width. - - Returns: - ``BaseModelOutputWithPooling`` with ``pooler_output`` containing merged features. - """ - hidden_states = self.patch_embed(hidden_states) - - pos_embeds = self.fast_pos_embed_interpolate(grid_thw) - hidden_states = hidden_states + pos_embeds - - rotary_pos_emb = self.rot_pos_emb(grid_thw) - - seq_len, _ = hidden_states.size() - hidden_states = hidden_states.reshape(seq_len, -1) - rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - position_embeddings = (emb.cos(), emb.sin()) - - cu_seqlens = torch.repeat_interleave( - grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] - ).cumsum(dim=0, dtype=torch.int32) - cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) - - for blk in self.blocks: - hidden_states = blk( - hidden_states, - cu_seqlens=cu_seqlens, - position_embeddings=position_embeddings, - ) - - merged_hidden_states = self.merger(hidden_states) - - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - -# ============================================================================= -# Multimodal Wrapper (plain PyTorch -- NOT exported) -# ============================================================================= - - -@dataclass -class Qwen3_5MoeConditionalOutput(ModelOutput): - """Output of the Qwen3.5 MoE conditional generation model.""" - - logits: Optional[torch.FloatTensor] = None - - -def compute_mrope_positions( - input_ids: torch.LongTensor, - image_grid_thw: Optional[torch.LongTensor], - video_grid_thw: Optional[torch.LongTensor], - image_token_id: int, - video_token_id: int, - vision_start_token_id: int, - spatial_merge_size: int, - attention_mask: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute 3D mRoPE position IDs for multimodal sequences. - - Standalone function usable by both the model forward and the input processor. - For each sample in the batch, scans for vision placeholder tokens and assigns - spatial (T, H, W) positions to vision tokens while text tokens get sequential - positions. - - Args: - input_ids: Token IDs, shape ``(B, S)``. - image_grid_thw: Grid dimensions ``(N_images, 3)`` with ``(T, H, W)`` per image. - video_grid_thw: Grid dimensions ``(N_videos, 3)`` with ``(T, H, W)`` per video. - image_token_id: Token ID for image placeholders. - video_token_id: Token ID for video placeholders. - vision_start_token_id: Token ID marking the start of a vision segment. - spatial_merge_size: Factor by which the vision patch merger reduces spatial dims. - attention_mask: Optional mask, shape ``(B, S)``. - - Returns: - ``(position_ids, mrope_position_deltas)`` where ``position_ids`` has - shape ``(3, B, S)`` and ``mrope_position_deltas`` has shape ``(B, 1)``. - """ - if video_grid_thw is not None: - video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0) - video_grid_thw[:, 0] = 1 - - mrope_position_deltas = [] - - if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): - total_input_ids = input_ids - if attention_mask is None: - attention_mask = torch.ones_like(total_input_ids) - position_ids = torch.ones( - 3, - input_ids.shape[0], - input_ids.shape[1], - dtype=input_ids.dtype, - device=input_ids.device, - ) - image_index, video_index = 0, 0 - attention_mask = attention_mask.to(total_input_ids.device) - - for i, ids in enumerate(total_input_ids): - ids = ids[attention_mask[i] == 1] - vision_start_indices = torch.argwhere(ids == vision_start_token_id).squeeze(1) - vision_tokens = ids[vision_start_indices + 1] - image_nums = int((vision_tokens == image_token_id).sum().item()) - video_nums = int((vision_tokens == video_token_id).sum().item()) - input_tokens = ids.tolist() - llm_pos_ids_list: list = [] - st = 0 - remain_images, remain_videos = image_nums, video_nums - - for _ in range(image_nums + video_nums): - ed_image = ( - input_tokens.index(image_token_id, st) - if image_token_id in input_tokens and remain_images > 0 - else len(input_tokens) + 1 - ) - ed_video = ( - input_tokens.index(video_token_id, st) - if video_token_id in input_tokens and remain_videos > 0 - else len(input_tokens) + 1 - ) - - if ed_image < ed_video: - t, h, w = image_grid_thw[image_index].tolist() - image_index += 1 - remain_images -= 1 - ed = ed_image - else: - t, h, w = video_grid_thw[video_index].tolist() - video_index += 1 - remain_videos -= 1 - ed = ed_video - - llm_grid_t = int(t) - llm_grid_h = int(h) // spatial_merge_size - llm_grid_w = int(w) // spatial_merge_size - text_len = ed - st - - st_idx = llm_pos_ids_list[-1].max() + 1 if llm_pos_ids_list else 0 - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - t_index = ( - torch.arange(llm_grid_t) - .view(-1, 1) - .expand(-1, llm_grid_h * llm_grid_w) - .flatten() - ) - h_index = ( - torch.arange(llm_grid_h) - .view(1, -1, 1) - .expand(llm_grid_t, -1, llm_grid_w) - .flatten() - ) - w_index = ( - torch.arange(llm_grid_w) - .view(1, 1, -1) - .expand(llm_grid_t, llm_grid_h, -1) - .flatten() - ) - llm_pos_ids_list.append( - torch.stack([t_index, h_index, w_index]) + text_len + st_idx - ) - st = ed + llm_grid_t * llm_grid_h * llm_grid_w - - if st < len(input_tokens): - st_idx = llm_pos_ids_list[-1].max() + 1 if llm_pos_ids_list else 0 - text_len = len(input_tokens) - st - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) - position_ids[..., i, attention_mask[i] == 1] = llm_positions.to( - device=position_ids.device, dtype=position_ids.dtype - ) - mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) - - mrope_position_deltas = torch.tensor( - mrope_position_deltas, device=input_ids.device - ).unsqueeze(1) - return position_ids, mrope_position_deltas - else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) - max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], device=input_ids.device, dtype=input_ids.dtype - ) - - return position_ids, mrope_position_deltas - - -def _normalize_video_grid_for_mrope( - video_grid_thw: Optional[torch.Tensor], -) -> Optional[torch.Tensor]: - if video_grid_thw is None: - return None - video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0) - video_grid_thw = video_grid_thw.clone() - video_grid_thw[:, 0] = 1 - return video_grid_thw - - -def _extract_mm_item_types_from_input_ids( - input_ids: torch.Tensor, - image_token_id: int, - video_token_id: int, - vision_start_token_id: int, -) -> List[int]: - """Return multimodal item types in prompt order for a single request.""" - flat_ids = input_ids.reshape(-1).tolist() - item_types: List[int] = [] - for idx in range(len(flat_ids) - 1): - if flat_ids[idx] != vision_start_token_id: - continue - next_token = flat_ids[idx + 1] - if next_token == image_token_id: - item_types.append(0) - elif next_token == video_token_id: - item_types.append(1) - return item_types - - -def _is_qwen_video_frame(value: Any) -> bool: - return isinstance(value, (Image.Image, torch.Tensor)) - - -def _normalize_qwen_image_items(images: Any) -> list[Any]: - if images is None: - return [] - if isinstance(images, list): - return images - return [images] - - -def _normalize_qwen_video_items(videos: Any) -> list[Any]: - if videos is None: - return [] - if isinstance(videos, VideoData): - return [videos] - if isinstance(videos, list): - if not videos: - return [] - if all(_is_qwen_video_frame(frame) for frame in videos): - return [videos] - normalized_items = [] - for item in videos: - if isinstance(item, VideoData): - normalized_items.append(item) - elif ( - isinstance(item, list) - and item - and all(_is_qwen_video_frame(frame) for frame in item) - ): - normalized_items.append(item) - else: - normalized_items.append(item) - return normalized_items - return [videos] - - -def _get_qwen_video_num_spans(video: Any) -> int: - if isinstance(video, VideoData): - return len(video.frames) - if isinstance(video, list): - if not video: - return 0 - if all(_is_qwen_video_frame(frame) for frame in video): - return len(video) - shape = getattr(video, "shape", None) - if shape is not None and len(shape) >= 4: - return int(shape[0]) - return 1 - - -def _compute_mm_item_special_counts( - mm_token_lengths: torch.Tensor, - mm_special_offsets_cu_seqlen: torch.Tensor, - mm_special_offsets: torch.Tensor, - req_idx: int, -) -> List[int]: - item_lengths = mm_token_lengths.tolist() - special_start = int(mm_special_offsets_cu_seqlen[req_idx].item()) - special_end = int(mm_special_offsets_cu_seqlen[req_idx + 1].item()) - special_offsets = mm_special_offsets[special_start:special_end].tolist() - counts: List[int] = [] - mm_offset = 0 - for item_len in item_lengths: - item_end = mm_offset + int(item_len) - num_special = sum(1 for off in special_offsets if mm_offset <= int(off) < item_end) - counts.append(num_special) - mm_offset = item_end - return counts - - -def _compute_request_mrope_delta( - mm_item_types: torch.Tensor, - mm_token_lengths: torch.Tensor, - special_counts: Sequence[int], - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - spatial_merge_size: int, -) -> int: - image_idx = 0 - video_idx = 0 - total_delta = 0 - for item_type, item_len, num_special in zip( - mm_item_types.tolist(), mm_token_lengths.tolist(), special_counts - ): - num_placeholders = int(item_len) - int(num_special) - if item_type == 0: - if image_grid_thw is None: - raise ValueError("Expected image_grid_thw for image multimodal item") - t, h, w = [int(v) for v in image_grid_thw[image_idx].tolist()] - image_idx += 1 - else: - if video_grid_thw is None: - raise ValueError("Expected video_grid_thw for video multimodal item") - t, h, w = [int(v) for v in video_grid_thw[video_idx].tolist()] - video_idx += 1 - llm_grid_t = int(t) - llm_grid_h = int(h) // spatial_merge_size - llm_grid_w = int(w) // spatial_merge_size - total_delta += max(llm_grid_t, llm_grid_h, llm_grid_w) - num_placeholders - return total_delta - - -@torch.library.custom_op("auto_deploy::qwen3_mrope_delta", mutates_args=()) -def qwen3_mrope_delta( - batch_info_host: torch.Tensor, - mm_item_cu_seqlen: torch.Tensor, - mm_item_types: torch.Tensor, - mm_token_lengths: torch.Tensor, - mm_special_offsets_cu_seqlen: torch.Tensor, - mm_special_offsets: torch.Tensor, - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - spatial_merge_size: int, -) -> torch.Tensor: - num_prefill, _, num_decode = BatchInfo(batch_info_host).get_absorbed_info() - num_seq = num_prefill + num_decode - device = mm_item_cu_seqlen.device - out = torch.zeros((num_seq, 1), dtype=torch.int32, device=device) - video_grid_norm = _normalize_video_grid_for_mrope(video_grid_thw) - img_idx = 0 - vid_idx = 0 - for req_idx in range(num_prefill): - item_start = int(mm_item_cu_seqlen[req_idx].item()) - item_end = int(mm_item_cu_seqlen[req_idx + 1].item()) - req_item_types = mm_item_types[item_start:item_end] - req_item_lengths = mm_token_lengths[item_start:item_end] - if req_item_lengths.numel() == 0: - continue - num_images = int((req_item_types == 0).sum().item()) - num_videos = int((req_item_types == 1).sum().item()) - req_image_grid = image_grid_thw[img_idx : img_idx + num_images] if num_images > 0 else None - req_video_grid = video_grid_norm[vid_idx : vid_idx + num_videos] if num_videos > 0 else None - special_counts = _compute_mm_item_special_counts( - req_item_lengths, - mm_special_offsets_cu_seqlen, - mm_special_offsets, - req_idx, - ) - out[req_idx, 0] = _compute_request_mrope_delta( - req_item_types, - req_item_lengths, - special_counts, - req_image_grid, - req_video_grid, - spatial_merge_size, - ) - img_idx += num_images - vid_idx += num_videos - return out - - -@qwen3_mrope_delta.register_fake -def qwen3_mrope_delta_fake( - batch_info_host: torch.Tensor, - mm_item_cu_seqlen: torch.Tensor, - mm_item_types: torch.Tensor, - mm_token_lengths: torch.Tensor, - mm_special_offsets_cu_seqlen: torch.Tensor, - mm_special_offsets: torch.Tensor, - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - spatial_merge_size: int, -) -> torch.Tensor: - num_prefill, _, num_decode = BatchInfo(batch_info_host).get_absorbed_info() - num_seq = num_prefill + num_decode - return torch.zeros((num_seq, 1), dtype=torch.int32, device=batch_info_host.device) - - -@torch.library.custom_op( - "auto_deploy::qwen3_mrope_delta_with_cache", mutates_args=("mrope_delta_cache",) -) -def qwen3_mrope_delta_with_cache( - batch_info_host: torch.Tensor, - slot_idx: torch.Tensor, - mm_item_cu_seqlen: Optional[torch.Tensor], - mm_item_types: Optional[torch.Tensor], - mm_token_lengths: Optional[torch.Tensor], - mm_special_offsets_cu_seqlen: Optional[torch.Tensor], - mm_special_offsets: Optional[torch.Tensor], - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - mrope_delta_cache: torch.Tensor, - spatial_merge_size: int, -) -> torch.Tensor: - num_prefill, _, num_decode = BatchInfo(batch_info_host).get_absorbed_info() - num_seq = num_prefill + num_decode - out = torch.zeros((num_seq, 1), dtype=torch.int32, device=mrope_delta_cache.device) - video_grid_norm = _normalize_video_grid_for_mrope(video_grid_thw) - if num_prefill > 0: - has_mm_metadata = all( - arg is not None - for arg in ( - mm_item_cu_seqlen, - mm_item_types, - mm_token_lengths, - mm_special_offsets_cu_seqlen, - mm_special_offsets, - ) - ) - if has_mm_metadata: - img_idx = 0 - vid_idx = 0 - for req_idx in range(num_prefill): - item_start = int(mm_item_cu_seqlen[req_idx].item()) - item_end = int(mm_item_cu_seqlen[req_idx + 1].item()) - req_item_types = mm_item_types[item_start:item_end] - req_item_lengths = mm_token_lengths[item_start:item_end] - if req_item_lengths.numel() == 0: - continue - num_images = int((req_item_types == 0).sum().item()) - num_videos = int((req_item_types == 1).sum().item()) - req_image_grid = ( - image_grid_thw[img_idx : img_idx + num_images] if num_images > 0 else None - ) - req_video_grid = ( - video_grid_norm[vid_idx : vid_idx + num_videos] if num_videos > 0 else None - ) - special_counts = _compute_mm_item_special_counts( - req_item_lengths, - mm_special_offsets_cu_seqlen, - mm_special_offsets, - req_idx, - ) - out[req_idx, 0] = _compute_request_mrope_delta( - req_item_types, - req_item_lengths, - special_counts, - req_image_grid, - req_video_grid, - spatial_merge_size, - ) - img_idx += num_images - vid_idx += num_videos - mrope_delta_cache.index_copy_( - 0, - slot_idx[:num_prefill].to(torch.long), - out[:num_prefill].to(mrope_delta_cache.dtype), - ) - if num_decode > 0: - out[num_prefill:num_seq] = mrope_delta_cache[ - slot_idx[num_prefill:num_seq].to(torch.long) - ].to(torch.int32) - return out - - -@qwen3_mrope_delta_with_cache.register_fake -def qwen3_mrope_delta_with_cache_fake( - batch_info_host: torch.Tensor, - slot_idx: torch.Tensor, - mm_item_cu_seqlen: Optional[torch.Tensor], - mm_item_types: Optional[torch.Tensor], - mm_token_lengths: Optional[torch.Tensor], - mm_special_offsets_cu_seqlen: Optional[torch.Tensor], - mm_special_offsets: Optional[torch.Tensor], - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - mrope_delta_cache: torch.Tensor, - spatial_merge_size: int, -) -> torch.Tensor: - num_prefill, _, num_decode = BatchInfo(batch_info_host).get_absorbed_info() - num_seq = num_prefill + num_decode - return torch.zeros((num_seq, 1), dtype=torch.int32, device=slot_idx.device) - - -class Qwen3_5MoeModel(nn.Module): - """Multimodal wrapper: vision tower + embedding merge + mRoPE + language model. - - This module is NOT exported. It orchestrates the vision pipeline in plain - PyTorch and calls the (potentially exported) language model with 3D - ``position_ids (3, B, S)`` so that the text model's internal ``rotary_emb`` - computes the correct mRoPE cos/sin. - """ - - def __init__(self, config: Qwen3_5MoeConfig): - super().__init__() - self.config = config - self.visual = Qwen3_5MoeVisionModel(config.vision_config) - self.language_model = Qwen3_5MoeTextModel(config.text_config) - - def get_input_embeddings(self): - return self.language_model.get_input_embeddings() - - def get_rope_index( - self, - input_ids: torch.LongTensor, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute 3D mRoPE position IDs. Delegates to ``compute_mrope_positions``.""" - return compute_mrope_positions( - input_ids=input_ids, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - image_token_id=self.config.image_token_id, - video_token_id=self.config.video_token_id, - vision_start_token_id=self.config.vision_start_token_id, - spatial_merge_size=self.config.vision_config.spatial_merge_size, - attention_mask=attention_mask, - ) - - def get_image_features( - self, pixel_values: torch.Tensor, image_grid_thw: torch.LongTensor - ) -> List[torch.Tensor]: - """Run vision tower on images and split by grid dimensions.""" - vision_output: BaseModelOutputWithPooling = self.visual( - pixel_values, grid_thw=image_grid_thw - ) - image_embeds = vision_output.pooler_output - split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - return list(torch.split(image_embeds, split_sizes)) - - def get_video_features( - self, pixel_values_videos: torch.Tensor, video_grid_thw: torch.LongTensor - ) -> List[torch.Tensor]: - """Run vision tower on videos (same as images).""" - return self.get_image_features(pixel_values_videos, video_grid_thw) - - def get_placeholder_mask( - self, - input_ids: torch.LongTensor, - inputs_embeds: torch.FloatTensor, - image_features: Optional[torch.Tensor] = None, - video_features: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Find image/video placeholder token positions in the embedding sequence.""" - special_image_mask = ( - (input_ids == self.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds) - ) - special_video_mask = ( - (input_ids == self.config.video_token_id).unsqueeze(-1).expand_as(inputs_embeds) - ) - return special_image_mask, special_video_mask - - def _select_request_chunk_multimodal_embeds( - self, - req_input_pos: int, - req_seq_len: int, - req_mm_item_types: Sequence[int], - req_mm_positions: Sequence[int], - req_mm_lengths: Sequence[int], - req_special_offsets: Sequence[int], - image_embeds_list: Optional[Sequence[torch.Tensor]], - video_embeds_list: Optional[Sequence[torch.Tensor]], - ) -> torch.Tensor: - chunk_end = req_input_pos + req_seq_len - mm_cumulative_offset = 0 - img_idx = 0 - vid_idx = 0 - chunks: list[torch.Tensor] = [] - hidden_size = self.config.text_config.hidden_size - special_offsets_set = set(int(x) for x in req_special_offsets) - - for item_type, mm_start, mm_len in zip(req_mm_item_types, req_mm_positions, req_mm_lengths): - item_mm_offset = mm_cumulative_offset - item_mm_len = int(mm_len) - item_abs_start = int(mm_start) - item_abs_end = item_abs_start + item_mm_len - overlap_start = max(req_input_pos, item_abs_start) - overlap_end = min(chunk_end, item_abs_end) - - if item_type == 0: - if image_embeds_list is None: - raise ValueError("Missing image embeddings for image multimodal item") - item_embeds = image_embeds_list[img_idx] - img_idx += 1 - elif item_type == 1: - if video_embeds_list is None: - raise ValueError("Missing video embeddings for video multimodal item") - item_embeds = video_embeds_list[vid_idx] - vid_idx += 1 - else: - raise ValueError(f"Unsupported multimodal item type: {item_type}") - - local_to_feature_idx: list[Optional[int]] = [] - feature_idx = 0 - for rel in range(item_mm_len): - if item_mm_offset + rel in special_offsets_set: - local_to_feature_idx.append(None) - else: - local_to_feature_idx.append(feature_idx) - feature_idx += 1 - - if feature_idx != item_embeds.shape[0]: - raise ValueError( - "Multimodal embedding length mismatch for Qwen3.5 item: " - f"type={item_type}, expected={feature_idx}, actual={item_embeds.shape[0]}, " - f"mm_len={item_mm_len}, item_start={item_abs_start}, " - f"special_offsets={sorted(special_offsets_set)}" - ) - - if overlap_start < overlap_end: - selected_indices = [ - local_to_feature_idx[rel] - for rel in range(overlap_start - item_abs_start, overlap_end - item_abs_start) - if local_to_feature_idx[rel] is not None - ] - if selected_indices: - chunks.append(item_embeds[selected_indices]) - - mm_cumulative_offset += item_mm_len - - if chunks: - return torch.cat(chunks, dim=0) - - device = None - dtype = None - if image_embeds_list: - device = image_embeds_list[0].device - dtype = image_embeds_list[0].dtype - elif video_embeds_list: - device = video_embeds_list[0].device - dtype = video_embeds_list[0].dtype - if device is None or dtype is None: - raise ValueError( - "Cannot build empty multimodal chunk without image or video embeddings" - ) - return torch.empty(0, hidden_size, device=device, dtype=dtype) - - def _expand_video_embeds_by_span( - self, - video_embeds_list: Optional[Sequence[torch.Tensor]], - video_grid_thw: Optional[torch.Tensor], - ) -> Optional[List[torch.Tensor]]: - if video_embeds_list is None or video_grid_thw is None: - return None - - merge = self.config.vision_config.spatial_merge_size - video_span_embeds: List[torch.Tensor] = [] - for video_embeds, grid in zip(video_embeds_list, video_grid_thw): - t, h, w = [int(v) for v in grid.tolist()] - frame_tokens = (int(h) // merge) * (int(w) // merge) - expected = int(t) * frame_tokens - if video_embeds.shape[0] != expected: - raise ValueError( - "Video embedding length mismatch in Qwen3.5 VLM forward: " - f"expected={expected}, actual={video_embeds.shape[0]}, grid={tuple(grid.tolist())}" - ) - video_span_embeds.extend(list(torch.split(video_embeds, frame_tokens, dim=0))) - return video_span_embeds - - def _build_chunked_multimodal_embeds( - self, - input_ids: torch.LongTensor, - batch_info: torch.Tensor, - cu_seqlen: torch.Tensor, - input_pos: torch.Tensor, - seq_len: torch.Tensor, - image_embeds_list: Optional[Sequence[torch.Tensor]], - video_span_embeds_list: Optional[Sequence[torch.Tensor]], - mm_item_cu_seqlen: torch.Tensor, - mm_item_types: torch.Tensor, - mm_token_positions: torch.Tensor, - mm_token_lengths: torch.Tensor, - mm_special_offsets_cu_seqlen: Optional[torch.Tensor], - mm_special_offsets: Optional[torch.Tensor], - ) -> torch.Tensor: - num_prefill_seqs = int(batch_info[0].item()) - img_idx = 0 - vid_idx = 0 - chunks: list[torch.Tensor] = [] - - for i in range(num_prefill_seqs): - item_start = int(mm_item_cu_seqlen[i].item()) - item_end = int(mm_item_cu_seqlen[i + 1].item()) - req_mm_item_types = mm_item_types[item_start:item_end].tolist() - req_mm_positions = mm_token_positions[item_start:item_end].tolist() - req_mm_lengths = mm_token_lengths[item_start:item_end].tolist() - - req_special_offsets: list[int] = [] - if mm_special_offsets_cu_seqlen is not None and mm_special_offsets is not None: - special_start = int(mm_special_offsets_cu_seqlen[i].item()) - special_end = int(mm_special_offsets_cu_seqlen[i + 1].item()) - req_special_offsets = mm_special_offsets[special_start:special_end].tolist() - - num_images = sum(item_type == 0 for item_type in req_mm_item_types) - num_videos = sum(item_type == 1 for item_type in req_mm_item_types) - req_image_embeds = ( - image_embeds_list[img_idx : img_idx + num_images] - if image_embeds_list is not None - else None - ) - req_video_embeds = ( - video_span_embeds_list[vid_idx : vid_idx + num_videos] - if video_span_embeds_list is not None - else None - ) - img_idx += num_images - vid_idx += num_videos - - req_chunk_embeds = self._select_request_chunk_multimodal_embeds( - req_input_pos=int(input_pos[i].item()), - req_seq_len=int(seq_len[i].item()), - req_mm_item_types=req_mm_item_types, - req_mm_positions=req_mm_positions, - req_mm_lengths=req_mm_lengths, - req_special_offsets=req_special_offsets, - image_embeds_list=req_image_embeds, - video_embeds_list=req_video_embeds, - ) - chunks.append(req_chunk_embeds) - - if chunks: - return torch.cat(chunks, dim=0) - - hidden_size = self.config.text_config.hidden_size - return torch.empty( - 0, hidden_size, device=input_ids.device, dtype=self.get_input_embeddings().weight.dtype - ) - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - pixel_values: Optional[torch.Tensor] = None, - pixel_values_videos: Optional[torch.Tensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - mrope_position_deltas: Optional[torch.Tensor] = None, - batch_info: Optional[torch.Tensor] = None, - **kwargs, - ) -> Qwen3_5MoeOutput: - """Multimodal forward: vision encoding + embedding merge + mRoPE + text model. - - 3D mRoPE positions are computed per request at forward time using - ``cu_seqlen`` to identify request boundaries and ``image_grid_thw`` - to derive spatial positions for multimodal requests. - - Position assembly cases: - - 1. Images present + ``batch_info`` (mixed or prefill-only with images): - iterate prefill requests via ``cu_seqlen``, call - ``compute_mrope_positions`` for multimodal requests and expand 2D - positions to 3D for text-only requests. Decode tokens get - delta-adjusted 3D expansion. - 2. Otherwise (decode-only or text-only prefill without images): expand - ``position_ids + delta`` to 3D where delta defaults to 0. - """ - inputs_embeds = self.get_input_embeddings()(input_ids) - - has_images = pixel_values is not None and image_grid_thw is not None - has_videos = pixel_values_videos is not None and video_grid_thw is not None - - image_embeds_list = None - if has_images: - image_embeds_list = [ - embeds.to(inputs_embeds.device, inputs_embeds.dtype) - for embeds in self.get_image_features(pixel_values, image_grid_thw) - ] - - video_embeds_list = None - if has_videos: - video_embeds_list = [ - embeds.to(inputs_embeds.device, inputs_embeds.dtype) - for embeds in self.get_video_features(pixel_values_videos, video_grid_thw) - ] - video_span_embeds_list = self._expand_video_embeds_by_span( - video_embeds_list, video_grid_thw - ) - - delta = mrope_position_deltas if mrope_position_deltas is not None else 0 - - vision_grid = image_grid_thw if has_images else video_grid_thw if has_videos else None - if batch_info is None: - batch_info = kwargs.get("batch_info_host") - batch_info_host = kwargs.get("batch_info_host", batch_info) - cu_seqlen = kwargs.get("cu_seqlen") - if cu_seqlen is None: - cu_seqlen = kwargs.get("cu_seqlen_host") - seq_len = kwargs.get("seq_len") - if seq_len is None and cu_seqlen is not None: - seq_len = cu_seqlen[1:] - cu_seqlen[:-1] - input_pos = kwargs.get("input_pos") - if input_pos is None: - seq_len_with_cache = kwargs.get("seq_len_with_cache") - if seq_len_with_cache is None: - seq_len_with_cache = kwargs.get("seq_len_with_cache_host") - if seq_len_with_cache is not None and seq_len is not None: - input_pos = seq_len_with_cache.to(seq_len.device) - seq_len - mm_item_cu_seqlen = kwargs.get("mm_item_cu_seqlen") - mm_token_positions = kwargs.get("mm_token_positions") - mm_token_lengths = kwargs.get("mm_token_lengths") - mm_item_types = kwargs.get("mm_item_types") - mm_special_offsets_cu_seqlen = kwargs.get("mm_special_offsets_cu_seqlen") - mm_special_offsets = kwargs.get("mm_special_offsets") - slot_idx = kwargs.get("slot_idx") - mrope_delta_cache = kwargs.get("mrope_delta_cache") - if mrope_delta_cache is None: - for key, value in kwargs.items(): - if key.endswith("_mrope_delta_cache"): - mrope_delta_cache = value - break - has_chunk_mm_layout = ( - mm_item_cu_seqlen is not None - and mm_item_types is not None - and mm_token_positions is not None - and mm_token_lengths is not None - and mm_item_cu_seqlen.numel() > 0 - and int(mm_item_cu_seqlen[-1].item()) > 0 - and mm_token_positions.numel() > 0 - and mm_token_lengths.numel() > 0 - ) - - if has_images or has_videos: - multimodal_mask = ( - ( - (input_ids == self.config.image_token_id) - | (input_ids == self.config.video_token_id) - ) - .unsqueeze(-1) - .expand_as(inputs_embeds) - ) - num_multimodal_tokens = int( - ( - (input_ids == self.config.image_token_id) - | (input_ids == self.config.video_token_id) - ) - .sum() - .item() - ) - if ( - batch_info is not None - and cu_seqlen is not None - and input_pos is not None - and seq_len is not None - and has_chunk_mm_layout - ): - multimodal_embeds = self._build_chunked_multimodal_embeds( - input_ids=input_ids, - batch_info=batch_info, - cu_seqlen=cu_seqlen, - input_pos=input_pos, - seq_len=seq_len, - image_embeds_list=image_embeds_list, - video_span_embeds_list=video_span_embeds_list, - mm_item_cu_seqlen=mm_item_cu_seqlen, - mm_item_types=mm_item_types, - mm_token_positions=mm_token_positions, - mm_token_lengths=mm_token_lengths, - mm_special_offsets_cu_seqlen=mm_special_offsets_cu_seqlen, - mm_special_offsets=mm_special_offsets, - ) - else: - if image_embeds_list is not None: - image_embeds = torch.cat(image_embeds_list, dim=0) - image_mask = ( - (input_ids == self.config.image_token_id) - .unsqueeze(-1) - .expand_as(inputs_embeds) - ) - inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) - if video_embeds_list is not None: - video_embeds = torch.cat(video_embeds_list, dim=0) - video_mask = ( - (input_ids == self.config.video_token_id) - .unsqueeze(-1) - .expand_as(inputs_embeds) - ) - inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) - multimodal_embeds = None - - if ( - multimodal_embeds is not None - and multimodal_embeds.shape[0] != num_multimodal_tokens - ): - raise ValueError( - "Multimodal embedding count mismatch in Qwen3.5 VLM forward: " - f"selected={multimodal_embeds.shape[0]}, placeholders={num_multimodal_tokens}, " - f"input_shape={tuple(input_ids.shape)}" - ) - if multimodal_embeds is not None: - inputs_embeds = inputs_embeds.masked_scatter(multimodal_mask, multimodal_embeds) - if mrope_delta_cache is not None and batch_info_host is not None and slot_idx is not None: - delta = torch.ops.auto_deploy.qwen3_mrope_delta_with_cache( - batch_info_host, - slot_idx, - mm_item_cu_seqlen, - mm_item_types, - mm_token_lengths, - mm_special_offsets_cu_seqlen, - mm_special_offsets, - image_grid_thw, - video_grid_thw, - mrope_delta_cache, - self.config.vision_config.spatial_merge_size, - ).to(input_ids.dtype) - - if ( - vision_grid is not None - and batch_info is not None - and cu_seqlen is not None - and input_pos is not None - and seq_len is not None - and has_chunk_mm_layout - ): - position_ids_3d = self._build_chunked_multimodal_positions( - input_ids, - position_ids, - delta, - batch_info, - cu_seqlen, - input_pos, - seq_len, - image_grid_thw if has_images else None, - video_grid_thw if has_videos else None, - mm_item_cu_seqlen, - mm_item_types, - mm_token_positions, - mm_token_lengths, - mm_special_offsets_cu_seqlen, - mm_special_offsets, - ) - elif vision_grid is not None and batch_info is not None and cu_seqlen is not None: - position_ids_3d = self._build_mixed_positions( - input_ids, - position_ids, - delta, - batch_info, - cu_seqlen, - image_grid_thw if has_images else None, - video_grid_thw if has_videos else None, - ) - elif vision_grid is not None: - position_ids_3d, _ = compute_mrope_positions( - input_ids=input_ids, - image_grid_thw=image_grid_thw if has_images else None, - video_grid_thw=video_grid_thw if has_videos else None, - image_token_id=self.config.image_token_id, - video_token_id=self.config.video_token_id, - vision_start_token_id=self.config.vision_start_token_id, - spatial_merge_size=self.config.vision_config.spatial_merge_size, - ) - else: - if position_ids is None: - raise ValueError("position_ids is required for text-only or decode-only forward") - is_flattened_cached_layout = position_ids.ndim == 1 or ( - position_ids.ndim == 2 and position_ids.shape[0] == 1 - ) - if is_flattened_cached_layout: - flat_position_ids = position_ids.reshape(-1) - token_delta = torch.zeros_like(flat_position_ids) - if ( - torch.is_tensor(delta) - and cu_seqlen is not None - and delta.ndim == 2 - and delta.shape[0] == cu_seqlen.numel() - 1 - ): - seq_lens = (cu_seqlen[1:] - cu_seqlen[:-1]).to(torch.long) - token_delta = torch.repeat_interleave( - delta.squeeze(-1).to(flat_position_ids.device, flat_position_ids.dtype), - seq_lens.to(flat_position_ids.device), - ) - position_ids_3d = (flat_position_ids + token_delta).view(1, 1, -1).expand(3, 1, -1) - else: - position_ids_3d = (position_ids + delta)[None].expand(3, -1, -1) - - for key in ( - "input_pos", - "mm_chunk_flat_start", - "mm_chunk_count", - "mm_item_cu_seqlen", - "mm_item_types", - "mm_token_positions", - "mm_token_lengths", - "mm_special_offsets_cu_seqlen", - "mm_special_offsets", - "mrope_delta_cache", - ): - kwargs.pop(key, None) - for key in list(kwargs.keys()): - if key.endswith("_mrope_delta_cache"): - kwargs.pop(key, None) - - return self.language_model( - inputs_embeds=inputs_embeds, - position_ids=position_ids_3d, - **kwargs, - ) - - def _build_chunked_multimodal_positions( - self, - input_ids: torch.LongTensor, - position_ids: Optional[torch.LongTensor], - delta, - batch_info: torch.Tensor, - cu_seqlen: torch.Tensor, - input_pos: torch.Tensor, - seq_len: torch.Tensor, - image_grid_thw: Optional[torch.LongTensor], - video_grid_thw: Optional[torch.LongTensor], - mm_item_cu_seqlen: torch.Tensor, - mm_item_types: torch.Tensor, - mm_token_positions: torch.Tensor, - mm_token_lengths: torch.Tensor, - mm_special_offsets_cu_seqlen: Optional[torch.Tensor], - mm_special_offsets: Optional[torch.Tensor], - ) -> torch.Tensor: - """Build 3D positions using chunk runtime metadata from the executor. - - This path is for chunked multimodal prefill where ``input_ids`` only contains the current - chunk but full multimodal tensors are still available. It reconstructs the chunk's 3D - mRoPE positions in absolute request coordinates from: - - per-request chunk start/end (`input_pos`, `seq_len`) - - per-request multimodal item layout (`mm_token_positions`, `mm_token_lengths`) - - full multimodal grids (`image_grid_thw` / `video_grid_thw`) - """ - num_prefill_seqs = batch_info[0].item() - num_prefill_tokens = batch_info[1].item() - - img_grid_idx = 0 - vid_grid_idx = 0 - prefill_3d_parts: list[torch.Tensor] = [] - normalized_video_grid_thw = _normalize_video_grid_for_mrope(video_grid_thw) - - for i in range(num_prefill_seqs): - start = cu_seqlen[i].item() - end = cu_seqlen[i + 1].item() - req_input_pos = int(input_pos[i].item()) - req_seq_len = int(seq_len[i].item()) - - item_start = int(mm_item_cu_seqlen[i].item()) - item_end = int(mm_item_cu_seqlen[i + 1].item()) - req_mm_item_types = mm_item_types[item_start:item_end].tolist() - req_mm_positions = mm_token_positions[item_start:item_end].tolist() - req_mm_lengths = mm_token_lengths[item_start:item_end].tolist() - - req_special_offsets: list[int] = [] - if mm_special_offsets_cu_seqlen is not None and mm_special_offsets is not None: - special_start = int(mm_special_offsets_cu_seqlen[i].item()) - special_end = int(mm_special_offsets_cu_seqlen[i + 1].item()) - req_special_offsets = mm_special_offsets[special_start:special_end].tolist() - - has_img = image_grid_thw is not None and len(req_mm_positions) > 0 - has_vid = normalized_video_grid_thw is not None and len(req_mm_positions) > 0 - - if has_img or has_vid: - req_img_grid = None - req_vid_grid = None - num_images = sum(item_type == 0 for item_type in req_mm_item_types) - num_videos = sum(item_type == 1 for item_type in req_mm_item_types) - if has_img: - req_img_grid = image_grid_thw[img_grid_idx : img_grid_idx + num_images] - img_grid_idx += num_images - if has_vid: - req_vid_grid = normalized_video_grid_thw[ - vid_grid_idx : vid_grid_idx + num_videos - ] - vid_grid_idx += num_videos - - pos_3d = self._compute_request_chunk_mrope_positions( - req_input_pos=req_input_pos, - req_seq_len=req_seq_len, - req_mm_item_types=req_mm_item_types, - req_mm_positions=req_mm_positions, - req_mm_lengths=req_mm_lengths, - req_special_offsets=req_special_offsets, - image_grid_thw=req_img_grid, - video_grid_thw=req_vid_grid, - dtype=input_ids.dtype, - device=input_ids.device, - ) - prefill_3d_parts.append(pos_3d) - else: - if position_ids is not None: - req_pos = position_ids[..., start:end] - else: - req_pos = torch.arange( - req_input_pos, - req_input_pos + req_seq_len, - device=input_ids.device, - dtype=input_ids.dtype, - ).unsqueeze(0) - prefill_3d_parts.append(req_pos[None].expand(3, -1, -1)) - - prefill_pos = torch.cat(prefill_3d_parts, dim=-1) - - if num_prefill_tokens < input_ids.shape[-1]: - if position_ids is None: - raise ValueError("position_ids is required when decode tokens are present") - decode_pos_2d = position_ids[..., num_prefill_tokens:] - if isinstance(delta, torch.Tensor): - gen_deltas = delta[num_prefill_seqs:] - decode_adjusted = decode_pos_2d + gen_deltas.T - else: - decode_adjusted = decode_pos_2d + delta - decode_pos_3d = decode_adjusted[None].expand(3, -1, -1) - return torch.cat([prefill_pos, decode_pos_3d], dim=-1) - - return prefill_pos - - def _compute_request_chunk_mrope_positions( - self, - req_input_pos: int, - req_seq_len: int, - req_mm_item_types: Sequence[int], - req_mm_positions: Sequence[int], - req_mm_lengths: Sequence[int], - req_special_offsets: Sequence[int], - image_grid_thw: Optional[torch.Tensor], - video_grid_thw: Optional[torch.Tensor], - dtype: torch.dtype, - device: torch.device, - ) -> torch.Tensor: - """Compute chunk-local 3D mRoPE positions for one request in absolute coordinates.""" - chunk_end = req_input_pos + req_seq_len - out = torch.empty((3, 1, req_seq_len), dtype=dtype, device=device) - special_offsets_set = set(int(x) for x in req_special_offsets) - mm_cumulative_offset = 0 - abs_cursor = 0 - comp_cursor = 0 - img_idx = 0 - vid_idx = 0 - - def fill_text(abs_start: int, abs_end: int, comp_start: int) -> None: - ov_start = max(req_input_pos, abs_start) - ov_end = min(chunk_end, abs_end) - if ov_start >= ov_end: - return - start_pos = comp_start + (ov_start - abs_start) - text_pos = torch.arange( - start_pos, start_pos + (ov_end - ov_start), device=device, dtype=dtype - ) - out[:, 0, ov_start - req_input_pos : ov_end - req_input_pos] = text_pos.unsqueeze( - 0 - ).expand(3, -1) - - def fill_vision(abs_start: int, grid: torch.Tensor, comp_start: int) -> Tuple[int, int]: - t, h, w = [int(v) for v in grid.tolist()] - llm_grid_t = int(t) - llm_grid_h = int(h) // self.config.vision_config.spatial_merge_size - llm_grid_w = int(w) // self.config.vision_config.spatial_merge_size - vision_len = llm_grid_t * llm_grid_h * llm_grid_w - - ov_start = max(req_input_pos, abs_start) - ov_end = min(chunk_end, abs_start + vision_len) - if ov_start < ov_end: - t_index = ( - torch.arange(llm_grid_t, device=device, dtype=dtype) - .view(-1, 1) - .expand(-1, llm_grid_h * llm_grid_w) - .flatten() - ) - h_index = ( - torch.arange(llm_grid_h, device=device, dtype=dtype) - .view(1, -1, 1) - .expand(llm_grid_t, -1, llm_grid_w) - .flatten() - ) - w_index = ( - torch.arange(llm_grid_w, device=device, dtype=dtype) - .view(1, 1, -1) - .expand(llm_grid_t, llm_grid_h, -1) - .flatten() - ) - positions = torch.stack([t_index, h_index, w_index]) + comp_start - local_start = ov_start - abs_start - local_end = ov_end - abs_start - out[:, 0, ov_start - req_input_pos : ov_end - req_input_pos] = positions[ - :, local_start:local_end - ] - - return vision_len, comp_start + max(llm_grid_t, llm_grid_h, llm_grid_w) - - for item_type, mm_start, mm_len in zip(req_mm_item_types, req_mm_positions, req_mm_lengths): - item_mm_offset = mm_cumulative_offset - leading_specials = 0 - while item_mm_offset + leading_specials in special_offsets_set: - leading_specials += 1 - - vision_abs_start = int(mm_start) + leading_specials - fill_text(abs_cursor, vision_abs_start, comp_cursor) - comp_cursor += vision_abs_start - abs_cursor - - if item_type == 0: - if image_grid_thw is None: - raise ValueError("Missing image_grid_thw for image multimodal item") - grid = image_grid_thw[img_idx] - img_idx += 1 - elif item_type == 1: - if video_grid_thw is None: - raise ValueError("Missing video_grid_thw for video multimodal item") - grid = video_grid_thw[vid_idx] - vid_idx += 1 - else: - raise ValueError(f"Unsupported multimodal item type: {item_type}") - - _, next_comp_cursor = fill_vision(vision_abs_start, grid, comp_cursor) - comp_cursor = next_comp_cursor - abs_cursor = int(mm_start) + int(mm_len) - mm_cumulative_offset += int(mm_len) - - fill_text(abs_cursor, chunk_end, comp_cursor) - return out - - def _build_mixed_positions( - self, - input_ids: torch.LongTensor, - position_ids: torch.LongTensor, - delta, - batch_info: torch.Tensor, - cu_seqlen: torch.Tensor, - image_grid_thw: Optional[torch.LongTensor], - video_grid_thw: Optional[torch.LongTensor], - ) -> torch.Tensor: - """Build 3D mRoPE positions for a batch with per-request granularity. - - Iterates over prefill requests using ``cu_seqlen`` boundaries. For - each request that contains vision tokens, calls - ``compute_mrope_positions`` with the matching ``image_grid_thw`` rows. - Text-only prefill requests get trivial 3D expansion. Decode tokens - are delta-adjusted uniformly. - """ - num_prefill_seqs = batch_info[0].item() - num_prefill_tokens = batch_info[1].item() - - img_grid_idx = 0 - vid_grid_idx = 0 - prefill_3d_parts: list = [] - - for i in range(num_prefill_seqs): - start = cu_seqlen[i].item() - end = cu_seqlen[i + 1].item() - req_ids = input_ids[..., start:end] - - has_img = image_grid_thw is not None and (req_ids == self.config.image_token_id).any() - has_vid = video_grid_thw is not None and (req_ids == self.config.video_token_id).any() - - if has_img or has_vid: - req_img_grid = None - req_vid_grid = None - req_item_types = _extract_mm_item_types_from_input_ids( - req_ids, - image_token_id=self.config.image_token_id, - video_token_id=self.config.video_token_id, - vision_start_token_id=self.config.vision_start_token_id, - ) - if has_img: - n_img = sum(item_type == 0 for item_type in req_item_types) - req_img_grid = image_grid_thw[img_grid_idx : img_grid_idx + n_img] - img_grid_idx += n_img - if has_vid: - n_vid = sum(item_type == 1 for item_type in req_item_types) - req_vid_grid = video_grid_thw[vid_grid_idx : vid_grid_idx + n_vid] - vid_grid_idx += n_vid - - pos_3d, _ = compute_mrope_positions( - input_ids=req_ids, - image_grid_thw=req_img_grid, - video_grid_thw=req_vid_grid, - image_token_id=self.config.image_token_id, - video_token_id=self.config.video_token_id, - vision_start_token_id=self.config.vision_start_token_id, - spatial_merge_size=self.config.vision_config.spatial_merge_size, - ) - prefill_3d_parts.append(pos_3d) - else: - req_pos = position_ids[..., start:end] - prefill_3d_parts.append((req_pos + 0)[None].expand(3, -1, -1)) - - prefill_pos = torch.cat(prefill_3d_parts, dim=-1) - - if num_prefill_tokens < input_ids.shape[-1]: - decode_pos_2d = position_ids[..., num_prefill_tokens:] - if isinstance(delta, torch.Tensor): - num_prefill_seqs_t = batch_info[0].item() - gen_deltas = delta[num_prefill_seqs_t:] - decode_adjusted = decode_pos_2d + gen_deltas.T - else: - decode_adjusted = decode_pos_2d + delta - decode_pos_3d = decode_adjusted[None].expand(3, -1, -1) - return torch.cat([prefill_pos, decode_pos_3d], dim=-1) - - return prefill_pos - - -class Qwen3_5MoeForConditionalGeneration(Qwen3_5MoePreTrainedModel): - """Top-level multimodal model: vision + language model + lm_head. - - This wraps ``Qwen3_5MoeModel`` (which contains the vision tower and the - text ``Qwen3_5MoeTextModel`` as ``language_model``) and adds an ``lm_head`` - at the top level -- matching the HF checkpoint weight layout. - """ - - config_class = Qwen3_5MoeConfig - - def __init__(self, config: Qwen3_5MoeConfig, **kwargs): - super().__init__(config) - self.model = Qwen3_5MoeModel(config) - self.lm_head = nn.Linear( - config.text_config.hidden_size, config.text_config.vocab_size, bias=False - ) - # Share lm_head with the text model so it's inside the exported graph - self.model.language_model.set_lm_head(self.lm_head) - - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self): - return self.model.language_model.get_input_embeddings() - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - self.model.language_model.set_lm_head(new_embeddings) - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - pixel_values: Optional[torch.Tensor] = None, - pixel_values_videos: Optional[torch.Tensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - **kwargs, - ) -> Qwen3_5MoeConditionalOutput: - outputs = self.model( - input_ids=input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - pixel_values=pixel_values, - pixel_values_videos=pixel_values_videos, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - **kwargs, - ) - logits = outputs.logits - return Qwen3_5MoeConditionalOutput(logits=logits) - - -# ============================================================================= -# Custom Export Info and Factory -# ============================================================================= - - -class Qwen3_5MoeTextExportInfo(TextModelExportInfo): - """Export info for mRoPE models that receive 3D position_ids ``(3, B, S)``. - - Dim 0 is always 3 (temporal, height, width) and is static; dims 1 and 2 - (batch, sequence) are dynamic. - """ - - def _init_dynamic_shape_lookup(self): - base = super()._init_dynamic_shape_lookup() - batch_size_dyn = Dim.DYNAMIC - seq_len_dyn = Dim.DYNAMIC - base["position_ids"] = {1: batch_size_dyn, 2: seq_len_dyn} - return base - - -class Qwen3_5MoeADInputProcessor: - """Qwen-specific AD input processor that emits exact multimodal spans from tokenized input.""" - - def __init__(self, base_processor): - self.base_processor = base_processor - # Bypass the generic hashing wrapper. We produce multimodal_input directly. - self.multimodal_hashing_supported = False - - def __getattr__(self, name: str): - return getattr(self.base_processor, name) - - @property - def get_num_multimodal_tokens(self): - """Delegate multimodal token counting to the wrapped Qwen HF processor.""" - if hasattr(self.processor, "_get_num_multimodal_tokens"): - return self.processor._get_num_multimodal_tokens - raise NotImplementedError( - f"get_num_multimodal_tokens not implemented for {self.__class__.__name__}. " - "Please ensure the processor exposes _get_num_multimodal_tokens." - ) - - def get_num_tokens_per_image(self, *, image: Image.Image, **kwargs) -> int: - image_size = (image.height, image.width) - return self.get_num_multimodal_tokens([image_size], **kwargs)["num_image_tokens"][0] - - def get_num_tokens_per_video(self, *, video: List[Image.Image], **kwargs) -> int: - video_size = (len(video), video[0].height, video[0].width) - num_video_tokens = self.get_num_multimodal_tokens(video_sizes=[video_size], **kwargs).get( - "num_video_tokens" - ) - if num_video_tokens is None: - raise NotImplementedError("Underlying processor does not expose num_video_tokens.") - return num_video_tokens[0] - - def get_vocab_size(self) -> Optional[int]: - """Return the tokenizer vocabulary size for Qwen multimodal hashing helpers.""" - if self.tokenizer is not None and hasattr(self.tokenizer, "vocab_size"): - return int(self.tokenizer.vocab_size) - wrapped_tokenizer = getattr(self.tokenizer, "tokenizer", None) - if wrapped_tokenizer is not None and hasattr(wrapped_tokenizer, "vocab_size"): - return int(wrapped_tokenizer.vocab_size) - processor_tokenizer = getattr(self.processor, "tokenizer", None) - if processor_tokenizer is not None and hasattr(processor_tokenizer, "vocab_size"): - return int(processor_tokenizer.vocab_size) - return None - - def get_mm_token_ids(self) -> Optional[torch.Tensor]: - if hasattr(self.processor, "mm_token_ids"): - return self.processor.mm_token_ids - sources = [ - self.processor, - getattr(self.processor, "tokenizer", None), - self.tokenizer, - getattr(self.tokenizer, "tokenizer", None), - ] - token_ids = [] - for source in sources: - if source is None: - continue - for attr in ("image_token_id", "video_token_id"): - value = getattr(source, attr, None) - if value is not None: - token_ids.append(int(value)) - if token_ids: - return torch.tensor(sorted(set(token_ids)), dtype=torch.int32) - return None - - def get_mm_special_token_ids(self) -> Optional[torch.Tensor]: - if hasattr(self.processor, "mm_special_token_ids"): - return self.processor.mm_special_token_ids - sources = [ - self.processor, - getattr(self.processor, "tokenizer", None), - self.tokenizer, - getattr(self.tokenizer, "tokenizer", None), - ] - token_ids = [] - for source in sources: - if source is None: - continue - for attr in ("vision_start_token_id", "vision_end_token_id"): - value = getattr(source, attr, None) - if value is not None: - token_ids.append(int(value)) - if token_ids: - return torch.tensor(sorted(set(token_ids)), dtype=torch.int32) - return None - - def _build_multimodal_input( - self, - token_ids: List[int], - inputs: Dict[str, Any], - ) -> Optional[Tuple[MultimodalInput, List[int], List[int]]]: - mm_data = inputs.get("multi_modal_data") - if not mm_data or not any(k in mm_data for k in ("image", "video")): - return None - - image_token_id = int(self.processor.image_token_id) - video_token_id = int(self.processor.video_token_id) - vision_start_token_id = int(self.processor.vision_start_token_id) - ids = token_ids - - starts: List[int] = [] - lengths: List[int] = [] - special_offsets: List[int] = [] - item_types: List[int] = [] - mm_union_offset = 0 - i = 0 - while i < len(ids): - if ids[i] != vision_start_token_id: - i += 1 - continue - - if i + 1 >= len(ids): - i += 1 - continue - - if ids[i + 1] == image_token_id: - item_token_id = image_token_id - item_type = 0 - elif ids[i + 1] == video_token_id: - item_token_id = video_token_id - item_type = 1 - else: - i += 1 - continue - - j = i + 1 - while j < len(ids) and ids[j] == item_token_id: - j += 1 - if j == i + 1: - i += 1 - continue - - starts.append(i) - lengths.append(j - i) - special_offsets.append(mm_union_offset) - item_types.append(item_type) - mm_union_offset += j - i - i = j - - image_items = _normalize_qwen_image_items(mm_data.get("image")) - video_items = _normalize_qwen_video_items(mm_data.get("video")) - - num_video_spans = sum(item_type == 1 for item_type in item_types) - video_span_counts = [_get_qwen_video_num_spans(video) for video in video_items] - if num_video_spans != sum(video_span_counts): - raise ValueError( - "Mismatch between Qwen video prompt spans and video inputs: " - f"spans={num_video_spans}, expected_from_videos={sum(video_span_counts)}" - ) - - if len(starts) != len(image_items) + num_video_spans: - raise ValueError( - "Mismatch between multimodal prompt spans and multimodal items: " - f"spans={len(starts)}, images={len(image_items)}, video_spans={num_video_spans}" - ) - - mm_uuids = inputs.get("multi_modal_uuids", None) - mm_hash_inputs = {} - if image_items: - mm_hash_inputs["image"] = image_items - if video_items: - mm_hash_inputs["video"] = video_items - mm_hashes, _ = apply_mm_hashes(mm_hash_inputs, mm_uuids) - - image_hashes = [hexdigest_to_int32(h) for h in mm_hashes.get("image", [])] - video_hashes = [hexdigest_to_int32(h) for h in mm_hashes.get("video", [])] - image_uuids = list((mm_uuids or {}).get("image", [None] * len(image_items))) - video_uuids = list((mm_uuids or {}).get("video", [None] * len(video_items))) - image_idx = 0 - video_idx = 0 - remaining_video_spans = video_span_counts[0] if video_span_counts else 0 - mm_hashes_flat: List[List[int]] = [] - mm_uuid_list: List[Optional[str]] = [] - for item_type in item_types: - if item_type == 0: - mm_hashes_flat.append(image_hashes[image_idx]) - mm_uuid_list.append(image_uuids[image_idx]) - image_idx += 1 - else: - if video_idx >= len(video_hashes): - raise ValueError("Video span count exceeded available video items") - mm_hashes_flat.append(video_hashes[video_idx]) - mm_uuid_list.append(video_uuids[video_idx]) - remaining_video_spans -= 1 - if remaining_video_spans == 0: - video_idx += 1 - remaining_video_spans = ( - video_span_counts[video_idx] if video_idx < len(video_span_counts) else 0 - ) - return ( - MultimodalInput.from_components( - mm_hashes_flat, - starts, - lengths, - mm_uuid_list if mm_uuids is not None else None, - ), - special_offsets, - item_types, - ) - - def __call__(self, inputs, sampling_params): - token_ids, extra_processed_inputs = self.base_processor(inputs, sampling_params) - if "multi_modal_data" not in inputs: - return token_ids, extra_processed_inputs - - built = self._build_multimodal_input(token_ids, inputs) - if built is None: - return token_ids, extra_processed_inputs - - multimodal_input, special_offsets, item_types = built - if extra_processed_inputs is None: - extra_processed_inputs = {} - extra_processed_inputs["multimodal_input"] = multimodal_input - multimodal_data = extra_processed_inputs.get("multimodal_data", {}) - multimodal_data["layout_metadata"] = { - "special_token_offsets": torch.tensor(special_offsets, dtype=torch.int32), - "item_types": torch.tensor(item_types, dtype=torch.int32), - } - extra_processed_inputs["multimodal_data"] = multimodal_data - return token_ids, extra_processed_inputs - - -@ModelFactoryRegistry.register("Qwen3_5MoeForConditionalGeneration") -class Qwen3_5MoeFactory(AutoModelForImageTextToTextFactory): - """Factory for Qwen3.5 MoE that uses 3D mRoPE position_ids export info.""" - - def get_export_infos(self, model: nn.Module): - return [Qwen3_5MoeTextExportInfo.from_autoinferred(model)] - - def init_input_processor(self, base): - return Qwen3_5MoeADInputProcessor(base) - - -# ============================================================================= -# Registration -# ============================================================================= - -AutoConfig.register("qwen3_5_moe", Qwen3_5MoeConfig) -AutoConfig.register("qwen3_5_moe_text", Qwen3_5MoeTextConfig) - -AutoModelForCausalLMFactory.register_custom_model_cls("Qwen3_5MoeTextConfig", Qwen3_5MoeForCausalLM) -Qwen3_5MoeFactory.register_custom_model_cls("Qwen3_5MoeConfig", Qwen3_5MoeForConditionalGeneration) diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/hidden_states.py b/tensorrt_llm/_torch/auto_deploy/transform/library/hidden_states.py index 4a6ddfa32197..f86b7da8abbc 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/hidden_states.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/hidden_states.py @@ -130,16 +130,26 @@ def _get_layer_number(lin_node: Node) -> Optional[int]: # using a future residual add node for the next layer. # 2. is the last add node in a 1 user chain (for last layer or layers with no following residual add) # This stops us before we go to the next layer. + # + # IR-aware models (e.g. modeling_nemotron_h.py post-IR-fication) insert + # ``torch.ops.auto_deploy.all_reduce`` between the closing linear and the residual add. + # That op is identity at world_size=1 and a sharding-injected collective otherwise, so + # walk transparently through it without recording it as the residual add. res_node = lin_node_closing + last_add: Optional[Node] = None while len(res_node.users) == 1: user_node = list(res_node.users)[0] - if not is_op(user_node, torch.ops.aten.add): + if is_op(user_node, torch.ops.aten.add): + last_add = user_node + res_node = user_node + elif is_op(user_node, torch.ops.auto_deploy.all_reduce): + res_node = user_node + else: break - res_node = user_node - if is_op(res_node, torch.ops.aten.add): + if last_add is not None: # this stores the last residual add node encountered for each layer - residual_add_nodes[layer_number] = res_node + residual_add_nodes[layer_number] = last_add return residual_add_nodes diff --git a/tests/unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py b/tests/unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py index 32a7adcbf4cc..d20b4e86489b 100644 --- a/tests/unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py +++ b/tests/unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py @@ -1006,6 +1006,18 @@ def _run_pattern_detection_job( fused_weight_dims=None, ) ) + elif is_op(node, torch.ops.auto_deploy.torch_rmsnorm_gated): + expected_transformations.append( + WeightShardingInfo( + target_node=node.name, + split_dim=SplitDimension.COLUMN, + config=config, + dist_op=None, + min_local_shape=1, + layer_type=LayerType.SSM, + fused_weight_dims=None, + ) + ) elif len(node.args) > 1 and ( "norm_weight" in node.args[0].name or "a_log" in node.args[0].name ): From 66b6fb0fce42d9effbc7faacbace79cabaf6c1b8 Mon Sep 17 00:00:00 2001 From: YihuiLu512 <269394165+YihuiLu512@users.noreply.github.com> Date: Mon, 25 May 2026 18:18:06 +0800 Subject: [PATCH 016/308] [TRTLLM-12942][ci] Dedup misc unit tests on B200 (#14525) Signed-off-by: Yihui Lu <269394165+YihuiLu512@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_b200.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index f4f375871e2a..def40f798183 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -91,7 +91,6 @@ l0_b200: - unittest/_torch/compilation - unittest/_torch/debugger - unittest/_torch/executor - - unittest/_torch/misc # ------------- modules (non-MoE) --------------- - unittest/_torch/modules/test_mla_helix.py - unittest/_torch/modules/test_fused_add_rms_norm_quant.py @@ -278,6 +277,9 @@ l0_b200: backend: pytorch tests: # ------------- PyTorch tests --------------- + # Covered by H100 pre_merge for primary HW-agnostic signal; keep B200 runtime + # canary in post_merge for CUDA IPC / virtual memory / profiling paths. + - unittest/_torch/misc - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] From 4517988cb39689f5462000f9251c5255a0492825 Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 25 May 2026 18:48:40 +0800 Subject: [PATCH 017/308] [None][infra] Waive 9 failed cases for main in post-merge (#14515) Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index cfb251cbc87c..18831e91218b 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -19,6 +19,7 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp] SKIP (https://nvbugs/6215736) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6018046) @@ -48,7 +49,9 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mt accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6050489) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6162115) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6198785) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6185146) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6195110) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6184914) @@ -101,6 +104,9 @@ accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (htt accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[8gpus] SKIP (https://nvbugs/6162328) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) +accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_bf16 SKIP (https://nvbugs/6211185) +accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 SKIP (https://nvbugs/6211185) +accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_nvfp4 SKIP (https://nvbugs/6211185) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) @@ -140,6 +146,7 @@ accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-c accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6163033) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6162121) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype SKIP (https://nvbugs/6076767) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_on] SKIP (https://nvbugs/6094068) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] SKIP (https://nvbugs/6116088) @@ -346,6 +353,8 @@ perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwel perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] SKIP (https://nvbugs/6162857) +stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6207678) +stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6207678) stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) From 546a5b09125698ffad5a65edd98690410f8eab99 Mon Sep 17 00:00:00 2001 From: Tianyu Xiong <117647511+tianyuxbear@users.noreply.github.com> Date: Mon, 25 May 2026 19:05:26 +0800 Subject: [PATCH 018/308] [https://nvbugs/6182617][fix] Restore K2.5 multimodal dep8 accuracy test on transformers 5.5.x (#14392) Signed-off-by: Tianyu Xiong <117647511+tianyuxbear@users.noreply.github.com> --- .../_torch/models/modeling_kimi_k25.py | 36 +++++++++++++++++++ .../test_llm_api_pytorch_multimodal.py | 6 +++- .../test_lists/test-db/l0_dgx_b200.yml | 2 +- tests/integration/test_lists/waives.txt | 1 - 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index 6c536effb8f8..314ba47fdc46 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -1052,6 +1052,13 @@ def __init__( config, "media_placeholder_token_id", _MEDIA_PLACEHOLDER_TOKEN_ID ) + # transformers 5.5.x ``AutoTokenizer`` may route K2.5 to the Rust + # fast backend, which BPE-splits ``<|media_pad|>`` / ``<|im_user|>`` + # / etc. instead of mapping them to their canonical IDs. Force the + # K2.5 slow ``TikTokenTokenizer`` for deterministic tokenization. + # See NVBug 6182617. + self._ensure_k25_slow_tokenizer() + @property def config(self) -> PretrainedConfig: return self._config @@ -1168,6 +1175,35 @@ def get_num_tokens_per_video(self, *, video: List, **kwargs) -> int: total_tokens += self.get_num_tokens_per_image(image=chunk[0]) return total_tokens + def _ensure_k25_slow_tokenizer(self) -> None: + """Override ``self._tokenizer`` and ``self._processor.tokenizer`` + with the model's slow ``TikTokenTokenizer``. + + Done unconditionally because transformers 5.5.x's ``AutoTokenizer`` + sometimes returns a Rust fast backend that BPE-splits K2.5 special + tokens instead of preserving their canonical IDs. The slow class' + ``tokens_trie`` always splits them correctly. See NVBug 6182617. + """ + from transformers.dynamic_module_utils import get_class_from_dynamic_module + + slow_cls = get_class_from_dynamic_module( + "tokenization_kimi.TikTokenTokenizer", + self._model_path, + ) + slow_tok = slow_cls.from_pretrained(self._model_path, trust_remote_code=True) + + logger.info( + "K2.5 InputProcessor forcing slow TikTokenTokenizer " + "(originally %s). See NVBug 6182617.", + type(self._tokenizer).__name__, + ) + + self._tokenizer = slow_tok + # Image-only path uses ``self._processor.tokenizer`` (an + # independent instance from ``AutoProcessor``); swap it too. + if getattr(self._processor, "tokenizer", None) is not None: + self._processor.tokenizer = slow_tok + @torch.inference_mode() def __call__( self, diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 39ce3d05d54d..4638f01ecdda 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -524,9 +524,13 @@ class TestKimiK25(LlmapiAccuracyTestHarness): ) def test_nvfp4(self, ep_size, attention_dp): """NVFP4 accuracy on MMMU benchmark (8x B200).""" + # Do not pass ``max_num_tokens`` here: keep it at the LLM default + # (8192) so the resolved ``moe_max_num_tokens`` stays at + # ``8192 * dp_size`` and the per-call fused_moe workspace fits + # within activation headroom. Raising it pushed the workspace + # past the fragmented allocator budget on dep8 (NVBug 6182617). with LLM( self.MODEL_PATH, - max_num_tokens=self.MAX_NUM_TOKENS, kv_cache_config=self.kv_cache_config, tensor_parallel_size=8, pipeline_parallel_size=1, diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 92e3d3d0ddeb..6eec1c676946 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -154,7 +154,7 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] TIMEOUT (60) - - accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (60) + - accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (120) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-trtllm] TIMEOUT (60) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 18831e91218b..3d8bc423f454 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -161,7 +161,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_firs accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6189416) -accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] SKIP (https://nvbugs/6182617) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype SKIP (https://nvbugs/6114464) From 63b1d8d9b2bddfa49047f00eab4945961184cc26 Mon Sep 17 00:00:00 2001 From: Faraz <58580514+farazkh80@users.noreply.github.com> Date: Mon, 25 May 2026 12:25:12 -0400 Subject: [PATCH 019/308] =?UTF-8?q?[None][feat]=20FlashInfer=20NVFP4=20MoE?= =?UTF-8?q?=20backend=20(SM120/SM121)=20for=20Nemotron=20=E2=80=A6=20(#137?= =?UTF-8?q?73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: list <58580514+farazkh80@users.noreply.github.com> --- .../modules/fused_moe/MOE_DEVELOPER_GUIDE.md | 3 +- .../_torch/modules/fused_moe/__init__.py | 2 + .../_torch/modules/fused_moe/create_moe.py | 39 ++- .../fused_moe/fused_moe_cute_dsl_b12x.py | 285 ++++++++++++++++++ .../_torch/modules/fused_moe/quantization.py | 158 ++++++++++ .../test_lists/test-db/l0_rtx_pro_6000.yml | 1 + .../_torch/modules/moe/moe_test_utils.py | 38 ++- .../moe/test_cute_dsl_b12x_moe_backend.py | 207 +++++++++++++ .../_torch/modules/moe/test_moe_backend.py | 9 +- .../_torch/modules/moe/test_moe_module.py | 9 + 10 files changed, 743 insertions(+), 8 deletions(-) create mode 100644 tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl_b12x.py create mode 100644 tests/unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py diff --git a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md index 06735afe0564..2af1781c0d60 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -147,6 +147,7 @@ Still on old path (standalone, with embedded communication): | `fused_moe_deepgemm.py` | `DeepGemmFusedMoE` | SM100/SM103 | FP8 Block Scales on Blackwell | `EXTERNAL_COMM` | | `fused_moe_densegemm.py` | `DenseGEMMFusedMoE` | SM100/SM103 | NVFP4 min-latency; CuTe DSL dense GEMM packs all experts into one matrix (vs Cutlass per-expert scatter), efficient for small token counts | `EXTERNAL_COMM` | | `fused_moe_cute_dsl.py` | `CuteDslFusedMoE` | SM100/SM103 | High throughput NVFP4, generally faster than Cutlass | `EXTERNAL_COMM` | +| `fused_moe_cute_dsl_b12x.py` | `CuteDslB12xFusedMoE` | SM120/SM121 | NVFP4 hybrid CUTLASS-prefill / FlashInfer NVFP4 MoE decode — best perf on RTX PRO 6000 (SM120) and DGX Spark (SM121); select via the `CUTEDSL` backend path (auto-promoted when flashinfer is importable) | `EXTERNAL_COMM` | | `mega_moe/mega_moe_deepgemm.py` | `MegaMoEDeepGemm` | SM100/SM103 | W4A8_MXFP4_MXFP8 via DeepGEMM `fp8_fp4_mega_moe` fused dispatch+GEMM+act+GEMM+combine kernel; requires `hidden_size % 512 == 0` | `FUSED_COMM` | | `fused_moe_triton.py` | `TritonFusedMoE` | SM90 only | GPT-OSS on Hopper (requires `swiglu_gptoss_style=True`) | (legacy path) | | `fused_moe_wide_ep.py` | `WideEPMoE` | All GPUs | Deprecating — use ConfigurableMoE instead | (legacy path) | @@ -192,7 +193,7 @@ Each backend's `can_implement(quant_algo, dtype_activation, swiglu_gptoss_style, | Unquantized (BF16/FP16) | Y (SM80+) | N | N | N | N | N | Y (SM90, BF16) | Y | Y | | FP8 QDQ | Y (SM89+) | N | N | N | N | N | Y (SM90) | Y | Y | | FP8 Block Scales | Y (SM90, SM120) | Y (SM100/103) | Y (SM100/103) | N | Y (SM100/103) | N | N | Y | Y | -| NVFP4 | Y (SM100/103/120/121) | Y (SM100/103) | N | Y (SM100/103) | Y (SM100/103) | N | N | Y | Y | +| NVFP4 | Y (SM100/103/120/121) | Y (SM100/103) | N | Y (SM100/103) | Y (SM100/103/120/121) | N | N | Y | Y | | W4A8 NVFP4 FP8 | N | Y (SM100/103) | N | N | N | N | N | N | N | | W4A16 MXFP4 | Y (SM90) | Y (SM100/103) | N | N | N | N | Y (SM90) | N | N | | W4A8 MXFP4 FP8 | Y (SM100/103) | Y (SM100/103) | N | N | N | N | Y (SM90) | N | N | diff --git a/tensorrt_llm/_torch/modules/fused_moe/__init__.py b/tensorrt_llm/_torch/modules/fused_moe/__init__.py index 4b957c86246c..5cf48dc4fe2c 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/__init__.py +++ b/tensorrt_llm/_torch/modules/fused_moe/__init__.py @@ -1,5 +1,6 @@ from .create_moe import create_moe, get_moe_cls from .fused_moe_cute_dsl import CuteDslFusedMoE +from .fused_moe_cute_dsl_b12x import CuteDslB12xFusedMoE from .fused_moe_cutlass import CutlassFusedMoE from .fused_moe_triton import TritonFusedMoE from .fused_moe_trtllm_gen import TRTLLMGenFusedMoE @@ -26,6 +27,7 @@ "BaseMoeRoutingMethod", "create_load_balanced_logits", "create_moe", + "CuteDslB12xFusedMoE", "CuteDslFusedMoE", "CutlassFusedMoE", "DeepSeekV3MoeRoutingMethod", diff --git a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py index 264580986a0f..be26f03abeb0 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py @@ -12,6 +12,7 @@ from ...utils import ActivationType, AuxStreamType from .configurable_moe import ConfigurableMoE from .fused_moe_cute_dsl import CuteDslFusedMoE +from .fused_moe_cute_dsl_b12x import CuteDslB12xFusedMoE from .fused_moe_cutlass import CutlassFusedMoE from .fused_moe_deepgemm import DeepGemmFusedMoE from .fused_moe_densegemm import DenseGEMMFusedMoE @@ -40,6 +41,29 @@ def get_moe_cls( if quant_config is not None and ( quant_config.quant_mode.has_fp8_block_scales() or quant_config.quant_mode.has_nvfp4()): + # On SM120 / SM121 + NVFP4 the cuteDSL family member is the + # hybrid CUTLASS-prefill / FlashInfer NVFP4 MoE decode backend + # (CuteDslB12xFusedMoE). Prefer it when flashinfer is importable; + # otherwise fall through to CuteDslFusedMoE for SM100 / SM103. + if quant_config.quant_mode.has_nvfp4(): + from tensorrt_llm._utils import get_sm_version + sm_version = get_sm_version() + if sm_version in CuteDslB12xFusedMoE._SUPPORTED_SM_VERSIONS: + try: + import flashinfer # noqa: F401 + logger.info( + "Selecting CuteDslB12xFusedMoE for hybrid " + "CUTLASS-prefill / FlashInfer NVFP4 MoE decode " + "(SM%d + NVFP4).", + sm_version, + ) + return CuteDslB12xFusedMoE + except ImportError: + logger.warning( + "CuteDslB12xFusedMoE eligible (SM%d + NVFP4) " + "but flashinfer is not importable; using CuteDslFusedMoE.", + sm_version, + ) return CuteDslFusedMoE else: logger.warning( @@ -280,7 +304,10 @@ def create_moe_backend( without_comm=without_comm, activation_type=activation_type, ) - elif moe_cls == CutlassFusedMoE: + elif moe_cls is CutlassFusedMoE: + # CuteDslFusedMoE, DeepGemmFusedMoE, and CuteDslB12xFusedMoE + # also subclass CutlassFusedMoE but have narrower constructors, so + # they take their own branches below. return moe_cls( routing_method=routing_method, num_experts=num_experts, @@ -331,7 +358,9 @@ def create_moe_backend( layer_idx=layer_idx, activation_type=activation_type, ) - elif moe_cls == CuteDslFusedMoE: + elif moe_cls in (CuteDslFusedMoE, CuteDslB12xFusedMoE): + # CuteDslB12xFusedMoE subclasses CuteDslFusedMoE and shares + # its narrower constructor (no bias / swiglu_alpha-beta-limit args). return moe_cls( routing_method=routing_method, num_experts=num_experts, @@ -492,9 +521,11 @@ def create_moe( enable_configurable_moe = os.environ.get("ENABLE_CONFIGURABLE_MOE", "1") == "1" - if enable_configurable_moe or moe_cls == CuteDslFusedMoE: + if enable_configurable_moe or moe_cls in (CuteDslFusedMoE, + CuteDslB12xFusedMoE): if moe_cls in (DeepGemmFusedMoE, TRTLLMGenFusedMoE, CuteDslFusedMoE, - CutlassFusedMoE, DenseGEMMFusedMoE, MegaMoEDeepGemm): + CuteDslB12xFusedMoE, CutlassFusedMoE, DenseGEMMFusedMoE, + MegaMoEDeepGemm): return ConfigurableMoE( routing_method=routing_method, num_experts=num_experts, diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl_b12x.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl_b12x.py new file mode 100644 index 000000000000..392c076d954f --- /dev/null +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl_b12x.py @@ -0,0 +1,285 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple, Union + +import torch + +from tensorrt_llm._utils import get_sm_version, nvtx_range +from tensorrt_llm.models.modeling_utils import QuantAlgo + +from ...utils import ActivationType, Fp4QuantizedTensor +from .fused_moe_cute_dsl import CuteDslFusedMoE +from .fused_moe_cutlass import CutlassFusedMoE +from .interface import _warn_and_return + +# Shared MoE output buffer pool, keyed by (max_num_tokens, hidden_size, dtype, +# device). ``B12xMoEWrapper.__init__`` allocates a private +# ``(max_num_tokens, hidden_size)`` output tensor per instance; with one +# wrapper per MoE layer that is ``num_layers * max_num_tokens * hidden_size`` +# bytes of GPU memory holding identical-shape buffers that are written +# sequentially. We fold them into a single shared buffer because MoE layers +# run sequentially on the same CUDA stream, and the wrapper consumes its +# previous output before the next layer is dispatched. +_SHARED_MOE_OUTPUT_BUF: dict = {} + +# ActivationType -> b12x activation string. b12x currently exposes "relu2" +# (Nemotron-style x * relu(x)) and "silu" (SwiGLU-style x * silu(gate)). +_ACTIVATION_MAP = { + ActivationType.Relu2: "relu2", + ActivationType.Swiglu: "silu", +} + + +class CuteDslB12xFusedMoE(CuteDslFusedMoE): + """Hybrid CUTLASS-prefill / b12x-decode NVFP4 fused-MoE backend for SM120 / SM121. + + Member of the cuteDSL backend family: the decode kernel + (``flashinfer.B12xMoEWrapper.run``) is JIT-compiled CuTe DSL, so the + backend slots in next to :class:`CuteDslFusedMoE` (which targets SM100 / + SM103). The hybrid prefill path still routes through the C++ CUTLASS + NVFP4 GroupGEMM via explicit :class:`CutlassFusedMoE` method calls; the + parent class on the MRO does not change which kernels execute, only + where the b12x backend sits in the family. + + Composition (see ``MOE_DEVELOPER_GUIDE.md`` for the full explainer): + + - **Prefill (``m >= _PREFILL_VIA_CUTLASS_THRESHOLD``)** explicitly + invokes :class:`CutlassFusedMoE` NVFP4 GroupGEMM. The b12x kernel's + 12-CTA-per-token MMA pattern is suboptimal at large ``m``. + - **Decode (``m < _PREFILL_VIA_CUTLASS_THRESHOLD``)** dispatches to + FlashInfer's ``B12xMoEWrapper.run`` — a kernel purpose-built for + ``m=1`` / small routed-row counts. + + NVFP4 weights are loaded via :class:`NVFP4CuteDslB12xFusedMoEMethod` + (an :class:`NVFP4CutlassFusedMoEMethod` subclass returned by + ``_get_quant_method``). The inherited CUTLASS NVFP4 layout is finalised + by the base class, and the b12x-shaped tensors (un-normalised FP8 SF, + ``convert_sf_to_mma_layout`` reshape, ``B12xMoEWrapper`` instance) are + materialised on top by the quant method's ``post_load_weights``. Both + layouts coexist in memory and the dispatcher picks per call based on + ``x.shape[0]``. + + CUDA graph capture only covers decode, so captured graphs always replay + the b12x path; eager prefill always runs CUTLASS — there is no graph + capture conflict. + + The backend hard-rejects EP (b12x has no dispatch / combine kernel), + MoE alltoall, ``Fp4QuantizedTensor`` input, ``swiglu_gptoss_style`` + biased SwiGLU, and activations outside ``{Relu2, Swiglu}``. It is + selected on the ``CUTEDSL`` MoE path when SM120 / SM121 + NVFP4 + + flashinfer-importable gates pass (see ``create_moe.get_moe_cls``). + """ + + # SM versions on which the FlashInfer b12x NVFP4 MoE kernel is available. + # SM120 = desktop Blackwell (RTX 5090 / GB202); SM121 = GB10 / DGX Spark. + _SUPPORTED_SM_VERSIONS = frozenset({120, 121}) + + # Prefill chunks (``x.shape[0] >= threshold``) route via CUTLASS NVFP4 + # GroupGEMM; decode (``x.shape[0] < threshold``) uses b12x. 64 cleanly + # separates conc=1 prefill (m=2048 with ``max_num_tokens=2048``) from + # decode (m=1) and stays robust against future chunked-prefill splits + # that might shrink prefill chunk size. + _PREFILL_VIA_CUTLASS_THRESHOLD = 64 + + @classmethod + def can_implement( + cls, + quant_algo: Optional[QuantAlgo], + dtype_activation: torch.dtype = torch.bfloat16, + swiglu_gptoss_style: bool = False, + ) -> Tuple[bool, Optional[str]]: + sm_version = get_sm_version() + if sm_version not in cls._SUPPORTED_SM_VERSIONS: + sm_list = "/".join(f"SM{v}" for v in sorted(cls._SUPPORTED_SM_VERSIONS)) + return _warn_and_return(f"CuteDslB12xFusedMoE requires {sm_list}, got SM{sm_version}") + if quant_algo != QuantAlgo.NVFP4: + return _warn_and_return( + f"CuteDslB12xFusedMoE only supports NVFP4 quantization " + f"(got quant_algo={quant_algo})" + ) + if dtype_activation not in {torch.float16, torch.bfloat16}: + return _warn_and_return( + f"CuteDslB12xFusedMoE NVFP4 requires float16 or bfloat16 " + f"activation dtype (got {dtype_activation})" + ) + if swiglu_gptoss_style: + return _warn_and_return("CuteDslB12xFusedMoE does not support swiglu_gptoss_style") + return True, None + + def __init__(self, *args, **kwargs): + # ``ModelConfig`` is consumed by the inherited ``__init__`` for cache + # / mapping setup but isn't kept on ``self``. b12x's wrapper needs the + # ``use_cuda_graph`` flag at construction time, so capture it here + # before delegating. + model_config = kwargs.get("model_config", None) + self._b12x_use_cuda_graph = bool(getattr(model_config, "use_cuda_graph", False)) + + super().__init__(*args, **kwargs) + + # b12x has no expert-parallel dispatch/combine kernel, so EP must be + # disabled. dp_size > 1 implies the alltoall path which b12x can't run. + if self.ep_size != 1: + raise ValueError( + f"CuteDslB12xFusedMoE requires ep_size == 1 " + f"(got ep_size={self.ep_size}); use --moe_backend CUTLASS for EP." + ) + if self.enable_alltoall: + raise ValueError("CuteDslB12xFusedMoE does not support MoE alltoall communication.") + if self.activation_type not in _ACTIVATION_MAP: + supported = ", ".join(a.name for a in _ACTIVATION_MAP) + raise ValueError( + f"CuteDslB12xFusedMoE does not support activation " + f"{ActivationType(self.activation_type).name}; " + f"supported: {supported}." + ) + + self._b12x_weights: Optional[dict] = None + self.b12x_wrapper = None + + def _get_quant_method(self): + # Route NVFP4 to the b12x-aware quant method so weight prep + # (SF un-normalization, ``convert_sf_to_mma_layout``, + # ``B12xMoEWrapper`` instantiation) lives next to the rest of the + # NVFP4 quant-method family, while every other quant algo (and the + # unquantized fallback) continues to resolve via the parent. + if ( + self.quant_config is not None + and self.quant_config.layer_quant_mode.has_any_quant(exclude_kv_cache=True) + and self.quant_config.layer_quant_mode.has_nvfp4() + ): + from .quantization import NVFP4CuteDslB12xFusedMoEMethod + + return NVFP4CuteDslB12xFusedMoEMethod() + return super()._get_quant_method() + + def _route_to_cutlass(self, x) -> bool: + """Return ``True`` iff this call should fall back to the inherited + CUTLASS path (prefill chunk). ``Fp4QuantizedTensor`` inputs always + stay on the b12x path (which rejects them) so the existing error + message is preserved.""" + return isinstance(x, torch.Tensor) and x.shape[0] >= self._PREFILL_VIA_CUTLASS_THRESHOLD + + # ``post_load_weights`` is inherited from ``CutlassFusedMoE`` and + # dispatches to ``self.quant_method.post_load_weights(self)`` — for this + # backend ``self.quant_method`` is ``NVFP4CuteDslB12xFusedMoEMethod`` + # (see ``_get_quant_method`` override), which performs the SF un-normalization, + # ``convert_sf_to_mma_layout`` reshape, ``B12xMoEWrapper`` instantiation, + # and the cross-layer shared output buffer dance. The wrapper and the + # bundled weight dict are attached to this module as ``self.b12x_wrapper`` + # / ``self._b12x_weights``, which the decode path below consumes. + + @nvtx_range("[b12x] quantize_input") + def quantize_input( + self, + x: Union[torch.Tensor, Fp4QuantizedTensor], + post_quant_comm: bool = True, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Hybrid dispatch entrypoint for activation handling. + + Prefill chunks (``x.shape[0] >= _PREFILL_VIA_CUTLASS_THRESHOLD``) take + the inherited :meth:`CutlassFusedMoE.quantize_input` path so the + downstream ``run_moe`` can call CUTLASS NVFP4 GroupGEMM. Decode + chunks pass through unchanged because b12x quantizes activations + internally (consumes a bf16 / fp16 ``x`` and produces its own scale + factors). + """ + if self._route_to_cutlass(x): + return CutlassFusedMoE.quantize_input( + self, x, post_quant_comm=post_quant_comm, **kwargs + ) + if isinstance(x, Fp4QuantizedTensor): + raise ValueError( + "CuteDslB12xFusedMoE does not accept Fp4QuantizedTensor input " + "on the b12x decode path; b12x performs its own input quantization." + ) + return x, None + + @nvtx_range("[b12x] run_moe") + def run_moe( + self, + x: torch.Tensor, + token_selected_experts: torch.Tensor, + token_final_scales: torch.Tensor, + x_sf: Optional[torch.Tensor] = None, + is_sf_swizzled: bool = True, + output_dtype: Optional[torch.dtype] = None, + tuner_num_tokens: Optional[int] = None, + tuner_top_k: Optional[int] = None, + moe_output: Optional[torch.Tensor] = None, + enable_alltoall: Optional[bool] = None, + ) -> torch.Tensor: + if self._route_to_cutlass(x): + # ``CutlassFusedMoE.run_moe`` forwards ``output_dtype`` straight + # into the C++ ``trtllm::fused_moe`` op, which requires a concrete + # high-precision ``ScalarType`` (uint8 / FP4-packed activations are + # rejected at the kernel epilogue with "Invalid output type Byte"). + # Schedulers that drive ``run_moe`` directly (the KV-cache capacity + # probe, for one) leave ``output_dtype`` unset, so fall back to + # ``x.dtype`` if it is a real compute dtype, else bf16. Mirrors the + # ``forward_chunk`` convention while staying safe for the FP4 + # quant-input path (``x`` is uint8 after ``quantize_input``). + _HIGH_PRECISION = {torch.float16, torch.bfloat16, torch.float32} + cutlass_output_dtype = output_dtype + if cutlass_output_dtype is None: + cutlass_output_dtype = ( + x.dtype + if isinstance(x, torch.Tensor) and x.dtype in _HIGH_PRECISION + else torch.bfloat16 + ) + return CutlassFusedMoE.run_moe( + self, + x, + token_selected_experts=token_selected_experts, + token_final_scales=token_final_scales, + x_sf=x_sf, + is_sf_swizzled=is_sf_swizzled, + output_dtype=cutlass_output_dtype, + tuner_num_tokens=tuner_num_tokens, + tuner_top_k=tuner_top_k, + moe_output=moe_output, + enable_alltoall=enable_alltoall, + ) + if self.b12x_wrapper is None or self._b12x_weights is None: + raise RuntimeError( + "CuteDslB12xFusedMoE.run_moe called before process_weights_after_loading completed." + ) + if x_sf is not None: + raise ValueError( + "CuteDslB12xFusedMoE expects unquantized input (x_sf=None) " + "on the b12x decode path; got a precomputed scale factor." + ) + + # Annotate the kwargs spread + wrapper entry separately so we can + # attribute the per-layer Python dispatch cost vs. the kernel cost. + with nvtx_range("[b12x] wrapper.run"): + out = self.b12x_wrapper.run( + x=x, + token_selected_experts=token_selected_experts, + token_final_scales=token_final_scales, + **self._b12x_weights, + ) + + # B12xMoEWrapper allocates its own output buffer for CUDA-graph + # compatibility. If the caller provided ``moe_output`` (e.g. an + # alltoall workspace tensor), copy into it; CuteDslB12xFusedMoE + # currently rejects alltoall in __init__, so this is a defensive + # path for future workspace-driven uses. + if moe_output is not None: + with nvtx_range("[b12x] out_copy"): + moe_output.copy_(out) + return moe_output + return out diff --git a/tensorrt_llm/_torch/modules/fused_moe/quantization.py b/tensorrt_llm/_torch/modules/fused_moe/quantization.py index da5109091501..e6769b241646 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/quantization.py +++ b/tensorrt_llm/_torch/modules/fused_moe/quantization.py @@ -2974,6 +2974,164 @@ def process_weights_after_loading(self, module: torch.nn.Module): module, module.w3_w1_weight_scale.data[expert_idx]) +class NVFP4CuteDslB12xFusedMoEMethod(NVFP4CutlassFusedMoEMethod): + """NVFP4 quant method for the FlashInfer B12x MoE backend (SM120 / SM121). + + Inherits the full CUTLASS NVFP4 weight pipeline (cat + pad + + block_scale_interleave + setup_quant_scales) so the backend's + hybrid prefill path can continue to consume the standard CUTLASS + NVFP4 GroupGEMM layout via the inherited ``CutlassFusedMoE.run_moe``. + + On top of that base layout, ``post_load_weights`` materialises the + b12x-specific weight tensors: SF un-normalization (multiply per-block + FP8 scales by ``weight_scale_2 = fc_alpha * fc_input_scale``), + ``convert_sf_to_mma_layout`` reshape, per-expert ``w*_alpha = 1 / + fc_input_scale`` vectors, and a ``B12xMoEWrapper`` instance with a + shared cross-layer output buffer. The wrapper and the bundled + weight dict are attached to the MoE module as ``module.b12x_wrapper`` + / ``module._b12x_weights`` so the backend's ``run_moe`` can dispatch + to them on decode-shape inputs without holding any kernel-prep logic + of its own. + """ + + # b12x exposes two activation strings today: ``relu2`` (Nemotron-style + # ``x * relu(x)``) and ``silu`` (SwiGLU-style ``x * silu(gate)``). + _ACTIVATION_MAP = { + ActivationType.Relu2: "relu2", + ActivationType.Swiglu: "silu", + } + + def post_load_weights(self, module: torch.nn.Module): + # Base class handles shared-weight finalize, load-balancer init, + # and setup_quant_scales. Leaves the standard CUTLASS NVFP4 + # weight + SF layout in place for the inherited prefill path. + super().post_load_weights(module) + + try: + from flashinfer import B12xMoEWrapper + from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout + except ImportError as e: + raise RuntimeError( + "NVFP4CuteDslB12xFusedMoEMethod requires the `flashinfer` package " + "(B12xMoEWrapper, cute_dsl.utils.convert_sf_to_mma_layout). " + f"Original import error: {e}") from e + + # ``_SHARED_MOE_OUTPUT_BUF`` lives next to the backend so all MoE + # layers across the model share one wrapper-owned output buffer. + from .fused_moe_cute_dsl_b12x import _SHARED_MOE_OUTPUT_BUF + + num_local_experts = module.w3_w1_weight.shape[0] + # Tensor shapes use the *padded* per-rank dims because TP partitions + # may pad ``intermediate_size`` up to a kernel-friendly boundary. + # Recover them from the actual stored tensors rather than the logical + # model config so reshapes stay valid under TP > 1. + _, w3w1_out_dim, _ = module.w3_w1_weight.shape # (E, 2*I_pad, H//16) + _, w2_out_dim, w2_in_packed = module.w2_weight.shape # (E, H, I_pad//16) + w3w1_in_dim = module.hidden_size + w2_in_dim = w2_in_packed * 16 + + # b12x reuses the per-expert ``w1_alpha`` tensor as both (a) the + # online activation-quant ``global_scale`` and (b) the FC1 epilogue + # output-dequant multiplier. That dual use is only self-consistent + # when the FP4 weight block scales are stored in their *unnormalized* + # form (raw ``max_block / FP4_MAX``), not divided out by the + # per-tensor ``weight_scale_2``. HF / ModelOpt NVFP4 checkpoints + # store the normalized variant so the FP8 block scales fit in range, + # and TRT-LLM's NVFP4 loader preserves that form. To match b12x's + # convention we recover ``weight_scale_2 = fc_alpha * fc_input_scale`` + # and multiply each expert's FP8 block scales by it before handing + # them to ``convert_sf_to_mma_layout``. With the un-normalized scales + # in place we pass ``w1_alpha = w2_alpha = 1 / fc_input_scale`` + # (== ``s_in``) so the kernel's dual-use cancels algebraically and + # the stored input-side block scales remain FP8-representable. + w1_w_scale_2 = (module.fc31_alpha * module.fc31_input_scale).to( + torch.float32) + w2_w_scale_2 = (module.fc2_alpha * module.fc2_input_scale).to( + torch.float32) + + w1_sf_fp8_norm = module.w3_w1_weight_scale.view( + torch.float8_e4m3fn).float() + w2_sf_fp8_norm = module.w2_weight_scale.view( + torch.float8_e4m3fn).float() + + # Broadcast per-expert scalar over the trailing dims (E, *). + bcast1 = w1_w_scale_2.view(-1, *([1] * (w1_sf_fp8_norm.dim() - 1))) + bcast2 = w2_w_scale_2.view(-1, *([1] * (w2_sf_fp8_norm.dim() - 1))) + w1_sf_fp8 = (w1_sf_fp8_norm * bcast1).to(torch.float8_e4m3fn) + w2_sf_fp8 = (w2_sf_fp8_norm * bcast2).to(torch.float8_e4m3fn) + + w1_sf_b12x = convert_sf_to_mma_layout(w1_sf_fp8, + m=w3w1_out_dim, + k=w3w1_in_dim, + num_groups=num_local_experts) + w2_sf_b12x = convert_sf_to_mma_layout(w2_sf_fp8, + m=w2_out_dim, + k=w2_in_dim, + num_groups=num_local_experts) + + w1_alpha_b12x = ((1.0 / module.fc31_input_scale).expand( + module.num_experts).to(torch.float32).contiguous()) + w2_alpha_b12x = ((1.0 / module.fc2_input_scale).expand( + module.num_experts).to(torch.float32).contiguous()) + fc2_input_scale_b12x = (1.0 / module.fc2_input_scale).to(torch.float32) + + # TRT-LLM packs 16 FP4 values per int64. flashinfer's internal + # ``view(torch.float4_e2m1fn_x2)`` requires byte-contiguous storage + # (stride[-1] == 1 in bytes); a uint8 view of the int64 tensor + # provides that without copying. + module._b12x_weights = dict( + w1_weight=module.w3_w1_weight.view(torch.uint8), + w1_weight_sf=w1_sf_b12x, + w1_alpha=w1_alpha_b12x, + w2_weight=module.w2_weight.view(torch.uint8), + w2_weight_sf=w2_sf_b12x, + w2_alpha=w2_alpha_b12x, + fc2_input_scale=fc2_input_scale_b12x, + ) + + if module.activation_type not in self._ACTIVATION_MAP: + supported = ", ".join(a.name for a in self._ACTIVATION_MAP) + raise ValueError( + f"NVFP4CuteDslB12xFusedMoEMethod does not support activation " + f"{ActivationType(module.activation_type).name}; " + f"supported: {supported}.") + + module.b12x_wrapper = B12xMoEWrapper( + num_experts=module.num_experts, + top_k=module.routing_method.experts_per_token, + hidden_size=module.hidden_size, + intermediate_size=module.intermediate_size_per_partition, + use_cuda_graph=getattr(module, "_b12x_use_cuda_graph", False), + max_num_tokens=module.moe_max_num_tokens, + activation=self._ACTIVATION_MAP[module.activation_type], + ) + + # Replace the wrapper's per-instance output buffer with a shared one. + # Layers run sequentially on a single stream, so a single buffer of the + # right shape is correct and saves + # ``(num_moe_layers - 1) * max_num_tokens * hidden_size * 2`` bytes — + # ~2.5 GB on Nemotron-Super-120B with ``max_num_tokens=2048``, + # ``hidden=8192``, bf16, 80 MoE layers. + if module.b12x_wrapper._moe_output is not None: + buf = module.b12x_wrapper._moe_output + key = (buf.shape[0], buf.shape[1], buf.dtype, str(buf.device)) + shared = _SHARED_MOE_OUTPUT_BUF.get(key) + if shared is None: + _SHARED_MOE_OUTPUT_BUF[key] = buf + else: + # Free the freshly allocated buffer; reuse the existing one. + module.b12x_wrapper._moe_output = shared + + logger.info_once( + f"NVFP4CuteDslB12xFusedMoEMethod active: hidden={module.hidden_size}, " + f"intermediate={module.intermediate_size_per_partition}, " + f"experts={module.num_experts}, top_k=" + f"{module.routing_method.experts_per_token}, " + f"activation={self._ACTIVATION_MAP[module.activation_type]}.", + key="cute_dsl_b12x_moe_active", + ) + + class NVFP4TRTLLMGenFusedMoEBaseMethod(NVFP4FusedMoEMethod): weight_dtype = float4_sf_dtype block_scales_dtype = torch.float8_e4m3fn diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 0ceba9c3bc7f..376139457596 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -15,6 +15,7 @@ l0_rtx_pro_6000: tests: # ------------- PyTorch tests --------------- - unittest/_torch/modeling -k "modeling_out_of_tree" + - unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py # - unittest/_torch/modeling -k "modeling_qwen" # https://nvbugs/5234573 - unittest/_torch/attention/test_attention_mla.py # SM120 W4A16 / W4A8 mixed-dtype GEMM coverage (paired with FinegrainedMixedDtypeGemm diff --git a/tests/unittest/_torch/modules/moe/moe_test_utils.py b/tests/unittest/_torch/modules/moe/moe_test_utils.py index 13f916db7da7..abbaaab80236 100644 --- a/tests/unittest/_torch/modules/moe/moe_test_utils.py +++ b/tests/unittest/_torch/modules/moe/moe_test_utils.py @@ -44,6 +44,7 @@ CutlassFusedMoE, TRTLLMGenFusedMoE, ) +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl_b12x import CuteDslB12xFusedMoE from tensorrt_llm._torch.modules.fused_moe.fused_moe_deepgemm import DeepGemmFusedMoE from tensorrt_llm._torch.modules.fused_moe.fused_moe_densegemm import DenseGEMMFusedMoE from tensorrt_llm._torch.modules.fused_moe.interface import MoE @@ -66,6 +67,7 @@ class MoeBackendType(str, Enum): DEEPGEMM = "DEEPGEMM" DENSEGEMM = "DENSEGEMM" MEGAMOE = "MEGAMOE_DEEPGEMM" + CUTE_DSL_B12X = "CUTE_DSL_B12X" def get_backend_class(backend_type: MoeBackendType) -> Type[MoE]: @@ -77,6 +79,7 @@ def get_backend_class(backend_type: MoeBackendType) -> Type[MoE]: MoeBackendType.DEEPGEMM: DeepGemmFusedMoE, MoeBackendType.DENSEGEMM: DenseGEMMFusedMoE, MoeBackendType.MEGAMOE: MegaMoEDeepGemm, + MoeBackendType.CUTE_DSL_B12X: CuteDslB12xFusedMoE, } return backend_class_map[backend_type] @@ -864,6 +867,32 @@ def should_skip_megamoe( return None +def should_skip_cute_dsl_b12x( + backend_type: MoeBackendType, + comm_method: Optional[str] = None, + moe_tp_size: int = 1, + parallel_mode: Optional[str] = None, +) -> Optional[str]: + """Check CuteDslB12xFusedMoE constraints not covered by can_implement(). + + can_implement() already gates SM version, quant_algo, dtype_activation, and + swiglu_gptoss_style. This helper covers the additional EP / alltoall hard + rejects enforced in __init__ (b12x has no expert-parallel dispatch/combine + kernel). + """ + if backend_type != MoeBackendType.CUTE_DSL_B12X: + return None + + if comm_method is not None or parallel_mode is not None: + return ( + "CuteDslB12xFusedMoE rejects expert parallelism / alltoall; " + f"got comm_method={comm_method}, parallel_mode={parallel_mode}." + ) + if moe_tp_size != 1: + return f"CuteDslB12xFusedMoE requires ep_size=1; got moe_tp_size={moe_tp_size}." + return None + + def should_skip_multi_gpu( parallel_mode: str, model_config: "MoeModelConfig", @@ -970,8 +999,12 @@ def supports_autotuner_capture( Returns: True if autotuner capture/replay is supported, False otherwise """ - # DEEPGEMM and MEGAMOE do not support autotuner capture - if backend_type in (MoeBackendType.DEEPGEMM, MoeBackendType.MEGAMOE): + # DEEPGEMM, MEGAMOE, and CUTE_DSL_B12X do not support autotuner capture + if backend_type in ( + MoeBackendType.DEEPGEMM, + MoeBackendType.MEGAMOE, + MoeBackendType.CUTE_DSL_B12X, + ): return False if use_flashinfer: @@ -1050,6 +1083,7 @@ def get_quick_skip_reason( model_config=model_config, swiglu_gptoss_style=swiglu_gptoss_style, ), + lambda: should_skip_cute_dsl_b12x(backend_type), ] for check in skip_checks: skip_reason = check() diff --git a/tests/unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py b/tests/unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py new file mode 100644 index 000000000000..1feac564b176 --- /dev/null +++ b/tests/unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Negative-path + dispatch tests for CuteDslB12xFusedMoE. + +These checks run without a GPU: they verify the can_implement() gating +matrix, the SM120/SM121 + NVFP4 selection in create_moe.get_moe_cls (the +backend is selected on the `moe_backend=CUTEDSL` path when flashinfer +is importable, never from `moe_backend=CUTLASS`), and the hybrid +CUTLASS-prefill / b12x-decode dispatch predicate. Functional +correctness of the b12x kernel is covered by end-to-end model tests on +SM120/SM121 hardware. +""" + +from unittest.mock import patch + +import pytest +import torch + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modules.fused_moe.create_moe import get_moe_cls +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl import CuteDslFusedMoE +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl_b12x import CuteDslB12xFusedMoE +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE +from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig + +_FUSED_MOE_MODULE = "tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl_b12x" + + +@pytest.mark.parametrize("sm_version", [80, 89, 90, 100, 103]) +def test_can_implement_rejects_unsupported_sm(sm_version): + """can_implement returns False on every SM outside the supported set.""" + with patch(f"{_FUSED_MOE_MODULE}.get_sm_version", return_value=sm_version): + ok, reason = CuteDslB12xFusedMoE.can_implement(QuantAlgo.NVFP4) + assert not ok + assert reason is not None and f"SM{sm_version}" in reason + + +@pytest.mark.parametrize("sm_version", sorted(CuteDslB12xFusedMoE._SUPPORTED_SM_VERSIONS)) +def test_can_implement_accepts_supported_sm_with_nvfp4(sm_version): + with patch(f"{_FUSED_MOE_MODULE}.get_sm_version", return_value=sm_version): + ok, reason = CuteDslB12xFusedMoE.can_implement(QuantAlgo.NVFP4) + assert ok + assert reason is None + + +@pytest.mark.parametrize( + "quant_algo", + [ + None, + QuantAlgo.FP8, + QuantAlgo.FP8_BLOCK_SCALES, + QuantAlgo.W4A16_MXFP4, + QuantAlgo.W4A8_MXFP4_FP8, + ], +) +def test_can_implement_rejects_non_nvfp4(quant_algo): + """Only NVFP4 is supported; everything else must be turned away.""" + with patch(f"{_FUSED_MOE_MODULE}.get_sm_version", return_value=120): + ok, reason = CuteDslB12xFusedMoE.can_implement(quant_algo) + assert not ok + assert reason is not None and "NVFP4" in reason + + +def test_can_implement_rejects_swiglu_gptoss_style(): + with patch(f"{_FUSED_MOE_MODULE}.get_sm_version", return_value=120): + ok, reason = CuteDslB12xFusedMoE.can_implement(QuantAlgo.NVFP4, swiglu_gptoss_style=True) + assert not ok + assert reason is not None and "swiglu_gptoss_style" in reason + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float8_e4m3fn]) +def test_can_implement_rejects_unsupported_activation_dtype(dtype): + with patch(f"{_FUSED_MOE_MODULE}.get_sm_version", return_value=120): + ok, reason = CuteDslB12xFusedMoE.can_implement(QuantAlgo.NVFP4, dtype_activation=dtype) + assert not ok + assert reason is not None + + +def test_get_moe_cls_cutlass_path_never_auto_promotes(): + """Explicit ``moe_backend=CUTLASS`` always returns ``CutlassFusedMoE`` — + no silent override to the b12x backend even on eligible hardware. b12x + is opted into via ``moe_backend=CUTEDSL``.""" + cfg = ModelConfig() + cfg.moe_backend = "CUTLASS" + cfg.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + with patch("tensorrt_llm._utils.get_sm_version", return_value=120): + cls = get_moe_cls(cfg) + assert cls is CutlassFusedMoE + + +def test_get_moe_cls_cutedsl_falls_back_to_cutlass_on_unsupported_quant(): + """CUTEDSL + non-(fp8_block_scales|nvfp4) → warn + fall back to CutlassFusedMoE.""" + cfg = ModelConfig() + cfg.moe_backend = "CUTEDSL" + cfg.quant_config = QuantConfig(quant_algo=QuantAlgo.FP8) + with patch("tensorrt_llm._utils.get_sm_version", return_value=120): + cls = get_moe_cls(cfg) + assert cls is CutlassFusedMoE + + +def test_get_moe_cls_cutedsl_falls_back_to_cutlass_on_missing_quant(): + cfg = ModelConfig() + cfg.moe_backend = "CUTEDSL" + cfg.quant_config = None + with patch("tensorrt_llm._utils.get_sm_version", return_value=120): + cls = get_moe_cls(cfg) + assert cls is CutlassFusedMoE + + +def test_get_moe_cls_cutedsl_returns_plain_cutedsl_on_unsupported_sm(): + """CUTEDSL + NVFP4 + non-SM120/121 → plain CuteDslFusedMoE (the SM100/103 + cuteDSL backend); the b12x branch is bypassed.""" + cfg = ModelConfig() + cfg.moe_backend = "CUTEDSL" + cfg.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + with patch("tensorrt_llm._utils.get_sm_version", return_value=100): + cls = get_moe_cls(cfg) + assert cls is CuteDslFusedMoE + + +@pytest.mark.parametrize("sm_version", sorted(CuteDslB12xFusedMoE._SUPPORTED_SM_VERSIONS)) +def test_get_moe_cls_cutedsl_selects_b12x_on_supported_sm(sm_version): + """CUTEDSL + NVFP4 + SM120/121 + flashinfer importable → CuteDslB12xFusedMoE.""" + cfg = ModelConfig() + cfg.moe_backend = "CUTEDSL" + cfg.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + with patch("tensorrt_llm._utils.get_sm_version", return_value=sm_version): + cls = get_moe_cls(cfg) + assert cls is CuteDslB12xFusedMoE + + +def test_get_moe_cls_cutedsl_falls_back_to_plain_cutedsl_when_flashinfer_missing(monkeypatch): + """CUTEDSL + NVFP4 + SM120/121 + flashinfer NOT importable → CuteDslFusedMoE.""" + import builtins + + cfg = ModelConfig() + cfg.moe_backend = "CUTEDSL" + cfg.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + + real_import = builtins.__import__ + + def _raise_on_flashinfer(name, *args, **kwargs): + if name == "flashinfer": + raise ImportError("flashinfer not installed (simulated)") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _raise_on_flashinfer) + with patch("tensorrt_llm._utils.get_sm_version", return_value=120): + cls = get_moe_cls(cfg) + assert cls is CuteDslFusedMoE + + +# -------------------------------------------------------------------------- +# Hybrid CUTLASS-prefill / b12x-decode dispatch predicate tests +# +# ``_route_to_cutlass`` is a pure shape predicate on its input ``x``; we test +# it via a stub that holds the class constant, sidestepping the full +# CutlassFusedMoE constructor (which needs a routing method, real model +# config, etc.). +# -------------------------------------------------------------------------- + + +class _RoutePredicateStub: + """Minimal carrier for ``_PREFILL_VIA_CUTLASS_THRESHOLD`` so we can call + the unbound ``_route_to_cutlass`` without instantiating the whole MoE + backend.""" + + _PREFILL_VIA_CUTLASS_THRESHOLD = CuteDslB12xFusedMoE._PREFILL_VIA_CUTLASS_THRESHOLD + + _route_to_cutlass = CuteDslB12xFusedMoE._route_to_cutlass + + +def test_dispatch_routes_prefill_shape_via_cutlass(): + stub = _RoutePredicateStub() + x = torch.empty(_RoutePredicateStub._PREFILL_VIA_CUTLASS_THRESHOLD, 1024) + assert stub._route_to_cutlass(x) is True + + +def test_dispatch_just_below_threshold_takes_b12x(): + stub = _RoutePredicateStub() + x = torch.empty(_RoutePredicateStub._PREFILL_VIA_CUTLASS_THRESHOLD - 1, 1024) + assert stub._route_to_cutlass(x) is False + + +def test_dispatch_decode_shape_takes_b12x(): + stub = _RoutePredicateStub() + x = torch.empty(1, 1024) + assert stub._route_to_cutlass(x) is False + + +def test_dispatch_rejects_non_tensor(): + """Non-tensor inputs (e.g. Fp4QuantizedTensor) stay on the b12x path + so the existing ValueError surfaces in quantize_input.""" + stub = _RoutePredicateStub() + assert stub._route_to_cutlass(object()) is False diff --git a/tests/unittest/_torch/modules/moe/test_moe_backend.py b/tests/unittest/_torch/modules/moe/test_moe_backend.py index e9957bf4ca89..9243fe82c041 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_backend.py +++ b/tests/unittest/_torch/modules/moe/test_moe_backend.py @@ -139,11 +139,17 @@ def create_test_backend( pretrained_config.intermediate_size = intermediate_size pretrained_config.torch_dtype = dtype + # CUTE_DSL_B12X is internal-only: the user-facing API selects it on the + # CUTEDSL path when SM120/121 + NVFP4 + flashinfer is importable. Route + # through "CUTEDSL" so the test exercises the same code path users hit. + moe_backend_value = ( + "CUTEDSL" if backend_type == MoeBackendType.CUTE_DSL_B12X else backend_type.value + ) model_config = ModelConfig( pretrained_config=pretrained_config, quant_config=quant_config, mapping=mapping, - moe_backend=backend_type.value, + moe_backend=moe_backend_value, ) return create_moe_backend( @@ -294,6 +300,7 @@ def run_backend_moe( MoeBackendType.DEEPGEMM, MoeBackendType.DENSEGEMM, MoeBackendType.MEGAMOE, + MoeBackendType.CUTE_DSL_B12X, ] # Data types to test diff --git a/tests/unittest/_torch/modules/moe/test_moe_module.py b/tests/unittest/_torch/modules/moe/test_moe_module.py index 44ba462af23c..2b793a173494 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_module.py +++ b/tests/unittest/_torch/modules/moe/test_moe_module.py @@ -249,6 +249,14 @@ def _create_model_config( else None ) + # CUTE_DSL_B12X is an internal-only MoeBackendType — it has no + # corresponding user-facing MoeConfig.backend literal. Route through + # "CUTEDSL" so the test exercises the cuteDSL-family selection path that + # users hit on SM120/121 + NVFP4 (where get_moe_cls returns the hybrid + # CuteDslB12xFusedMoE backend when flashinfer is importable). + if moe_backend == MoeBackendType.CUTE_DSL_B12X.value: + moe_backend = MoeBackendType.CUTEDSL.value + kwargs = dict( pretrained_config=pretrained_config, mapping=mapping, @@ -817,6 +825,7 @@ def init_worker(custom_paths, comm_method_type, master_port): MoeBackendType.DEEPGEMM, MoeBackendType.DENSEGEMM, MoeBackendType.MEGAMOE, + MoeBackendType.CUTE_DSL_B12X, ] # Data types to test From 28abcad55a1d3e058af5bd21d4638564c19506e8 Mon Sep 17 00:00:00 2001 From: Guoming Zhang <137257613+nv-guomingz@users.noreply.github.com> Date: Tue, 26 May 2026 09:03:49 +0800 Subject: [PATCH 020/308] [None][perf] Integrate the flashinfer gdn prefill kernel for qwen3.5 (#13644) Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- .../_torch/modules/fla/flashinfer_chunk.py | 179 +++++++++ .../_torch/modules/mamba/gdn_mixer.py | 9 +- .../mamba/test_flashinfer_chunk_gdn.py | 369 ++++++++++++++++++ 3 files changed, 555 insertions(+), 2 deletions(-) create mode 100644 tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py create mode 100644 tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py new file mode 100644 index 000000000000..20bc49119037 --- /dev/null +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""FlashInfer GDN prefill adapter for ``Qwen3NextGatedDeltaNet.forward_extend``. + +Exposes ``chunk_gated_delta_rule`` with a signature call-compatible with the +vendored Triton ``tensorrt_llm._torch.modules.fla.chunk.chunk_gated_delta_rule``. + +Differences vs the Triton path are absorbed inside this wrapper: + * Layout: TRT-LLM ``[1, T, H, D]`` -> FlashInfer packed ``[T, H, D]``. + * Forget-gate space: TRT-LLM's Triton ``chunk_gated_delta_rule`` consumes + ``g`` in **log space** (``fla/chunk.py`` doc: "(forget) gating tensor (in + log space!)"); FlashInfer's ``chunk_gated_delta_rule`` consumes the same + quantity in **linear space** (default 1.0 = no decay; alpha = exp(log_g)). + The wrapper converts ``g_linear = exp(g_log)`` before calling FlashInfer. + * Pre-L2-normalize Q/K when ``use_qk_l2norm_in_kernel=True`` + (the FlashInfer prefill kernel does NOT apply L2 norm internally; the + ``use_qk_l2norm_in_kernel`` parameter on ``flashinfer.chunk_gated_delta_rule`` + is currently a dead arg, see ``flashinfer/gdn_prefill.py:317-356``). + * Pre-gather and post-scatter of indexed SSM state (FlashInfer requires + packed ``[num_seqs, H, D, D]`` fp32 initial/output state). + * State layout: TRT-LLM's Triton uses ``[N, H, K, V]`` ordering of the last + two dims; FlashInfer's prefill kernel uses ``[N, H, V, K]`` (per docstring + in ``flashinfer/gdn_prefill.py``: "The final state layout is ``[N, H, V, K]``"; + confirmed by ``flashinfer/tests/gdn/test_prefill_delta_rule.py`` which + transposes the last two dims to compare with the reference implementation). + The wrapper transposes initial_state on the way in and output_state on the + way out so callers see TRT-LLM's ``[N, H, K, V]`` everywhere. + +This module is only imported when ``TLLM_USE_FLASHINFER_GDN_PREFILL=1`` is set +at process start; do not import it lazily inside hot paths. +""" + +from typing import Optional, Tuple + +import torch + +from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd + + +# Mirror the @torch.compiler.disable on the legacy Triton wrapper +# (chunk.py:119): Dynamo must not trace this wrapper because it imports +# `flashinfer` lazily and calls into FI's CuTe-DSL kernels, neither of +# which compiles cleanly. +@torch.compiler.disable +def chunk_gated_delta_rule( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: Optional[float] = None, + initial_state: Optional[torch.Tensor] = None, + initial_state_indices: Optional[torch.Tensor] = None, + inplace_indexed_state_update: bool = False, + output_final_state: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Adapter for FlashInfer's chunk_gated_delta_rule.""" + # FlashInfer is imported lazily so importing this module on a non-FlashInfer + # build does not error until the function is actually called. + import flashinfer + + # --- Step 1: pre-flight asserts -------------------------------------- + assert head_first is False, "head_first=True is not supported by this wrapper" + assert q.dim() == 4 and q.shape[0] == 1, f"q must be [1, T, H_q, D_k], got {tuple(q.shape)}" + assert k.shape[2] == q.shape[2], ( + f"num_q_heads ({q.shape[2]}) must equal num_k_heads ({k.shape[2]})" + ) + assert q.dtype in (torch.bfloat16, torch.float16), f"q dtype must be bf16/fp16, got {q.dtype}" + assert g.dtype == torch.float32, f"g must be fp32, got {g.dtype}" + assert cu_seqlens is not None, "cu_seqlens is required (varlen mode)" + assert initial_state is not None, "initial_state is required" + if inplace_indexed_state_update: + assert initial_state_indices is not None, ( + "inplace_indexed_state_update=True requires initial_state_indices" + ) + + # --- Step 2: layout [1, T, H, D] -> [T, H, D] ------------------------ + # q/k/v are slices of mixed_qkv produced by torch.split on the last dim, + # so the squeeze(0) view is *non-contiguous*. The .contiguous() here pays + # a real copy and is required (FlashInfer reads contiguous TMA tiles). + q3 = q.squeeze(0).contiguous() + k3 = k.squeeze(0).contiguous() + v3 = v.squeeze(0).contiguous() + # Convert g from Triton's log-space convention to FlashInfer's linear-space + # alpha (default 1.0 = no decay). FlashInfer's prefill kernel multiplies the + # SSM state by ``g`` directly each chunk; passing log-space values produces + # NaN. ``torch.exp`` always returns a fresh contiguous tensor, so no + # explicit .contiguous() is needed. + g2 = torch.exp(g.squeeze(0)) + # ``.to(torch.float32)`` allocates a fresh contiguous fp32 tensor when the + # source is bf16 (the common case), or returns the (already contiguous) + # input view when beta is already fp32 — no .contiguous() needed. + beta2 = beta.squeeze(0).to(torch.float32) + + # --- Step 3: emulate use_qk_l2norm_in_kernel ------------------------- + # Use the fused Triton l2norm_fwd (eps=1e-6 matches the Triton/FI decode + # kernels) instead of F.normalize, which incurs extra kernel launches and + # an intermediate buffer at short ISL. + if use_qk_l2norm_in_kernel: + q3 = l2norm_fwd(q3) + k3 = l2norm_fwd(k3) + + # --- Step 4: gather initial state and convert layout ----------------- + # TRT-LLM stores SSM state with last two dims as (K, V); FlashInfer expects + # (V, K). Transpose last two dims and make contiguous so the kernel reads + # the buffer in the layout it expects. + if initial_state_indices is not None: + gathered_init = initial_state[initial_state_indices].to(torch.float32) + else: + gathered_init = initial_state.to(torch.float32) + gathered_init = gathered_init.transpose(-1, -2).contiguous() + + # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers + # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its + # internal `torch.empty` per call. `num_o_heads` per FI docstring is + # max(num_q_heads, num_v_heads) — equivalently num_v_heads for GVA. + # Only allocate / request final state when a caller actually consumes it + # (either inplace scatter back to the SSM pool, or return to the caller); + # otherwise FI skips the final-state aggregation entirely. + total_seq_len = q3.shape[0] + num_o_heads = max(q3.shape[1], v3.shape[1]) + head_size = q3.shape[2] + need_state = inplace_indexed_state_update or output_final_state + output_buf = q3.new_empty(total_seq_len, num_o_heads, head_size) + if need_state: + num_seqs = cu_seqlens.shape[0] - 1 + state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=torch.float32) + out_packed, out_state = flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g2, + beta=beta2, + scale=scale, + initial_state=gathered_init, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, # dead param in FlashInfer; we already normalized + output=output_buf, + output_state=state_buf, + ) + else: + # FI returns a single tensor (not a tuple) when output_final_state=False. + out_packed = flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g2, + beta=beta2, + scale=scale, + initial_state=gathered_init, + output_final_state=False, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=output_buf, + ) + out_state = None + + # --- Step 7: convert state layout back, scatter / return ----------- + # Convert FlashInfer's (V, K) state layout back to TRT-LLM's (K, V) before + # scattering to the SSM pool / returning to the caller. + if inplace_indexed_state_update: + out_state_kv = out_state.transpose(-1, -2).contiguous() + initial_state[initial_state_indices] = out_state_kv.to(initial_state.dtype, copy=False) + final_to_return: Optional[torch.Tensor] = None + elif output_final_state: + out_state_kv = out_state.transpose(-1, -2).contiguous() + final_to_return = out_state_kv.to(initial_state.dtype, copy=False) + else: + final_to_return = None + + # --- Step 8: restore output layout ---------------------------------- + out = out_packed.unsqueeze(0) + + # --- Step 9: return ------------------------------------------------- + return out, final_to_return diff --git a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py index 210d4f23d63b..d2550a1505db 100644 --- a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py @@ -3,6 +3,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import os from typing import Optional import torch @@ -11,7 +12,12 @@ from torch import nn from transformers import Qwen3NextConfig -from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule +# Default: FlashInfer GDN prefill ON. Set TLLM_USE_FLASHINFER_GDN_PREFILL=0 to +# fall back to the vendored Triton chunk_gated_delta_rule. +if os.getenv("TLLM_USE_FLASHINFER_GDN_PREFILL", "1") == "1": + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule +else: + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule_update from tensorrt_llm._torch.modules.fla.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, @@ -743,7 +749,6 @@ def forward_extend( beta_p = b_p.sigmoid().unsqueeze(0) g_p = fused_gdn_gating(self.A_log, a_p, self.dt_bias).unsqueeze(0) recurrent_state_p = ssm_states[state_indices_p] - attn_out_prefill, last_recurrent_state = chunk_gated_delta_rule( q=query_p, k=key_p, diff --git a/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py b/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py new file mode 100644 index 000000000000..929fa9612a97 --- /dev/null +++ b/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py @@ -0,0 +1,369 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Operator-level tests for the FlashInfer GDN prefill adapter wrapper. + +Compares ``tensorrt_llm._torch.modules.fla.flashinfer_chunk.chunk_gated_delta_rule`` +against the vendored Triton ``tensorrt_llm._torch.modules.fla.chunk.chunk_gated_delta_rule`` +across the call shapes used by ``Qwen3NextGatedDeltaNet.forward_extend``. +""" + +import pytest +import torch + +# Skip rules --------------------------------------------------------------- + + +def _supported_arch() -> bool: + if not torch.cuda.is_available(): + return False + major, _ = torch.cuda.get_device_capability(0) + # SM90 (Hopper) or SM100 (Blackwell) + return major in (9, 10) + + +skip_unsupported = pytest.mark.skipif( + not _supported_arch(), + reason="FlashInfer GDN prefill requires SM90 (Hopper) or SM100 (Blackwell)", +) + + +# Input factory ------------------------------------------------------------ + + +@torch.no_grad() +def _make_inputs( + seq_lens: list[int], + num_q_heads: int = 4, + num_v_heads: int = 16, + head_dim: int = 128, + dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", + seed: int = 0, +): + """Build (q, k, v, g, beta, cu_seqlens) packed in TRT-LLM ``[1, T, H, D]`` layout. + + Mirrors what ``Qwen3NextGatedDeltaNet.forward_extend`` passes after the QKV + split. ``g`` / ``beta`` are produced post-``fused_gdn_gating`` (fp32). + """ + torch.manual_seed(seed) + total_t = sum(seq_lens) + q = torch.randn(1, total_t, num_q_heads, head_dim, dtype=dtype, device=device) * 0.1 + k = torch.randn(1, total_t, num_q_heads, head_dim, dtype=dtype, device=device) * 0.1 + v = torch.randn(1, total_t, num_v_heads, head_dim, dtype=dtype, device=device) * 0.1 + # g is the "log-forget" gate; emulate post-`fused_gdn_gating` (negative, fp32). + g = -torch.rand(1, total_t, num_v_heads, dtype=torch.float32, device=device) * 0.05 + beta = torch.rand(1, total_t, num_v_heads, dtype=torch.float32, device=device) + cu = torch.tensor( + [0] + list(torch.tensor(seq_lens).cumsum(0).tolist()), + dtype=torch.int64, + device=device, + ) + return q, k, v, g, beta, cu + + +def _zero_initial_state(num_seqs, num_heads, head_dim, device, dtype=torch.float32): + return torch.zeros(num_seqs, num_heads, head_dim, head_dim, dtype=dtype, device=device) + + +# Pure-Python import smoke (no GPU required) ------------------------------ + + +def test_wrapper_module_importable(): + """Smoke import of the wrapper. Pure Python; does not require CUDA.""" + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import ( # noqa: F401 + chunk_gated_delta_rule, + ) + + +# Parity tests against the Triton reference ------------------------------- + + +@skip_unsupported +def test_basic_single_seq_no_l2norm_matches_triton(): + """Single-seq, no initial state, no L2 norm, no output_final_state.""" + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_cgdr + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule as fi_cgdr + + seq_lens = [4096] + q, k, v, g, beta, cu = _make_inputs(seq_lens) + init = _zero_initial_state(len(seq_lens), v.shape[2], v.shape[3], q.device) + + out_triton, _ = triton_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=False, + ) + + out_fi, final_fi = fi_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=False, + ) + + assert final_fi is None + assert out_fi.shape == out_triton.shape + torch.testing.assert_close(out_fi, out_triton, atol=2e-2, rtol=2e-2) + + +@skip_unsupported +def test_basic_single_seq_with_l2norm_matches_triton(): + """Single-seq, no initial state, with L2 norm (Qwen3.5 production setting).""" + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_cgdr + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule as fi_cgdr + + seq_lens = [8192] + q, k, v, g, beta, cu = _make_inputs(seq_lens) + init = _zero_initial_state(len(seq_lens), v.shape[2], v.shape[3], q.device) + + out_triton, _ = triton_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + out_fi, _ = fi_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + torch.testing.assert_close(out_fi, out_triton, atol=2e-2, rtol=2e-2) + + +@skip_unsupported +@pytest.mark.parametrize( + "seq_lens", + [ + [4096, 4096], + [4096, 8192, 4096], + [1024, 16384], + ], +) +def test_varlen_with_l2norm_matches_triton(seq_lens): + """Varlen batches — production prefill packs multiple requests.""" + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_cgdr + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule as fi_cgdr + + q, k, v, g, beta, cu = _make_inputs(seq_lens) + init = _zero_initial_state(len(seq_lens), v.shape[2], v.shape[3], q.device) + + out_triton, _ = triton_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + out_fi, _ = fi_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init, + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + torch.testing.assert_close(out_fi, out_triton, atol=2e-2, rtol=2e-2) + + +@skip_unsupported +def test_packed_initial_state_with_output_final_state_matches_triton(): + """target_verify prefill path: caller pre-gathers ssm_states[state_indices_p] and + writes the returned final state back manually (output_final_state=True).""" + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_cgdr + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule as fi_cgdr + + seq_lens = [4096, 8192] + q, k, v, g, beta, cu = _make_inputs(seq_lens) + num_seqs = len(seq_lens) + init = (torch.randn(num_seqs, v.shape[2], v.shape[3], v.shape[3], device=q.device) * 0.01).to( + torch.float32 + ) + + out_triton, final_triton = triton_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init.clone(), + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=True, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + out_fi, final_fi = fi_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=init.clone(), + initial_state_indices=None, + inplace_indexed_state_update=False, + output_final_state=True, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + assert final_fi is not None + assert final_triton is not None + torch.testing.assert_close(out_fi, out_triton, atol=2e-2, rtol=2e-2) + torch.testing.assert_close( + final_fi.to(torch.float32), + final_triton.to(torch.float32), + atol=5e-2, + rtol=5e-2, + ) + + +@skip_unsupported +def test_indexed_gather_inplace_scatter_matches_triton(): + """Non-spec prefill path: caller passes the full SSM pool plus cache_indices, + kernel does inplace gather/scatter (inplace_indexed_state_update=True, output_final_state=False).""" + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_cgdr + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import chunk_gated_delta_rule as fi_cgdr + + seq_lens = [4096, 8192] + q, k, v, g, beta, cu = _make_inputs(seq_lens) + num_v_heads, head_dim = v.shape[2], v.shape[3] + + # Simulate a 16-slot SSM pool; sequences live at slots [3, 7]. + pool_slots = 16 + cache_indices = torch.tensor([3, 7], dtype=torch.int32, device=q.device) + pool_init = ( + torch.randn(pool_slots, num_v_heads, head_dim, head_dim, device=q.device) * 0.01 + ).to(torch.float32) + + pool_triton = pool_init.clone() + out_triton, _ = triton_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=pool_triton, + initial_state_indices=cache_indices, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + pool_fi = pool_init.clone() + out_fi, final_fi = fi_cgdr( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=pool_fi, + initial_state_indices=cache_indices, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + assert final_fi is None # caller asks for None when inplace=True + torch.testing.assert_close(out_fi, out_triton, atol=2e-2, rtol=2e-2) + + # The two written slots must match within tolerance; the others must be untouched. + torch.testing.assert_close( + pool_fi[cache_indices].to(torch.float32), + pool_triton[cache_indices].to(torch.float32), + atol=5e-2, + rtol=5e-2, + ) + untouched = [i for i in range(pool_slots) if i not in cache_indices.tolist()] + torch.testing.assert_close(pool_fi[untouched], pool_init[untouched], atol=0.0, rtol=0.0) + + +# Env-flag routing test (no GPU required) --------------------------------- + + +def test_gdn_mixer_default_uses_flashinfer_wrapper(monkeypatch): + """Default (no env): gdn_mixer imports the FlashInfer wrapper. + Opt-out (``TLLM_USE_FLASHINFER_GDN_PREFILL=0``) restores the Triton path. + + Independent of GPU availability; only checks Python import wiring. + """ + import importlib + + # Default — env unset, expect FlashInfer wrapper. + monkeypatch.delenv("TLLM_USE_FLASHINFER_GDN_PREFILL", raising=False) + import tensorrt_llm._torch.modules.mamba.gdn_mixer as gdn_mixer + + importlib.reload(gdn_mixer) + from tensorrt_llm._torch.modules.fla.flashinfer_chunk import ( + chunk_gated_delta_rule as wrapper_fn, + ) + + assert gdn_mixer.chunk_gated_delta_rule is wrapper_fn + + # Opt out — env=0 restores the Triton path. + monkeypatch.setenv("TLLM_USE_FLASHINFER_GDN_PREFILL", "0") + importlib.reload(gdn_mixer) + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule as triton_fn + + assert gdn_mixer.chunk_gated_delta_rule is triton_fn + + # Reset to default for subsequent tests in the same process. + monkeypatch.delenv("TLLM_USE_FLASHINFER_GDN_PREFILL", raising=False) + importlib.reload(gdn_mixer) From acd224175c5f3eaae06dffa1a765e092021ddf55 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Tue, 26 May 2026 09:13:19 +0800 Subject: [PATCH 021/308] [None][chore] Update flashinfer-python from 0.6.11.post1 to 0.6.12rc1 (#14512) Signed-off-by: yihwang-nv --- ATTRIBUTIONS-Python.md | 2 +- requirements.txt | 2 +- security_scanning/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index 9640cf907776..39bfa93bf760 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -5261,7 +5261,7 @@ For more information, please refer to - `Tracker`: https://github.com/tox-dev/py-filelock/issues -## flashinfer-python (0.6.11.post1) +## flashinfer-python (0.6.12rc1) ### Licenses License: `Apache-2.0` diff --git a/requirements.txt b/requirements.txt index 19607b53b31b..9b76d52b0c86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ ordered-set peft>=0.18.1,<0.19.0 patchelf einops -flashinfer-python==0.6.11.post1 +flashinfer-python @ git+https://github.com/flashinfer-ai/flashinfer.git@v0.6.12rc1#egg=flashinfer-python opencv-python-headless xgrammar==0.1.32 llguidance==0.7.29 diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 1aa472688d36..6839347ecbb2 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -54,7 +54,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python (==0.6.11.post1)", + "flashinfer-python (==0.6.12rc1)", "opencv-python-headless (>=4.13.0.92,<5.0.0.0)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", From 31826c1296781e49ee12c468ae6badc6cdb33834 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 26 May 2026 09:22:57 +0800 Subject: [PATCH 022/308] [https://nvbugs/6162328][fix] Add a tiny compat shim in `load_hf_tokenizer` that, when `bytes_to_unicode` is m (#14090) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tensorrt_llm/tokenizer/tokenizer.py | 20 ++++++++++++++++++++ tests/integration/test_lists/waives.txt | 1 - 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index 35ddce766030..d08cb9ed5a42 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -584,6 +584,24 @@ def maybe_register_transformers_modules_by_value(): f"serialization: {e}") +def _ensure_gpt2_bytes_to_unicode_compat() -> None: + """Restore ``bytes_to_unicode`` on ``transformers.models.gpt2.tokenization_gpt2``. + + transformers 5.x removed it from that module, but some ``trust_remote_code`` + tokenizers (e.g. Kimi-K2's ``tokenization_kimi.py``) still import it from + the legacy location. The function survives under + ``transformers.convert_slow_tokenizer``. + """ + try: + from transformers.models.gpt2 import tokenization_gpt2 + if hasattr(tokenization_gpt2, "bytes_to_unicode"): + return + from transformers.convert_slow_tokenizer import bytes_to_unicode + except ImportError: + return + tokenization_gpt2.bytes_to_unicode = bytes_to_unicode + + def load_hf_tokenizer(model_dir: str, trust_remote_code: bool = True, use_fast: bool = True, @@ -599,6 +617,8 @@ def load_hf_tokenizer(model_dir: str, A TransformersTokenizer object if the tokenizer is loaded successfully. ''' + _ensure_gpt2_bytes_to_unicode_compat() + try: tokenizer = TransformersTokenizer.from_pretrained( model_dir, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3d8bc423f454..00e234355fa1 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -101,7 +101,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[8gpus] SKIP (https://nvbugs/6162328) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_bf16 SKIP (https://nvbugs/6211185) From e2497f02536e84d4a2916713a1c0f7bfed5d4afe Mon Sep 17 00:00:00 2001 From: "Wang, Xiao" <24860335+xwang233@users.noreply.github.com> Date: Mon, 25 May 2026 18:38:28 -0700 Subject: [PATCH 023/308] [https://nvbugs/6114610][test] unwaive disagg tests fixed by UCX_TLS setter (#14440) Signed-off-by: Xiao Wang <24860335+xwang233@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) --- tests/integration/test_lists/waives.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 00e234355fa1..d57693acde38 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -178,10 +178,7 @@ cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (h disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114612) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_gentp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114140) -disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114610) disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_cache_aware_balance[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_conditional[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) @@ -201,7 +198,6 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) -disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6011317) disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) From 7142d077021a9ce881d5f25cae5e6ed247f8c60d Mon Sep 17 00:00:00 2001 From: Zhenhuan Chen Date: Tue, 26 May 2026 09:38:43 +0800 Subject: [PATCH 024/308] [None][fix] Route trtllm-bench and trtllm-serve tokenizer load through TransformersTokenizer (#14452) Signed-off-by: Zhenhuan Chen --- tensorrt_llm/bench/utils/data.py | 15 ++++++--- tensorrt_llm/serve/router.py | 19 +++++------ tests/unittest/disaggregated/test_router.py | 37 +++++++++++++++++++-- tests/unittest/others/test_bench_data.py | 32 +++++++++++++++++- 4 files changed, 85 insertions(+), 18 deletions(-) diff --git a/tensorrt_llm/bench/utils/data.py b/tensorrt_llm/bench/utils/data.py index 8ec44b72f657..147fb465f89f 100644 --- a/tensorrt_llm/bench/utils/data.py +++ b/tensorrt_llm/bench/utils/data.py @@ -16,13 +16,14 @@ from functools import partial from typing import List, TextIO, Tuple -from transformers import AutoTokenizer, PreTrainedTokenizer +from transformers import PreTrainedTokenizer from tensorrt_llm.bench.dataclasses.general import (DatasetMetadata, InferenceRequest) from tensorrt_llm.bench.dataclasses.statistics import PercentileStats from tensorrt_llm.executor.request import LoRARequest from tensorrt_llm.inputs import default_multimodal_input_loader +from tensorrt_llm.tokenizer.tokenizer import TransformersTokenizer class DatasetFormatError(ValueError): @@ -62,9 +63,15 @@ def initialize_tokenizer(model_name: str, f"Failed to load custom_tokenizer '{custom_tokenizer}'. " "Expected alias or 'module.path.ClassName'.") from e else: - tokenizer = AutoTokenizer.from_pretrained(model_name, - padding_side="left", - trust_remote_code=True) + # Route through TransformersTokenizer so trtllm-bench inherits its + # post-load fixes -- in particular ``maybe_fix_byte_level_tokenizer``, + # which reloads DeepSeek-V3-style models that would otherwise come + # back with a Metaspace pre-tokenizer that silently strips spaces + # ("hello world" -> "helloworld") on transformers >= 5.x. Peel off + # the wrapper to return the raw HF tokenizer the rest of bench code + # expects. + tokenizer = TransformersTokenizer.from_pretrained( + model_name, padding_side="left", trust_remote_code=True).tokenizer if tokenizer.pad_token_id is None: tokenizer.add_special_tokens({"pad_token": "[PAD]"}) diff --git a/tensorrt_llm/serve/router.py b/tensorrt_llm/serve/router.py index 1f477dba2fa6..053c9bc19aa7 100644 --- a/tensorrt_llm/serve/router.py +++ b/tensorrt_llm/serve/router.py @@ -20,7 +20,6 @@ from typing import Awaitable, Callable, Dict, Iterable, List, Optional, Union import aiohttp -from transformers import AutoTokenizer from tensorrt_llm.bindings.internal.batch_manager import (BlockKey, BlockKeyHasher) @@ -695,15 +694,15 @@ def _get_tokenizer(self, model: str): self._tokenizers[model] = load_custom_tokenizer( self._custom_tokenizer, model) else: - from tensorrt_llm.tokenizer import \ - maybe_fix_byte_level_tokenizer - tokenizer = AutoTokenizer.from_pretrained( - model, trust_remote_code=True) - # Work around Transformers 5.x LlamaTokenizer overriding - # tokenizer.json's ByteLevel pre-tokenizer with Metaspace, - # which silently strips spaces from prompts (see tokenizer.py). - self._tokenizers[model] = maybe_fix_byte_level_tokenizer( - tokenizer, model, trust_remote_code=True) + # Route through TransformersTokenizer so block-hash routing + # inherits the same post-load fixes as the rest of TRT-LLM + # (e.g. maybe_fix_byte_level_tokenizer for DeepSeek-V3 + # Metaspace bug, _fallback_to_fast_tokenizer for DeepSeek-V3.2 + # AttributeError on transformers >= 5.x). Peel off .tokenizer + # to return the raw HF tokenizer used by _tokenize() below. + from tensorrt_llm.tokenizer import TransformersTokenizer + self._tokenizers[model] = TransformersTokenizer.from_pretrained( + model, trust_remote_code=True).tokenizer return self._tokenizers[model] def _tokenize(self, request: OpenAIRequest) -> list[list[int]]: diff --git a/tests/unittest/disaggregated/test_router.py b/tests/unittest/disaggregated/test_router.py index 3d14665c1232..48eae3480133 100644 --- a/tests/unittest/disaggregated/test_router.py +++ b/tests/unittest/disaggregated/test_router.py @@ -10,9 +10,9 @@ from tensorrt_llm.serve.openai_protocol import (ChatCompletionRequest, CompletionRequest, DisaggregatedParams) -from tensorrt_llm.serve.router import (ConversationRouter, KvCacheAwareRouter, - LoadBalancingRouter, RoundRobinRouter, - create_router) +from tensorrt_llm.serve.router import (BlockHashMixin, ConversationRouter, + KvCacheAwareRouter, LoadBalancingRouter, + RoundRobinRouter, create_router) def _make_mock_aiohttp_session(return_value=None): @@ -858,3 +858,34 @@ def test_create_router_conversation(): router = create_router(RouterConfig(type="conversation"), ["server1", "server2"]) assert isinstance(router, ConversationRouter) + + +def test_block_hash_mixin_routes_through_transformers_tokenizer(): + """``BlockHashMixin._get_tokenizer`` must call ``TransformersTokenizer.from_pretrained``. + + Routing through ``TransformersTokenizer`` is what lets block-hash KV-cache + routing inherit the post-load fixes (``maybe_fix_byte_level_tokenizer`` for + DeepSeek-V3 Metaspace, ``_fallback_to_fast_tokenizer`` for DeepSeek-V3.2 + on transformers >= 5.x). Without this routing, ``trtllm-serve`` would + tokenize prompts differently from the rest of TRT-LLM when computing + block hashes for cache hits. + """ + + class _Probe(BlockHashMixin): + pass + + probe = _Probe() + probe._init_block_hashing() + + inner = mock.MagicMock() + wrapper = mock.MagicMock(tokenizer=inner) + with mock.patch( + "tensorrt_llm.tokenizer.TransformersTokenizer.from_pretrained", + return_value=wrapper) as routed: + out = probe._get_tokenizer("dummy/model") + + routed.assert_called_once_with("dummy/model", trust_remote_code=True) + # The cached tokenizer must be the raw HF tokenizer used by _tokenize, + # not the TransformersTokenizer wrapper. + assert out is inner + assert probe._tokenizers["dummy/model"] is inner diff --git a/tests/unittest/others/test_bench_data.py b/tests/unittest/others/test_bench_data.py index da902f243fab..0f1e69fedc64 100644 --- a/tests/unittest/others/test_bench_data.py +++ b/tests/unittest/others/test_bench_data.py @@ -13,10 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import io +from unittest import mock import pytest -from tensorrt_llm.bench.utils.data import DatasetFormatError, create_dataset_from_stream +from tensorrt_llm.bench.utils.data import ( + DatasetFormatError, + create_dataset_from_stream, + initialize_tokenizer, +) class _FakeTokenizer: @@ -36,3 +41,28 @@ def test_empty_stream_raises_dataset_format_error(): with pytest.raises(DatasetFormatError, match="No data was read from the dataset stream"): create_dataset_from_stream(tokenizer, empty_stream) + + +def test_initialize_tokenizer_routes_through_transformers_tokenizer(): + """``initialize_tokenizer`` must call ``TransformersTokenizer.from_pretrained``. + + Routing through ``TransformersTokenizer`` is what lets ``trtllm-bench`` + inherit the post-load fixes (e.g. ``maybe_fix_byte_level_tokenizer``, + which prevents DeepSeek-V3 from loading with a Metaspace pre-tokenizer + that silently strips spaces -- "hello world" -> "helloworld" -- on + transformers >= 5.x). Without this routing the bench would see a + different tokenizer than the rest of TRT-LLM uses. + """ + inner = mock.MagicMock() + inner.pad_token_id = 0 # already set => add_special_tokens not invoked + wrapper = mock.MagicMock(tokenizer=inner) + + with mock.patch( + "tensorrt_llm.bench.utils.data.TransformersTokenizer.from_pretrained", return_value=wrapper + ) as routed: + out = initialize_tokenizer("dummy/model") + + routed.assert_called_once_with("dummy/model", padding_side="left", trust_remote_code=True) + # Bench code uses the raw HF tokenizer (calls __call__, encode, + # add_special_tokens on it), so the wrapper must be peeled off. + assert out is inner From 68d3db1423ad175a4a5492acacfa592470d16702 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Tue, 26 May 2026 10:23:40 +0800 Subject: [PATCH 025/308] [https://nvbugs/6184914][test] Unwaive related tests (#14523) Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d57693acde38..26a0305d3af7 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -54,7 +54,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_sched accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6185146) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6195110) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162115) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6184914) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6112497) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162122) @@ -74,10 +73,8 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True] SKIP (https://nvbugs/5955773) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5945081) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6184914) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6184914) accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype SKIP (https://nvbugs/6164924) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5981293) From 13778e15d5e33029ce9a8092787f4d0b5124d2a3 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 26 May 2026 03:26:53 +0000 Subject: [PATCH 026/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/models/core/mllama/poetry.lock | 9 +- security_scanning/examples/serve/poetry.lock | 101 ++++++++++-------- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 61 ++++++----- security_scanning/pyproject.toml | 4 +- 5 files changed, 95 insertions(+), 84 deletions(-) diff --git a/security_scanning/examples/models/core/mllama/poetry.lock b/security_scanning/examples/models/core/mllama/poetry.lock index 45d084740e47..d7320df7f6c2 100644 --- a/security_scanning/examples/models/core/mllama/poetry.lock +++ b/security_scanning/examples/models/core/mllama/poetry.lock @@ -1089,17 +1089,18 @@ xmp = ["defusedxml"] [[package]] name = "pulp" -version = "3.3.1" +version = "3.3.2" description = "PuLP is an LP modeler written in python. PuLP can generate MPS or LP files and call GLPK, COIN CLP/CBC, CPLEX, and GUROBI to solve linear problems." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pulp-3.3.1-py3-none-any.whl", hash = "sha256:45aa73db3368eb13b156564e092784c8fa0c1feefa64c2afb0410d9dc0bb5cd9"}, - {file = "pulp-3.3.1.tar.gz", hash = "sha256:a9ec237a56981b11c2096e8ba6bb72006833410ba5b400aa257426f85df5e293"}, + {file = "pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0"}, + {file = "pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0"}, ] [package.extras] +cbc = ["cbcbox"] copt = ["coptpy"] cplex = ["cplex ; sys_platform != \"darwin\" or python_full_version < \"3.12.0\""] gurobi = ["gurobipy"] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index f868175de0f2..6e1bcb9b90a6 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -985,14 +985,14 @@ tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "cyclopts" -version = "4.16.0" +version = "4.16.1" description = "Intuitive, easy CLIs based on type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cyclopts-4.16.0-py3-none-any.whl", hash = "sha256:cbb9f8af92ace82c250178a3a51f5ecec1df95ab99116af3aa7140b218ccd2a1"}, - {file = "cyclopts-4.16.0.tar.gz", hash = "sha256:6a07b8ada2fa3d7611e227a98b661523c39644a50e04c92839832d9f599f398d"}, + {file = "cyclopts-4.16.1-py3-none-any.whl", hash = "sha256:617795392c4113a2c2cc7af716f20244900e87f23daa05442d1268d81472a592"}, + {file = "cyclopts-4.16.1.tar.gz", hash = "sha256:8aa47bf92a5fb33abca5af05e576eecdb0d2f79893ad29238046df78370fc4a8"}, ] [package.dependencies] @@ -1579,55 +1579,62 @@ trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httptools" -version = "0.7.1" +version = "0.8.0" description = "A collection of framework independent HTTP protocol utils." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "httptools-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78"}, - {file = "httptools-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4"}, - {file = "httptools-0.7.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c8c751014e13d88d2be5f5f14fc8b89612fcfa92a9cc480f2bc1598357a23a05"}, - {file = "httptools-0.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:654968cb6b6c77e37b832a9be3d3ecabb243bbe7a0b8f65fbc5b6b04c8fcabed"}, - {file = "httptools-0.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b580968316348b474b020edf3988eecd5d6eec4634ee6561e72ae3a2a0e00a8a"}, - {file = "httptools-0.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d496e2f5245319da9d764296e86c5bb6fcf0cf7a8806d3d000717a889c8c0b7b"}, - {file = "httptools-0.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cbf8317bfccf0fed3b5680c559d3459cccf1abe9039bfa159e62e391c7270568"}, - {file = "httptools-0.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:474d3b7ab469fefcca3697a10d11a32ee2b9573250206ba1e50d5980910da657"}, - {file = "httptools-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3c3b7366bb6c7b96bd72d0dbe7f7d5eead261361f013be5f6d9590465ea1c70"}, - {file = "httptools-0.7.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:379b479408b8747f47f3b253326183d7c009a3936518cdb70db58cffd369d9df"}, - {file = "httptools-0.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad6b591a682dcc6cf1397c3900527f9affef1e55a06c4547264796bbd17cf5e"}, - {file = "httptools-0.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:eb844698d11433d2139bbeeb56499102143beb582bd6c194e3ba69c22f25c274"}, - {file = "httptools-0.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f65744d7a8bdb4bda5e1fa23e4ba16832860606fcc09d674d56e425e991539ec"}, - {file = "httptools-0.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:135fbe974b3718eada677229312e97f3b31f8a9c8ffa3ae6f565bf808d5b6bcb"}, - {file = "httptools-0.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5"}, - {file = "httptools-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5"}, - {file = "httptools-0.7.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03"}, - {file = "httptools-0.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2"}, - {file = "httptools-0.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362"}, - {file = "httptools-0.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c"}, - {file = "httptools-0.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321"}, - {file = "httptools-0.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3"}, - {file = "httptools-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca"}, - {file = "httptools-0.7.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c"}, - {file = "httptools-0.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66"}, - {file = "httptools-0.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346"}, - {file = "httptools-0.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650"}, - {file = "httptools-0.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6"}, - {file = "httptools-0.7.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270"}, - {file = "httptools-0.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3"}, - {file = "httptools-0.7.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1"}, - {file = "httptools-0.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b"}, - {file = "httptools-0.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60"}, - {file = "httptools-0.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca"}, - {file = "httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96"}, - {file = "httptools-0.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ac50afa68945df63ec7a2707c506bd02239272288add34539a2ef527254626a4"}, - {file = "httptools-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de987bb4e7ac95b99b805b99e0aae0ad51ae61df4263459d36e07cf4052d8b3a"}, - {file = "httptools-0.7.1-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d169162803a24425eb5e4d51d79cbf429fd7a491b9e570a55f495ea55b26f0bf"}, - {file = "httptools-0.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49794f9250188a57fa73c706b46cb21a313edb00d337ca4ce1a011fe3c760b28"}, - {file = "httptools-0.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:aeefa0648362bb97a7d6b5ff770bfb774930a327d7f65f8208394856862de517"}, - {file = "httptools-0.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0d92b10dbf0b3da4823cde6a96d18e6ae358a9daa741c71448975f6a2c339cad"}, - {file = "httptools-0.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:5ddbd045cfcb073db2449563dd479057f2c2b681ebc232380e63ef15edc9c023"}, - {file = "httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9"}, + {file = "httptools-0.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bf3b6f807c8541503cecfbb8a8dffb385640d0d96102f3d112aa8740f9b7c826"}, + {file = "httptools-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da684f2e1aa2ee9bdcb083f3f3a68c5956750b375bc5df864d3a5f0c42a40b77"}, + {file = "httptools-0.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6f21e2a3b0067bbe7f67e34cfd16276af556e5e52f4c7503be0cb5f90e905e4"}, + {file = "httptools-0.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb"}, + {file = "httptools-0.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0d726cc107fceb7d45f978483b4b70dd8caa836f5914d3434bb18628eb73813"}, + {file = "httptools-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9878eb2785ba5eb70631ad269b37976f73d647955e26c91d490eb8a4edfda4ba"}, + {file = "httptools-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b205e5f5523fa039679da0dfe5a10132b2a4abeae6a86fdd1ddc035f7f836557"}, + {file = "httptools-0.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed377e64805bdba4943c82717333f8f8603a13b09aff9cead2717c6c817fb168"}, + {file = "httptools-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9518c406d7b310f05adb1a37f80acabac40504a575d7c0da6d3e365c695ac20d"}, + {file = "httptools-0.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376"}, + {file = "httptools-0.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d"}, + {file = "httptools-0.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085"}, + {file = "httptools-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124"}, + {file = "httptools-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1b4c8e7a489a0d750d91894e9a8cdc295838f1924c0ca903ae993456fddec07"}, + {file = "httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d"}, + {file = "httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5"}, + {file = "httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2"}, + {file = "httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09"}, + {file = "httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a"}, + {file = "httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745"}, + {file = "httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150"}, + {file = "httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8"}, + {file = "httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c"}, + {file = "httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7"}, + {file = "httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d"}, + {file = "httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681"}, + {file = "httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683"}, + {file = "httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1"}, + {file = "httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6"}, + {file = "httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b"}, + {file = "httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0"}, + {file = "httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e"}, + {file = "httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b"}, + {file = "httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0"}, + {file = "httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527"}, + {file = "httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568"}, + {file = "httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b"}, + {file = "httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca"}, + {file = "httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f"}, + {file = "httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d"}, + {file = "httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081"}, + {file = "httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77"}, + {file = "httptools-0.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:df31ef5494f406ab6cf827b7e64a22841c6e2d654100e6a116ea15b46d02d5e8"}, + {file = "httptools-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5eb911c515b96ee44bbd861e42cbefc488681d450545b1d02127f6136e3a86f5"}, + {file = "httptools-0.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c08ffe3e79756e0963cbc8fe410139f38a5884874b6f2e17761bef6563fdcd9b"}, + {file = "httptools-0.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe2a4c95aeba2209434e7b31172da572846cae8ca0bf1e7013e61b99fbbf5e72"}, + {file = "httptools-0.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7b71e7d7031928c650e1006e6c03e911bf967f7c69c011d37d541c3e7bf55005"}, + {file = "httptools-0.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9fc1644f415372cec4f8a5be3a64183737398f10dbb1263602a036427fe75247"}, + {file = "httptools-0.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5d7fa4ba7292c1139c0526f0b5aad507c6263c948206ea1b1cbca015c8af1b62"}, + {file = "httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999"}, ] [[package]] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 2e3adc4fe213..6f9083cbc70d 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "5712b9a35de51dbf233f890d8b4ea75c049f5af7", - "timestamp": "2026-05-25T02:47:40Z" + "commit_hash": "68d3db1423ad175a4a5492acacfa592470d16702", + "timestamp": "2026-05-26T02:47:45Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index ee6c15d68330..9c9e7c2dbd8c 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -568,14 +568,14 @@ serving = ["fastapi (>=0.104.0)", "peft", "pydantic (>=2.0.0)", "uvicorn (>=0.24 [[package]] name = "cache-dit" -version = "1.3.7" +version = "1.3.8" description = "Cache-DiT: A PyTorch-native Inference Engine with Cache, Parallelism and Quantization for Diffusion Transformers." optional = false python-versions = ">=3.12" groups = ["main"] markers = "python_version == \"3.12\"" files = [ - {file = "cache_dit-1.3.7-py3-none-any.whl", hash = "sha256:58d1db7f6ee2560af73bf4a6cbe2206ad09a0eec6ef227f32dff9064519373bb"}, + {file = "cache_dit-1.3.8-py3-none-any.whl", hash = "sha256:7c7b2804929c770bbcd4c12ca25697066a51b1b72aec841351c084109cb9fa6a"}, ] [package.dependencies] @@ -1242,32 +1242,29 @@ all = ["cuda-bindings[all] (>=13.2.0,<13.3.0)"] [[package]] name = "cuda-tile" -version = "1.3.0" +version = "1.1.0" description = "CUDA Tile Compiler" optional = false python-versions = "<3.14,>=3.10" groups = ["main"] files = [ - {file = "cuda_tile-1.3.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c55616c648f06f84808648a521c67f2d7c790574d6b53ddf8c3bfbc995d36d45"}, - {file = "cuda_tile-1.3.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:8c71c2fd9b96c054c126a218f9927c8c8dde72441a532464551b865b416d452a"}, - {file = "cuda_tile-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:339769f95b3a5453b7f416da6d1285f24d0daf3a700a895b68dee3fa6fc93e8f"}, - {file = "cuda_tile-1.3.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:59d9843fa723ceb4d680ec246e12e3ded857266e4c2bf5c5d21e530d6d765060"}, - {file = "cuda_tile-1.3.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:2888d6b89fae053a53ca7bb703c508a5cf90671d266934573c5b6c25978022c4"}, - {file = "cuda_tile-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:791b363251fbc64db4402d92153ba3d14bc0aaa4d218cea66562af02a7a76bd9"}, - {file = "cuda_tile-1.3.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:375316b64c51ee7cfadb2f170a30c1547bc41eb39f1e233a6556713857d2e81f"}, - {file = "cuda_tile-1.3.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:e4865acbff1172aaee304bf9c550586088d8b4545a384423597a590899386709"}, - {file = "cuda_tile-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:93e20ed31e46e5bf704fb31d13e1c08338d2177838798876f7ee9ec4384b75ba"}, - {file = "cuda_tile-1.3.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:8a9bd4dae193cddf438f55d617b6f25b4b0b0fcf4ac4acde7d2695898e396c30"}, - {file = "cuda_tile-1.3.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:a44a81e255fdb7bf8e1f7511fe3a019e6045024574509ea8548e0f71f25f8473"}, - {file = "cuda_tile-1.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:efcb93c25563fe23d6aa083c22893fd703122eaf684b0d36874982d28a6dad0b"}, + {file = "cuda_tile-1.1.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:cb7a909bb76170f492537efccba89131022332c66acc4b0d56f3ec81f0f0f102"}, + {file = "cuda_tile-1.1.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:d0e0b1e49d57123c79403dae2aa3ce372d3f75298b4f1be904e3e8285a1161c5"}, + {file = "cuda_tile-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0a923938864eba2f2c00a175da2ee7f9632f881581574cd7c637b458f2a650f4"}, + {file = "cuda_tile-1.1.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:8cc2d5c4e263d369514c577c175ee6458a80bd50357351f5104790b46933842b"}, + {file = "cuda_tile-1.1.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:eb75d3caf9555f0b160f009e0c2dd995ccb986e4a766fe4278eedb52d086c89f"}, + {file = "cuda_tile-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4c4b09e0b9b6fb6514aa6192cf54113bcb5df4ca7fe7b1ff6c4773091fa0d13d"}, + {file = "cuda_tile-1.1.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:b5a777cfa88411ee12a77440f015ef5136a3c70208391f00626dd3f83bcdbb65"}, + {file = "cuda_tile-1.1.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:35b3e6a9368390339211ed7f1ee3416a829a44d1c1b335ef0df814a6e6ff4902"}, + {file = "cuda_tile-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a38e09f6335a93fa94a41c07f3cf9f6089f75eeafce77443cf7e30bf1abadbab"}, + {file = "cuda_tile-1.1.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:9008968989e83ffe705c9c8ac0d3fb4d9218fd98468eeff85f50b1ecc062d4c9"}, + {file = "cuda_tile-1.1.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:2758aac91c7c5a9cdacd39c44c2876aa8cb96d72a8a861fbe240c50b5e9ba235"}, + {file = "cuda_tile-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:52458f9329fed18630fc16505d7a21462481ad98ce2bb91a88a38fe943a831a9"}, ] [package.dependencies] typing-extensions = "*" -[package.extras] -tileiras = ["nvidia-cuda-nvcc (>=13.2,<13.3)", "nvidia-cuda-tileiras (>=13.2,<13.3)", "nvidia-nvvm (>=13.2,<13.3)"] - [[package]] name = "cuda-toolkit" version = "13.0.2" @@ -1568,20 +1565,18 @@ files = [ [[package]] name = "flashinfer-python" -version = "0.6.11.post1" +version = "0.6.12rc1" description = "FlashInfer: Kernel Library for LLM Serving" optional = false python-versions = "<4.0,>=3.10" groups = ["main"] -files = [ - {file = "flashinfer_python-0.6.11.post1-py3-none-any.whl", hash = "sha256:52c6dbd1262ff0d46dce0253996108e0c55e86faa7c25a1bed7072b31d23551a"}, - {file = "flashinfer_python-0.6.11.post1.tar.gz", hash = "sha256:b16ac9a4f229a6a0d875a2fd76dfd935d5c62e4a99ea7b128fe4b457530a5aed"}, -] +files = [] +develop = false [package.dependencies] apache-tvm-ffi = ">=0.1.6,<0.1.8 || >0.1.8,<0.1.8.post0 || >0.1.8.post0,<0.2" click = "*" -cuda-tile = "*" +cuda-tile = {version = "*", extras = ["tileiras"]} einops = "*" ninja = "*" numpy = "*" @@ -1597,6 +1592,13 @@ tqdm = "*" [package.extras] cu12 = ["nvidia-cutlass-dsl (>=4.5.0)"] cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.5.0)"] +nvep = ["cuda-python (>=13.0)"] + +[package.source] +type = "git" +url = "https://github.com/flashinfer-ai/flashinfer.git" +reference = "v0.6.12rc1" +resolved_reference = "a94541edac53dd5b3e3c944d30ed290e3f7accf0" [[package]] name = "fonttools" @@ -4789,17 +4791,18 @@ test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_na [[package]] name = "pulp" -version = "3.3.1" +version = "3.3.2" description = "PuLP is an LP modeler written in python. PuLP can generate MPS or LP files and call GLPK, COIN CLP/CBC, CPLEX, and GUROBI to solve linear problems." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pulp-3.3.1-py3-none-any.whl", hash = "sha256:45aa73db3368eb13b156564e092784c8fa0c1feefa64c2afb0410d9dc0bb5cd9"}, - {file = "pulp-3.3.1.tar.gz", hash = "sha256:a9ec237a56981b11c2096e8ba6bb72006833410ba5b400aa257426f85df5e293"}, + {file = "pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0"}, + {file = "pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0"}, ] [package.extras] +cbc = ["cbcbox"] copt = ["coptpy"] cplex = ["cplex ; sys_platform != \"darwin\" or python_full_version < \"3.12.0\""] gurobi = ["gurobipy"] @@ -7308,4 +7311,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "40d4bdcc33ff08c4d66194f59c11519f058eaa20245742bd97d0b102c52a3ad7" +content-hash = "d869c67d8f42a41bbe0f7dc231030cbef327c25feb579b859866bf864492a3ae" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 6839347ecbb2..5a96a617f64c 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "polygraphy (>=0.49.26,<0.50.0)", "psutil (>=7.2.2,<8.0.0)", "nvidia-ml-py (>=13)", - "pulp (>=3.3.1,<4.0.0)", + "pulp (>=3.3.2,<4.0.0)", "h5py (==3.12.1)", "strenum (>=0.4.15,<0.5.0)", "sentencepiece (>=0.1.99)", @@ -54,7 +54,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python (==0.6.12rc1)", + "flashinfer-python @ git+https://github.com/flashinfer-ai/flashinfer.git@v0.6.12rc1", "opencv-python-headless (>=4.13.0.92,<5.0.0.0)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", From 983541534f8454e59d47e8e9cf3d314e75af679a Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 26 May 2026 11:36:32 +0800 Subject: [PATCH 027/308] [https://nvbugs/6186880][fix] In deep_ep.py, fall back to the pre-quant dispatch path when hidden_states_sf is (#14404) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../_torch/modules/fused_moe/communication/deep_ep.py | 7 ++++++- tests/integration/test_lists/waives.txt | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep.py b/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep.py index d3e1c68edd98..b7d962806ef4 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep.py @@ -190,7 +190,12 @@ def dispatch( if token_final_scales is not None and token_final_scales.dtype != torch.float32: token_final_scales = token_final_scales.to(torch.float32) - if not self.supports_post_quant_dispatch(): + # Even when the comm strategy declares post-quant dispatch support, the + # caller may still pass `hidden_states_sf=None` if the local MoE module + # is excluded from quantization (e.g. an MTP layer with a non-quantized + # MoE in a partially-quantized checkpoint). In that case the data is + # actually unquantized, so dispatch the plain hidden_states. + if not self.supports_post_quant_dispatch() or hidden_states_sf is None: # Pre-quant dispatch (unquantized data) ( hidden_states, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 26a0305d3af7..2b4b19fbecda 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -34,10 +34,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_s accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_mtp1] SKIP (https://nvbugs/6185196) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp0] SKIP (https://nvbugs/6186880) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp1] SKIP (https://nvbugs/6186880) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp2] SKIP (https://nvbugs/6186880) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp3] SKIP (https://nvbugs/6186880) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[baseline] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[mtp3_fp8kv_chunked] SKIP (https://nvbugs/5989920) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) From b1d23c743502f2182bbaf387052e15f99fe8671a Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Tue, 26 May 2026 11:44:40 +0800 Subject: [PATCH 028/308] [None][infra] Waive 2 failed cases for main in post-merge 2734 (#14526) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2b4b19fbecda..4c8778230b8e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -242,6 +242,7 @@ examples/test_visual_gen.py::test_flux2_lpips_against_golden SKIP (https://nvbug examples/test_visual_gen.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6215688) examples/test_visual_gen.py::test_wan21_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) examples/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) +examples/test_visual_gen.py::test_wan_t2v_example SKIP (https://nvbugs/6215688) examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) full:A10/unittest/kv_cache_manager_v2_tests/ SKIP (https://nvbugs/5841954) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6185480) @@ -347,6 +348,7 @@ stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_360 stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) +test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/5989907) test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6114608) From 526d7ee6753f9164341fb0bf9368ce3d6a274138 Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Tue, 26 May 2026 11:45:57 +0800 Subject: [PATCH 029/308] [None][infra] Waive 1 failed cases for main in post-merge 2735 (#14542) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 4c8778230b8e..541a1e5a0a57 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -79,6 +79,7 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughpu accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5772993) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-no_overlap_scheduler] SKIP (https://nvbugs/6220815) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] SKIP (https://nvbugs/6215702) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) From 92ac6fc6ee83a1efe661f06b8afeff3e9ea8ef06 Mon Sep 17 00:00:00 2001 From: Kanghwan <861393+karljang@users.noreply.github.com> Date: Tue, 26 May 2026 00:09:44 -0700 Subject: [PATCH 030/308] [#11257][feat] Add LoRA support to llmapi triton backend (#14079) Signed-off-by: Kanghwan Jang <861393+karljang@users.noreply.github.com> --- ruff-legacy-baseline.json | 4 +- .../integration/defs/triton_server/common.py | 17 ++ .../defs/triton_server/conftest.py | 19 +++ .../defs/triton_server/test_triton_llm.py | 95 ++++++++++- .../test_lists/qa/llm_triton_integration.txt | 1 + .../integration/test_lists/test-db/l0_a30.yml | 1 + .../llmapi/tensorrt_llm/1/helpers.py | 110 +++++++++++-- .../all_models/llmapi/tensorrt_llm/1/model.py | 64 ++++++-- .../llmapi/tensorrt_llm/config.pbtxt | 33 ++++ .../tests/test_llmapi_python_backend.py | 110 ++++++++++++- triton_backend/tools/llmapi_client.py | 149 +++++++++++++++--- 11 files changed, 554 insertions(+), 49 deletions(-) diff --git a/ruff-legacy-baseline.json b/ruff-legacy-baseline.json index a5a4e4aaf8ec..cedd33dfadbe 100644 --- a/ruff-legacy-baseline.json +++ b/ruff-legacy-baseline.json @@ -1,7 +1,7 @@ { "_meta": { "generated_by": "scripts/legacy_utils.py lint-update-violations", - "total_violations": 5333, + "total_violations": 5338, "total_files": 492 }, ".github/scripts/label_community_user.py": { @@ -2015,7 +2015,7 @@ }, "tests/integration/defs/triton_server/test_triton_llm.py": { "F403": 2, - "F405": 169 + "F405": 174 }, "tests/integration/defs/triton_server/test_triton_memleak.py": { "F403": 2, diff --git a/tests/integration/defs/triton_server/common.py b/tests/integration/defs/triton_server/common.py index fb41bdc00c53..a9416cf838c7 100644 --- a/tests/integration/defs/triton_server/common.py +++ b/tests/integration/defs/triton_server/common.py @@ -202,6 +202,23 @@ def prepare_llmapi_model_repo(llm_backend_repo_root, new_model_repo): check_call(f"cp -R {origin_model_repo} {new_model_repo}", shell=True) +def set_llmapi_decoupled_mode(new_model_repo, decoupled): + """Set Triton transaction policy via config.pbtxt for the llmapi model. + + Append `model_transaction_policy { decoupled: ... }` so Triton sees the + model as decoupled (or not). Required because launch_triton_server.py + uses `--disable-auto-complete-config`, which prevents the llmapi + model.py `auto_complete_config` callback (where decoupled is taken from + model.yaml) from running. Without this, Triton treats the model as + non-decoupled and rejects multi-response sends from the streaming path. + """ + config_pbtxt = os.path.join(new_model_repo, "tensorrt_llm", "config.pbtxt") + with open(config_pbtxt, "a") as f: + f.write("\nmodel_transaction_policy {\n" + f" decoupled: {str(bool(decoupled)).lower()}\n" + "}\n") + + def modify_ib_config_pbtxt(REPO_PATH, DECODER_ENGINE_PATH, TOKENIZER_PATH, diff --git a/tests/integration/defs/triton_server/conftest.py b/tests/integration/defs/triton_server/conftest.py index a277a30d906c..18f6087ff6b9 100644 --- a/tests/integration/defs/triton_server/conftest.py +++ b/tests/integration/defs/triton_server/conftest.py @@ -369,6 +369,25 @@ def gpt_2b_lora_model_root(): return gpt_2b_lora_model_root +@pytest.fixture(scope="session") +def tiny_llama_lora_model_root(): + """HF-format LoRA adapter for TinyLlama-1.1B-Chat-v1.0. + + Used by the llmapi triton backend's E2E LoRA test + (`test_llmapi_lora`). Same base model as `tiny_llama_model_root`. + """ + models_root = llm_models_root() + assert models_root, "Did you set LLM_MODELS_ROOT?" + tiny_llama_lora_model_root = os.path.join( + models_root, "llama-models-v2", + "TinyLlama-1.1B-Chat-v1.0-mental-health-conversational") + + assert os.path.exists( + tiny_llama_lora_model_root + ), f"{tiny_llama_lora_model_root} does not exist under NFS LLM_MODELS_ROOT dir" + return tiny_llama_lora_model_root + + @pytest.fixture(scope="session") def blip2_opt_model_root(): models_root = llm_models_root() diff --git a/tests/integration/defs/triton_server/test_triton_llm.py b/tests/integration/defs/triton_server/test_triton_llm.py index 36287d1f55f4..857617eb3237 100644 --- a/tests/integration/defs/triton_server/test_triton_llm.py +++ b/tests/integration/defs/triton_server/test_triton_llm.py @@ -3644,6 +3644,7 @@ def test_llmapi_backend(E2E_MODEL_NAME, DECOUPLED_MODE, TRITON_MAX_BATCH_SIZE, # Prepare model repo new_model_repo = os.path.join(llm_backend_repo_root, "triton_repo") prepare_llmapi_model_repo(llm_backend_repo_root, new_model_repo) + set_llmapi_decoupled_mode(new_model_repo, DECOUPLED_MODE) model_config_path = os.path.join(new_model_repo, "tensorrt_llm", "1", "model.yaml") with open(model_config_path, "r") as f: @@ -3709,6 +3710,12 @@ def test_llmapi_backend(E2E_MODEL_NAME, DECOUPLED_MODE, TRITON_MAX_BATCH_SIZE, f"--tensorrt-llm-model-name={E2E_MODEL_NAME}", f"--protocol={protocol}", f"--test-llmapi", + ] + if DECOUPLED_MODE: + # Triton rejects ModelInfer RPC on decoupled models; the + # benchmark must use async_stream_infer in that mode. + run_cmd.append("--decoupled") + run_cmd += [ 'dataset', f"--dataset={os.path.join(llm_backend_dataset_root, 'mini_cnn_eval.json')}", f"--tokenizer-dir={tiny_llama_model_root}", @@ -3722,6 +3729,11 @@ def test_llmapi_backend(E2E_MODEL_NAME, DECOUPLED_MODE, TRITON_MAX_BATCH_SIZE, f"{llm_backend_repo_root}/tools/llmapi_client.py", "--request-output-len=200", '--stop-after-ms=25' ] + if DECOUPLED_MODE: + # On a decoupled server, ModelInfer RPC is rejected; + # llmapi_client.py must use the bidirectional stream + # RPC, which it routes through when --streaming is set. + run_cmd.append('--streaming') output = venv_check_output(llm_backend_venv, run_cmd) assert 'Request is cancelled' in output @@ -3731,11 +3743,83 @@ def test_llmapi_backend(E2E_MODEL_NAME, DECOUPLED_MODE, TRITON_MAX_BATCH_SIZE, output = venv_check_output(llm_backend_venv, run_cmd) assert 'Request is cancelled' in output - # Test request cancellation for non-existing request and completed request - run_cmd = [ - f"{llm_backend_repo_root}/tools/tests/test_llmapi_cancel.py" - ] - output = venv_check_output(llm_backend_venv, run_cmd) + # Test request cancellation for non-existing request and + # completed request. The helper script uses ModelInfer RPC + # (async_infer), which Triton rejects on a decoupled model; + # only exercise it in the non-decoupled config. + if not DECOUPLED_MODE: + run_cmd = [ + f"{llm_backend_repo_root}/tools/tests/test_llmapi_cancel.py" + ] + output = venv_check_output(llm_backend_venv, run_cmd) + + +@pytest.mark.parametrize("E2E_MODEL_NAME", ["tensorrt_llm"]) +@pytest.mark.parametrize("TENSOR_PARALLEL_SIZE", ["1"]) +def test_llmapi_lora(E2E_MODEL_NAME, TENSOR_PARALLEL_SIZE, + llm_backend_inflight_batcher_llm_root, llm_backend_venv, + tiny_llama_model_root, tiny_llama_lora_model_root): + """E2E LoRA test for the new llmapi triton backend. + + Templates `model.yaml` with `lora_config:` pointing at a TinyLlama + HF LoRA adapter, launches Triton with the llmapi backend, and sends + one request via `llmapi_client.py --lora-id/--lora-name/--lora-path`. + Asserts that the response carries generated text — proving the new + lora_id/lora_name/lora_path inputs reach `LLM.generate_async( + lora_request=...)` and adapter-applied inference completes. + """ + llm_backend_repo_root = os.path.join(LLM_ROOT, "triton_backend") + + if torch.cuda.device_count() < int(TENSOR_PARALLEL_SIZE): + pytest.skip("Skipping. Not enough GPUs.") + + # Prepare model repo with lora_config + new_model_repo = os.path.join(llm_backend_repo_root, "triton_repo") + prepare_llmapi_model_repo(llm_backend_repo_root, new_model_repo) + set_llmapi_decoupled_mode(new_model_repo, False) + model_config_path = os.path.join(new_model_repo, "tensorrt_llm", "1", + "model.yaml") + with open(model_config_path, "r") as f: + model_config = yaml.safe_load(f) + model_config["triton_config"]["decoupled"] = False + model_config["triton_config"]["max_batch_size"] = 0 + model_config["tensor_parallel_size"] = int(TENSOR_PARALLEL_SIZE) + model_config["kv_cache_config"] = {"free_gpu_memory_fraction": 0.8} + model_config["model"] = tiny_llama_model_root + model_config["lora_config"] = { + "lora_dir": [tiny_llama_lora_model_root], + "max_lora_rank": 64, + "max_loras": 1, + "max_cpu_loras": 1, + } + with open(model_config_path, "w") as f: + yaml.dump(model_config, f) + + # Launch Triton Server + launch_server_py = os.path.join(llm_backend_repo_root, "scripts", + "launch_triton_server.py") + cmd = (f"python3 {launch_server_py} " + f"--world_size={TENSOR_PARALLEL_SIZE} " + f"--model_repo={new_model_repo} --no-mpi") + print_info(f"DEBUG:: launch_server with args: {cmd}") + check_call(cmd, shell=True) + check_server_ready() + + # Send a LoRA request via llmapi_client.py + run_cmd = [ + f"{llm_backend_repo_root}/tools/llmapi_client.py", + "--text=I've noticed you seem a bit down lately. " + "Is there anything you'd like to talk about?", + "--request-output-len=32", + "--lora-id=0", + "--lora-name=mental-health", + f"--lora-path={tiny_llama_lora_model_root}", + f"--model-name={E2E_MODEL_NAME}", + ] + print_info("DEBUG:: run_cmd: python3 " + " ".join(run_cmd)) + output = venv_check_output(llm_backend_venv, run_cmd) + assert "Output text:" in output, ( + f"Expected 'Output text:' in client output, got: {output[:500]}") def test_llmapi_backend_multi_instance(llm_backend_inflight_batcher_llm_root, @@ -3747,6 +3831,7 @@ def test_llmapi_backend_multi_instance(llm_backend_inflight_batcher_llm_root, # Prepare model repo new_model_repo = os.path.join(llm_backend_repo_root, "triton_repo") prepare_llmapi_model_repo(llm_backend_repo_root, new_model_repo) + set_llmapi_decoupled_mode(new_model_repo, True) # Modify model.yaml model_config_path = os.path.join(new_model_repo, "tensorrt_llm", "1", diff --git a/tests/integration/test_lists/qa/llm_triton_integration.txt b/tests/integration/test_lists/qa/llm_triton_integration.txt index c0f622831fce..29a55da4cddb 100644 --- a/tests/integration/test_lists/qa/llm_triton_integration.txt +++ b/tests/integration/test_lists/qa/llm_triton_integration.txt @@ -538,6 +538,7 @@ triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-te triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm] triton_server/test_triton_llm.py::test_llmapi_backend[4-0-disableDecoupleMode-tensorrt_llm] triton_server/test_triton_llm.py::test_llmapi_backend[4-0-enableDecoupleMode-tensorrt_llm] +triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm] triton_server/test_triton_llm.py::test_llmapi_backend_multi_instance triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-input_only-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble] triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-input_only-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls] diff --git a/tests/integration/test_lists/test-db/l0_a30.yml b/tests/integration/test_lists/test-db/l0_a30.yml index b582b0a02f89..34b835c961d4 100644 --- a/tests/integration/test_lists/test-db/l0_a30.yml +++ b/tests/integration/test_lists/test-db/l0_a30.yml @@ -207,6 +207,7 @@ l0_a30: - triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls] - triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-tensorrt_llm] - triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm] + - triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm] # ------------- AutoDeploy Backend Stages --------------- - condition: ranges: diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py index 22f2562bf4a8..f0a2547e4946 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py @@ -1,8 +1,50 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + import numpy as np import torch import triton_python_backend_utils as pb_utils from torch.utils.dlpack import from_dlpack +_LORA_CKPT_SOURCES = ("hf", "nemo") + + +def _decode_string_scalar(value): + """Decode a Triton STRING-tensor scalar to a Python str. + + A STRING tensor's `.item()` may return `bytes`, `numpy.bytes_`, or + `str` depending on whether the array dtype is object/bytes-based or + unicode-based, and on numpy version. Normalize to `str`. + """ + if isinstance(value, (bytes, np.bytes_)): + return value.decode("utf-8") + return str(value) + def convert_request_input_to_dict(request, param_mappings, default_values, batch_size, batch_index): @@ -19,9 +61,9 @@ def convert_request_input_to_dict(request, param_mappings, default_values, def get_sampling_params_from_request(request, batch_size=1, batch_index=0): - """ - Helper function to get trtllm.SamplingParams (LLMAPI) parameters from request - Used in llmapi/tensorrt_llm + """Get trtllm.SamplingParams (LLMAPI) parameters from request. + + Used in llmapi/tensorrt_llm. """ sampling_params_args = [ 'best_of', @@ -52,9 +94,9 @@ def get_sampling_params_from_request(request, batch_size=1, batch_index=0): def get_output_config_from_request(request, batch_size=1, batch_index=0): - """ - Helper function to get trtllm.SamplingParams (LLMAPI) parameters from request - Used in llmapi/tensorrt_llm + """Get trtllm output-config parameters from request. + + Used in llmapi/tensorrt_llm. """ output_config_args = [ 'return_finish_reason', 'return_stop_reason', @@ -76,15 +118,65 @@ def get_output_config_from_request(request, batch_size=1, batch_index=0): def get_streaming_from_request(request, batch_size=1, batch_index=0): - """ - Helper function to get streaming from request - Used in llmapi/tensorrt_llm + """Get the streaming flag from request. + + Used in llmapi/tensorrt_llm. """ streaming = get_input_scalar_by_name(request, 'streaming', batch_size, batch_index) or False return streaming +def get_lora_request_from_request(request, batch_size=1, batch_index=0): + """Construct a LoRARequest from triton request inputs. + + Returns None if none of lora_id/lora_name/lora_path are provided. If + any of those three is provided, all three are required. The optional + lora_ckpt_source input selects the checkpoint format and defaults to + "hf"; "nemo" is also accepted. + + Used in llmapi/tensorrt_llm. + """ + lora_id = get_input_scalar_by_name(request, 'lora_id', batch_size, + batch_index) + lora_name = get_input_scalar_by_name(request, 'lora_name', batch_size, + batch_index) + lora_path = get_input_scalar_by_name(request, 'lora_path', batch_size, + batch_index) + if lora_id is None and lora_name is None and lora_path is None: + return None + if lora_id is None or lora_name is None or lora_path is None: + raise pb_utils.TritonModelException( + "lora_id, lora_name, and lora_path must all be provided together") + lora_name = _decode_string_scalar(lora_name) + lora_path = _decode_string_scalar(lora_path) + lora_ckpt_source = get_input_scalar_by_name(request, 'lora_ckpt_source', + batch_size, batch_index) + if lora_ckpt_source is None: + lora_ckpt_source = "hf" + else: + lora_ckpt_source = _decode_string_scalar(lora_ckpt_source) + if lora_ckpt_source not in _LORA_CKPT_SOURCES: + raise pb_utils.TritonModelException( + f"lora_ckpt_source must be one of {_LORA_CKPT_SOURCES}, " + f"got {lora_ckpt_source!r}") + # Validate lora_path eagerly so a missing path raises the wrapper's + # TritonModelException (matching the partial-input error path) instead + # of letting LoRARequest.__post_init__ surface a raw ValueError. + if not os.path.exists(lora_path): + raise pb_utils.TritonModelException( + f"lora_path does not exist on the Triton server's filesystem: " + f"{lora_path}") + # Deferred import: importing tensorrt_llm at module load time + # initializes CUDA/MPI in the Triton Python backend process before the + # engine subprocess is spawned. Other helpers follow the same pattern. + from tensorrt_llm.executor.request import LoRARequest + return LoRARequest(lora_name=lora_name, + lora_int_id=int(lora_id), + lora_path=lora_path, + lora_ckpt_source=lora_ckpt_source) + + def get_input_scalar_by_name(request, name, expected_batch_size=1, diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index 804cc81a01f1..edd77e1a26a7 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -40,7 +40,8 @@ import pandas as pd import triton_python_backend_utils as pb_utils import yaml -from helpers import (get_input_tensor_by_name, get_output_config_from_request, +from helpers import (get_input_tensor_by_name, get_lora_request_from_request, + get_output_config_from_request, get_sampling_params_from_request, get_streaming_from_request) @@ -414,6 +415,18 @@ def handle_stop_request(self, triton_user_id, response_sender): req_ids = self.triton_user_id_to_req_ids[triton_user_id] for req_id in req_ids: request_data = self.req_id_to_request_data[req_id] + # Send a COMPLETE_FINAL CANCELLED response on the main + # request's response_sender so an in-flight streaming + # client doesn't wait forever for a final chunk after + # the iterator is aborted. Mirrors what + # cancellation_loop does for Triton-level cancels. + main_response_sender = ( + request_data.triton_request.get_response_sender()) + main_response_sender.send( + pb_utils.InferenceResponse(error=pb_utils.TritonError( + "Request cancelled by client", + pb_utils.TritonError.CANCELLED)), + flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) request_data.response_iterator.abort() del self.req_id_to_request_data[req_id] @@ -434,7 +447,6 @@ def execute(self, requests): """ # TODO: [JIRA-4040] Add health check here for request in requests: - # TODO : [JIRA-4040] Verify Lora if request is not None: assert ( self._llm_engine_shutdown_event.is_set() is False @@ -472,14 +484,17 @@ async def _execute_single_request(self, request): from tensorrt_llm import SamplingParams # TODO: [JIRA-4496] Implement when request contains batched prompts - (prompt, sampling_params, streaming, - output_config) = self._convert_request(request) + (prompt, sampling_params, streaming, output_config, + lora_request) = self._convert_request(request) if streaming and not self.decoupled: raise pb_utils.TritonModelException( "Streaming is only supported in decoupled mode.") # Generate the response. response_iterator = self._llm_engine.generate_async( - prompt, SamplingParams(**sampling_params), streaming) + prompt, + sampling_params=SamplingParams(**sampling_params), + lora_request=lora_request, + streaming=streaming) with self.lock: self.req_id_to_request_data[triton_req_id] = RequestData( @@ -509,6 +524,20 @@ async def _execute_single_request(self, request): self._response_queue.put_nowait( (response_state, response, flags)) + # Ensure COMPLETE_FINAL is always sent for streaming, even if the + # iterator exited without yielding a chunk whose .finished was True + # (e.g. trtllm exits the async generator after the engine completes + # without setting finished on the last RequestOutput). Without this + # marker, clients keep the gRPC stream open forever waiting on the + # next chunk. Mirrors the pattern in tensorrt_llm_bls/1/model.py + # which sends an empty COMPLETE_FINAL after its streaming loop. + if streaming and not response_state["last_response_generated"]: + response_state["last_response_generated"] = True + decrement_ongoing_request_count = False + self._response_queue.put_nowait( + (response_state, pb_utils.InferenceResponse(), + pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)) + # Send the last response which contains all the outputs if not streaming. if not streaming: # If the request was cancelled, we don't need to send the last response @@ -527,13 +556,21 @@ async def _execute_single_request(self, request): except Exception as e: self.logger.log_error(f"[trtllm] Error generating request: {e}") - error = pb_utils.TritonError(f"Error generating request: {e}") - text_output_tensor = pb_utils.Tensor( - "text_output", np.asarray(["N/A"], dtype=self.output_dtype)) - response = pb_utils.InferenceResponse( - output_tensors=[text_output_tensor], error=error) - response_sender.send( - response, flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) + # Skip sending if cancellation_loop or handle_stop_request + # already sent a COMPLETE_FINAL on this response_sender; they + # remove the entry from req_id_to_request_data as the signal. + with self.lock: + was_cancelled = (triton_req_id + not in self.req_id_to_request_data) + if not was_cancelled: + error = pb_utils.TritonError(f"Error generating request: {e}") + text_output_tensor = pb_utils.Tensor( + "text_output", np.asarray(["N/A"], dtype=self.output_dtype)) + response = pb_utils.InferenceResponse( + output_tensors=[text_output_tensor], error=error) + response_sender.send( + response, + flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) raise e finally: @@ -574,7 +611,8 @@ def _convert_request(self, request): sampling_params = get_sampling_params_from_request(request) output_config = get_output_config_from_request(request) streaming = get_streaming_from_request(request) - return prompt, sampling_params, streaming, output_config + lora_request = get_lora_request_from_request(request) + return prompt, sampling_params, streaming, output_config, lora_request def _create_response(self, request_output, output_config): """Process the generated request_output and create the client response. diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt index 7ec628be9aaa..a3f995aae882 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt @@ -148,6 +148,39 @@ input [ data_type: TYPE_BOOL dims: [ 1 ] optional: true + }, + ## LoRA Adapter Selection ## + # All three of lora_id, lora_name, and lora_path must be provided together + # (lora_ckpt_source is optional and defaults to "hf"). + # Adapter weights are read from lora_path on first use and cached by the + # LLM executor keyed on lora_id; subsequent requests sharing the same + # lora_id reuse the cached adapter. If a cached lora_id is evicted, the + # adapter is transparently reloaded from lora_path on next use. + # Note: lora_path must be readable on the Triton server's filesystem + # (e.g. a shared mount), not uploaded inline with the request. + { + name: "lora_id" + data_type: TYPE_UINT64 + dims: [ 1 ] + optional: true + }, + { + name: "lora_name" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true + }, + { + name: "lora_path" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true + }, + { + name: "lora_ckpt_source" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true } ] ################################################################### diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index 6ab4120aa445..b6b79e04aeed 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -37,6 +37,7 @@ sys.modules["triton_python_backend_utils"] = MagicMock() from helpers import (convert_request_input_to_dict, + get_lora_request_from_request, get_output_config_from_request, get_parameter, get_sampling_params_from_request, get_streaming_from_request) @@ -76,6 +77,10 @@ class MockTritonError: message: str +class MockTritonModelException(Exception): + """Stands in for pb_utils.TritonModelException in tests.""" + + @dataclass class MockTritonResponse: tensors: Dict[str, MockTritonTensor] @@ -129,7 +134,8 @@ def apply_patches(): patch("model.pb_utils.InferenceRequest", new=MockTritonRequest).start() patch("model.pb_utils.get_input_tensor_by_name", new=mock_pb_utils_get_input_tensor_by_name_side_effect).start() - patch("model.pb_utils.TritonModelException", new=Exception).start() + patch("model.pb_utils.TritonModelException", + new=MockTritonModelException).start() def inputs(streaming=False): @@ -216,6 +222,108 @@ def test_convert_request_input_to_dict(): } +def test_get_lora_request_from_request(tmp_path): + # Mock the deferred `from tensorrt_llm.executor.request import LoRARequest` + # inside the helper so the test doesn't require a built TRT-LLM. + fake_lora_request_cls = MagicMock() + trtllm_mod = MagicMock() + executor_mod = MagicMock() + request_mod = MagicMock() + request_mod.LoRARequest = fake_lora_request_cls + # Real existing path so the helper's eager os.path.exists check passes. + adapter_dir = str(tmp_path) + + with patch.dict( + sys.modules, { + "tensorrt_llm": trtllm_mod, + "tensorrt_llm.executor": executor_mod, + "tensorrt_llm.executor.request": request_mod, + }): + # No LoRA inputs -> returns None (backwards-compatible default) + request = make_mock_triton_request({"text_input": ["hi"]}) + assert get_lora_request_from_request(request) is None + + # All three inputs (bytes STRING tensors, like dtype=object) -> + # constructs LoRARequest with decoded strings + default ckpt source. + fake_lora_request_cls.reset_mock() + request = make_mock_triton_request({ + "lora_id": [42], + "lora_name": [b"my-adapter"], + "lora_path": [adapter_dir.encode("utf-8")], + }) + result = get_lora_request_from_request(request) + fake_lora_request_cls.assert_called_once_with(lora_name="my-adapter", + lora_int_id=42, + lora_path=adapter_dir, + lora_ckpt_source="hf") + assert result is fake_lora_request_cls.return_value + + # Same inputs but unicode STRING tensors (dtype=' still + # decodes through _decode_string_scalar's str fall-through. + fake_lora_request_cls.reset_mock() + request = make_mock_triton_request({ + "lora_id": [42], + "lora_name": ["unicode-adapter"], + "lora_path": [adapter_dir], + }) + get_lora_request_from_request(request) + fake_lora_request_cls.assert_called_once_with( + lora_name="unicode-adapter", + lora_int_id=42, + lora_path=adapter_dir, + lora_ckpt_source="hf") + + # Explicit lora_ckpt_source="nemo" propagates through. + fake_lora_request_cls.reset_mock() + request = make_mock_triton_request({ + "lora_id": [42], + "lora_name": [b"nemo-adapter"], + "lora_path": [adapter_dir.encode("utf-8")], + "lora_ckpt_source": [b"nemo"], + }) + get_lora_request_from_request(request) + fake_lora_request_cls.assert_called_once_with(lora_name="nemo-adapter", + lora_int_id=42, + lora_path=adapter_dir, + lora_ckpt_source="nemo") + + # Invalid lora_ckpt_source -> raises (must be hf or nemo). + request = make_mock_triton_request({ + "lora_id": [42], + "lora_name": [b"a"], + "lora_path": [adapter_dir.encode("utf-8")], + "lora_ckpt_source": [b"bogus"], + }) + with pytest.raises(MockTritonModelException): + get_lora_request_from_request(request) + + # All three partial-input permutations -> raise TritonModelException. + for partial in ( + { + "lora_id": [42] + }, + { + "lora_name": [b"adapter"], + "lora_path": [adapter_dir.encode("utf-8")] + }, + { + "lora_path": [adapter_dir.encode("utf-8")] + }, + ): + with pytest.raises(MockTritonModelException): + get_lora_request_from_request(make_mock_triton_request(partial)) + + # lora_path that doesn't exist -> raises TritonModelException (not + # raw ValueError from LoRARequest.__post_init__). + request = make_mock_triton_request({ + "lora_id": [42], + "lora_name": [b"a"], + "lora_path": [b"/nonexistent-path-xyzzy"], + }) + with pytest.raises(MockTritonModelException): + get_lora_request_from_request(request) + + def test_get_parameter(): # Test valid parameter cases model_config = { diff --git a/triton_backend/tools/llmapi_client.py b/triton_backend/tools/llmapi_client.py index bd63254faafa..275d4e6aa398 100755 --- a/triton_backend/tools/llmapi_client.py +++ b/triton_backend/tools/llmapi_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -43,12 +43,33 @@ def prepare_tensor(name, input): return t -def _prepare_inputs(prompt, output_len): +def _prepare_inputs(prompt, + output_len, + lora_id=None, + lora_name=None, + lora_path=None, + lora_ckpt_source=None): inputs = [ prepare_tensor("text_input", prompt), prepare_tensor("sampling_param_max_tokens", np.array([output_len], dtype=np.int32)), ] + if lora_id is not None: + inputs.append( + prepare_tensor("lora_id", np.array([lora_id], dtype=np.uint64))) + if lora_name is not None: + inputs.append( + prepare_tensor("lora_name", + np.array([lora_name.encode("utf-8")], dtype=object))) + if lora_path is not None: + inputs.append( + prepare_tensor("lora_path", + np.array([lora_path.encode("utf-8")], dtype=object))) + if lora_ckpt_source is not None: + inputs.append( + prepare_tensor( + "lora_ckpt_source", + np.array([lora_ckpt_source.encode("utf-8")], dtype=object))) return inputs @@ -196,14 +217,61 @@ def callback(user_data, result, error): required=False, default='tensorrt_llm', help='Specify model name') + parser.add_argument( + '--lora-id', + type=int, + required=False, + default=None, + help='LoRA adapter integer id (matches `lora_id` input on the model).', + ) + parser.add_argument( + '--lora-name', + type=str, + required=False, + default=None, + help='LoRA adapter name (matches `lora_name` input on the model).', + ) + parser.add_argument( + '--lora-path', + type=str, + required=False, + default=None, + help= + 'Filesystem path to the LoRA adapter checkpoint readable by the Triton server.', + ) + parser.add_argument( + '--lora-ckpt-source', + type=str, + required=False, + default=None, + choices=("hf", "nemo"), + help='LoRA checkpoint format. Defaults to "hf" on the server side.', + ) FLAGS = parser.parse_args() + # The llmapi triton backend requires lora_id, lora_name, and lora_path + # to be sent together (lora_ckpt_source is optional with default "hf"). + # Fail fast on partial input instead of letting the server return a + # cryptic error response. + lora_triplet = (FLAGS.lora_id, FLAGS.lora_name, FLAGS.lora_path) + if any(v is not None + for v in lora_triplet) and not all(v is not None + for v in lora_triplet): + parser.error( + "--lora-id, --lora-name, and --lora-path must be provided together." + ) + input_data = np.array([FLAGS.text], dtype=object) output_len = FLAGS.request_output_len - inputs = _prepare_inputs(input_data, output_len) + inputs = _prepare_inputs(input_data, + output_len, + lora_id=FLAGS.lora_id, + lora_name=FLAGS.lora_name, + lora_path=FLAGS.lora_path, + lora_ckpt_source=FLAGS.lora_ckpt_source) stop_inputs = None if FLAGS.stop_after_ms > 0 and not FLAGS.stop_via_request_cancel: @@ -220,14 +288,29 @@ def callback(user_data, result, error): certificate_chain=FLAGS.certificate_chain, ) as triton_client: try: - # Send request - infer_future = triton_client.async_infer( - FLAGS.model_name, - inputs, - outputs=None, - request_id=request_id, - callback=partial(callback, user_data), - parameters={'Streaming': FLAGS.streaming}) + # Triton rejects ModelInfer RPC (used by async_infer) on + # decoupled models. Streaming requests must use the + # bidirectional stream RPC. Cancellation (via either stop + # tensor or request-cancel) also needs the stream path when + # the server is decoupled. + use_stream_rpc = bool(FLAGS.streaming) or FLAGS.stop_after_ms > 0 + infer_future = None + if use_stream_rpc: + triton_client.start_stream( + callback=partial(callback, user_data)) + triton_client.async_stream_infer( + FLAGS.model_name, + inputs, + request_id=request_id, + parameters={'Streaming': FLAGS.streaming}) + else: + infer_future = triton_client.async_infer( + FLAGS.model_name, + inputs, + outputs=None, + request_id=request_id, + callback=partial(callback, user_data), + parameters={'Streaming': FLAGS.streaming}) expected_responses = 1 @@ -236,14 +319,30 @@ def callback(user_data, result, error): time.sleep(FLAGS.stop_after_ms / 1000.0) if FLAGS.stop_via_request_cancel: - infer_future.cancel() + if use_stream_rpc: + # cancel_requests=True closes the bidi stream + # AND cancels in-flight infers, which the + # cancellation_loop in model.py picks up via + # request.is_cancelled() and reports back as + # StatusCode.CANCELLED. Without the flag, + # stop_stream waits for completion instead. + triton_client.stop_stream(cancel_requests=True) + else: + infer_future.cancel() else: - triton_client.async_infer( - FLAGS.model_name, - stop_inputs, - request_id=request_id, - callback=partial(callback, user_data), - parameters={'Streaming': FLAGS.streaming}) + if use_stream_rpc: + triton_client.async_stream_infer( + FLAGS.model_name, + stop_inputs, + request_id=request_id, + parameters={'Streaming': FLAGS.streaming}) + else: + triton_client.async_infer( + FLAGS.model_name, + stop_inputs, + request_id=request_id, + callback=partial(callback, user_data), + parameters={'Streaming': FLAGS.streaming}) processed_count = 0 while processed_count < expected_responses: @@ -254,7 +353,19 @@ def callback(user_data, result, error): break if type(result) == InferenceServerException: - if result.status() == "StatusCode.CANCELLED": + # async_infer surfaces cancellation as a real gRPC + # status `StatusCode.CANCELLED`, but the bidi stream + # RPC delivers cancellation as an + # InferenceServerException whose status is None and + # whose message is the one model.py's + # cancellation_loop/handle_stop_request set on the + # TritonError (`Request cancelled by client`). + status = result.status() + msg = str(result).lower() + is_cancelled = (status == "StatusCode.CANCELLED" + or "cancelled by client" in msg + or "request cancelled" in msg) + if is_cancelled: print("Request is cancelled") else: print("Received an error from server:") From 8f052f4fa9e9dccd84211dab97b96b412ca48391 Mon Sep 17 00:00:00 2001 From: Zhenhuan Chen Date: Tue, 26 May 2026 15:17:15 +0800 Subject: [PATCH 031/308] [None][chore] Include layer_idx in MoE backend fallback warnings (#13409) Signed-off-by: Zhenhuan Chen --- .../_torch/models/modeling_qwen3_moe.py | 2 +- .../_torch/modules/fused_moe/create_moe.py | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_moe.py b/tensorrt_llm/_torch/models/modeling_qwen3_moe.py index d9541dc60bad..43b4499f4874 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_moe.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_moe.py @@ -111,7 +111,7 @@ def __init__( dtype=config.torch_dtype, apply_routing=False, routing_method_type=RoutingMethodType.Renormalize, - moe_backend_cls=get_moe_cls(model_config), + moe_backend_cls=get_moe_cls(model_config, layer_idx=layer_idx), ) self.weight_loading_mode = MoEWeightLoadingMode.FUSED_GATE_UP_PROJ if config.model_type == "qwen3_vl_moe_text" else MoEWeightLoadingMode.VANILLA diff --git a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py index be26f03abeb0..7cc3c0af3d80 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py @@ -27,12 +27,15 @@ def get_moe_cls( - model_config: ModelConfig, - override_quant_config: Optional[QuantConfig] = None) -> Type[MoE]: + model_config: ModelConfig, + override_quant_config: Optional[QuantConfig] = None, + layer_idx: Optional[int] = None, +) -> Type[MoE]: moe_backend = model_config.moe_backend quant_config = model_config.quant_config if override_quant_config is not None: quant_config = override_quant_config + layer_prefix = f"[layer_idx={layer_idx}] " if layer_idx is not None else "" if moe_backend.upper() == "CUTLASS": return CutlassFusedMoE elif moe_backend.upper() == "VANILLA": @@ -67,7 +70,7 @@ def get_moe_cls( return CuteDslFusedMoE else: logger.warning( - "CuteDslFusedMoE only supports fp8_block_scales and nvfp4. " + f"{layer_prefix}CuteDslFusedMoE only supports fp8_block_scales and nvfp4. " f"Check out details in quant_config: {quant_config}. Using CutlassFusedMoE instead." ) return CutlassFusedMoE @@ -76,7 +79,7 @@ def get_moe_cls( elif moe_backend.upper() == "DENSEGEMM": if quant_config is None or not quant_config.quant_mode.has_nvfp4(): logger.warning( - "DenseGEMMFusedMoE only supports nvfp4. " + f"{layer_prefix}DenseGEMMFusedMoE only supports nvfp4. " f"Check out details in quant_config: {quant_config}. Using CutlassFusedMoE instead." ) return CutlassFusedMoE @@ -85,7 +88,7 @@ def get_moe_cls( sm_version = get_sm_version() if sm_version not in DenseGEMMFusedMoE._SUPPORTED_SM_VERSIONS: logger.warning( - f"DenseGEMMFusedMoE only supports SM {DenseGEMMFusedMoE._SUPPORTED_SM_VERSIONS} " + f"{layer_prefix}DenseGEMMFusedMoE only supports SM {DenseGEMMFusedMoE._SUPPORTED_SM_VERSIONS} " f"(got SM {sm_version}). Using CutlassFusedMoE instead.") return CutlassFusedMoE return DenseGEMMFusedMoE @@ -109,7 +112,7 @@ def get_moe_cls( "trtllm_bf16_moe support, but it is not available.") else: logger.warning( - "TRTLLMGenFusedMoE only supports fp8_block_scales, nvfp4, w4a16_mxfp4, w4a8_nvfp4_fp8, w4a8_mxfp4_fp8, and w4a8_mxfp4_mxfp8. " + f"{layer_prefix}TRTLLMGenFusedMoE only supports fp8_block_scales, nvfp4, w4a16_mxfp4, w4a8_nvfp4_fp8, w4a8_mxfp4_fp8, and w4a8_mxfp4_mxfp8. " f"Check out details in quant_config: {quant_config}. Using CutlassFusedMoE instead." ) return CutlassFusedMoE @@ -167,11 +170,13 @@ def get_moe_cls( def resolve_moe_cls( - model_config: ModelConfig, - routing_method: BaseMoeRoutingMethod, - dtype: Optional[torch.dtype], - override_quant_config: Optional[QuantConfig] = None) -> Type[MoE]: - moe_cls = get_moe_cls(model_config, override_quant_config) + model_config: ModelConfig, + routing_method: BaseMoeRoutingMethod, + dtype: Optional[torch.dtype], + override_quant_config: Optional[QuantConfig] = None, + layer_idx: Optional[int] = None, +) -> Type[MoE]: + moe_cls = get_moe_cls(model_config, override_quant_config, layer_idx) effective_quant_config = override_quant_config or model_config.quant_config has_quant = (effective_quant_config is not None @@ -517,7 +522,7 @@ def create_moe( dtype = pretrained_config.torch_dtype moe_cls = resolve_moe_cls(model_config, routing_method, dtype, - override_quant_config) + override_quant_config, layer_idx) enable_configurable_moe = os.environ.get("ENABLE_CONFIGURABLE_MOE", "1") == "1" From 17cf970f2f0d19871028c1098b6bd46779a9de42 Mon Sep 17 00:00:00 2001 From: fredricz-20070104 <226039983+fredricz-20070104@users.noreply.github.com> Date: Tue, 26 May 2026 16:11:22 +0800 Subject: [PATCH 032/308] [None][chore] Add disagg local one-step run script for CI submit (#14557) Signed-off-by: FredricZ-2007 <226039983+fredricz-20070104@users.noreply.github.com> --- .gitignore | 1 + jenkins/scripts/perf/README.md | 161 +++++++++++ .../scripts/perf/local/configs/example.conf | 101 +++++++ jenkins/scripts/perf/local/run_disagg.sh | 268 ++++++++++++++++++ 4 files changed, 531 insertions(+) create mode 100644 jenkins/scripts/perf/local/configs/example.conf create mode 100755 jenkins/scripts/perf/local/run_disagg.sh diff --git a/.gitignore b/.gitignore index a3113ffccb54..a09c60676271 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,4 @@ enroot/tensorrt_llm.devel.sqsh .claude/agent-memory/ .claude/agent-tests/perf-test-sync/report.html .claude/agent-tests/perf-test-sync/results.json +.claude/settings.json diff --git a/jenkins/scripts/perf/README.md b/jenkins/scripts/perf/README.md index 170a7eaac121..5e68643ec06d 100644 --- a/jenkins/scripts/perf/README.md +++ b/jenkins/scripts/perf/README.md @@ -15,6 +15,9 @@ jenkins/scripts/perf/ submit.py # Local submit script (aggregated and disaggregated) slurm_install.sh # Build wheel + pip install inside container slurm_run.sh # Run pytest inside container + run_disagg.sh # Config-driven wrapper: read .conf, generate slurm_launch.sh, sbatch + configs/ # Per-cluster / per-case .conf files consumed by run_disagg.sh + example.conf # Reference template — copy and tweak perf_utils.py # Shared utilities (regression detection, baseline, charts, OpenSearch queries) get_pre_merge_html.py # Pre-merge HTML report with history, baseline, and threshold perf_sanity_triage.py # Query/update OpenSearch data and send Slack notifications @@ -61,6 +64,164 @@ Used by the **CI pipeline** (called from `jenkins/L0_Test.groovy`'s script prefix and srun args from the CI pipeline and combines them with disagg-specific environment variables and hardware configuration to generate `slurm_launch.sh`. +## Running Local Tests with `run_disagg.sh` + +`local/run_disagg.sh` is a config-driven wrapper around `local/submit.py`. It reads a +single `.conf` file, applies defaults, generates one `slurm_launch.sh` per test, and +submits each via `sbatch`. Despite the historical name, it supports **both** aggregated +and disaggregated runs — the runtime mode (and the right draft template) is derived +automatically from each `test_id`. + +### Quick Start + +Run this on a SLURM login node: + +```bash +cd ${YOUR_TRTLLM_PATH}/jenkins/scripts/perf/local + +# 1. Copy the example and edit it for your cluster + test case +cp configs/example.conf configs/mycluster.conf +$EDITOR configs/mycluster.conf + +# 2. Submit (one sbatch per entry in test_ids) +bash run_disagg.sh -c configs/mycluster.conf + +# 3. Watch — logs and outputs are under $work_dir as printed by the script +squeue -u $USER +``` + +`bash run_disagg.sh -h` prints the inline header plus a list of available `.conf` +files under `configs/`. + +### Config File Reference + +The `.conf` file is `source`d as bash. Variables not set in the file fall back to +defaults inside `run_disagg.sh`. Anything `export`ed in the shell before invoking the +script takes precedence over both. + +All examples below use placeholders like `${YOUR_TRTLLM_PATH}` — replace with your +own absolute paths. + +#### Paths + +| Variable | Required | Description | +|----------|----------|-------------| +| `trtllm` | recommended | Login-node path to your TensorRT-LLM checkout. The container must be able to see it via `mounts`. Example: `${YOUR_TRTLLM_PATH}`. | +| `work_dir` | no | One run goes in one work dir; created if missing. Holds `slurm_launch.sh`, `test_list.txt`, `slurm-.out`, `report.xml`, and per-role logs. Default: `$HOME/perf_runs/disagg_`. Keeping it under `$trtllm` lets a single mount of the source tree cover it. | +| `llm_models_path` | yes | Host path to the model weights tree. Must be reachable inside the container via `mounts`. Example: `${YOUR_MODELS_PATH}/llm-models`. | +| `mounts` | recommended | Comma-separated `host:container` bind-mount pairs. Every path the container touches at runtime — source tree, wheel, work_dir, models — must be reachable through one of these. Recommended: `"$trtllm:$trtllm,$llm_models_path:$llm_models_path"`. | + +#### SLURM + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `partition` | **yes** | — | SLURM partition. Script errors if left as the `CHANGE_ME` placeholder. | +| `account` | no | `coreai_comparch_trtllm` | SLURM billing account. | +| `job_name` | no | `disagg_test` | Base job name. For multi-test runs, each sub-job is suffixed with `_`. | +| `time_limit` | no | `02:00:00` | `HH:MM:SS` SLURM wall-time. | + +#### Docker image (pick ONE of two modes) + +| Mode | `image` | `image_var` | Behavior | +|------|---------|-------------|----------| +| (a) — pinned | non-empty | ignored | `image` used verbatim. Reproducible, recommended for benchmarking. | +| (b) — auto-resolve | empty / unset | used as key name | Resolved from `${YOUR_TRTLLM_PATH}/jenkins/current_image_tags.properties` by `image_var=` key. | + +- `image` — Full image URI (mode a). URIs beginning with `urm.nvidia.com/` are + auto-rewritten to enroot form (`urm.nvidia.com#`); the script is a no-op if you + already passed the enroot form. +- `image_var` — Key in `current_image_tags.properties` (mode b). Common keys: + `LLM_DOCKER_IMAGE` (x86 — H100 / B200), `LLM_SBSA_DOCKER_IMAGE` (SBSA — GB200). + Default: `LLM_DOCKER_IMAGE`. + +Do **not** expect setting both `image` and `image_var` to "combine" — `image` always +wins when set. + +#### Test selection + +- `test_ids` (preferred) — Bash array of full pytest IDs. Each entry is submitted as + its own `sbatch`. With >1 entry, each lands in its own subdir under `$work_dir` + named `_`. +- `test_id` (legacy) — Single test string. Equivalent to a 1-element `test_ids`. + +Test-ID format: + +``` +perf/test_perf_sanity.py::test_e2e[--[-]] +``` + +- `` = `disagg` | `aggr` +- `` (disagg) = `e2e` | `gen_only` | `ctx_only` +- `` matches a YAML file in `tests/scripts/perf-sanity/disaggregated/` + (or `aggregated/`) +- `` — only for normal aggregated tests — the `name:` field of one of + the YAML's `server_configs` entries + +`run_disagg.sh` errors out if any entry still contains the literal placeholder +`CHANGE_ME`. + +#### Install mode + +- `install_mode` — `wheel` (default) or `source`. + - `wheel`: container does `pip install `. + - `source`: container does `pip install -e .` against the mounted `$trtllm`. + `wheel_path` is ignored in this mode. +- `wheel_path` — Local `.whl` file. **Required when `install_mode=wheel`** (must + exist; script errors otherwise). Must follow PEP 427 naming + (`tensorrt_llm----.whl`); pip rejects malformed names. Find + the exact name via: + ```bash + ls ${YOUR_TRTLLM_PATH}/build/tensorrt_llm-*.whl + ``` + Keeping it under `$trtllm/build` lets the single `$trtllm` mount cover it. + +#### Optional flags (leave unset = disabled) + +- `build_wheel_flag` — Set to `"--build-wheel"` to build the wheel inside the + container before installing. Works with either `install_mode`. +- `capture_nsys_flag` — Set to `"--capture-nsys"` to wrap the worker pytest in + `nsys profile`. Profiles land in `$work_dir/nsys.*..qdrep` (per-role for + disagg, per-rank for aggregated). + +#### Cluster compatibility + +- `strip_sbatch_opts` — Comma-separated `#SBATCH` directives to comment out in the + generated `slurm_launch.sh`, for clusters whose SLURM version / configuration + doesn't accept them. Default in `run_disagg.sh`: `--segment`. Why these may need + stripping: + - `--segment` — newer SLURM topology option, missing on older clusters (e.g., + EOS). + - `--gres` — EOS doesn't register GPUs as a `gres` at all. + - `--gpus-per-node` — SLURM translates this into a `gres` request internally, + which also fails on EOS for the same reason. + + Recommended: + | Cluster | `strip_sbatch_opts` | + |---------|---------------------| + | EOS | `"--segment,--gres,--gpus-per-node"` | + | B200 / GB200 modern clusters | `"--segment"` (or `""` if even `--segment` works) | + +### Multi-Test Work Directory Layout + +For a `test_ids` array with more than one entry: + +``` +$work_dir/ +├── 00_/ +│ ├── slurm_launch.sh +│ ├── slurm-.out +│ ├── install.log +│ ├── (disagg only) gen_server_*.log, ctx_server_*.log, disagg_server.log +│ ├── test_list.txt +│ └── report.xml +├── 01_/ +│ └── ... +└── ... +``` + +Each sub-job is submitted independently — one bad `test_id` does not stop the rest. +The script exits non-zero if any `submit.py` or `sbatch` invocation failed. + ## Shared Utilities ### `perf_utils.py` diff --git a/jenkins/scripts/perf/local/configs/example.conf b/jenkins/scripts/perf/local/configs/example.conf new file mode 100644 index 000000000000..c11e768d02ce --- /dev/null +++ b/jenkins/scripts/perf/local/configs/example.conf @@ -0,0 +1,101 @@ +# Example config for run_disagg.sh — copy and tweak per cluster / test case. +# +# Loaded by run_disagg.sh via `source`. Plain bash key=value works. Any variable +# not set here falls back to the default defined inside run_disagg.sh. Anything +# explicitly exported in the shell before invoking the script takes precedence +# over this file. + +# Path to your TensorRT-LLM checkout (login-node view) +trtllm="${YOUR_TENSORRT_LLM_PATH:-/path/to/tensorrt-llm}" + +# Work directory (one per run; will be created). +# Kept under $trtllm so a single mount of the source tree covers it. +work_dir="$trtllm/perf_runs/disagg_$(date +%Y%m%d_%H%M%S)" + +# SLURM +partition="${YOUR_SLURM_PARTITION:-your_partition}" +account="${YOUR_SLURM_ACCOUNT:-your_account}" +job_name="disagg_test" +time_limit="04:00:00" + +# Docker image — pick ONE of two modes. The script supports both; the rule is: +# +# ┌──────────────────────┬──────────────────────┬────────────────────────────┐ +# │ image │ image_var │ Result │ +# ├──────────────────────┼──────────────────────┼────────────────────────────┤ +# │ non-empty (mode a) │ ANY (ignored) │ image used verbatim │ +# │ empty / unset (mode b)│ used as a key name │ resolve from properties │ +# └──────────────────────┴──────────────────────┴────────────────────────────┘ +# +# Mode (a) — pin a specific image URL. Reproducible, recommended: +image="urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-x86_64-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202605060827-13616" +# +# Mode (b) — auto-resolve the "current" image from +# jenkins/current_image_tags.properties (CI-updated file) by KEY NAME. To enable +# this, comment out the 'image=' line above (or set it to "") so it's empty. +# LLM_DOCKER_IMAGE — x86 (H100/B200) ← default of image_var +# LLM_SBSA_DOCKER_IMAGE — SBSA (GB200) ← only override for SBSA +# image_var="LLM_DOCKER_IMAGE" +# +# NOTE: do NOT set both 'image' and 'image_var' expecting them to combine — +# image always wins when set. The current_image_tags.properties file is only +# consulted in mode (b). + +# Path to model weights (must be reachable inside the container via 'mounts') +llm_models_path="${YOUR_LLM_MODELS_PATH:-/path/to/llm_models}" + +# Container bind-mounts (comma-separated host:container pairs). +# IMPORTANT: every path the container needs at runtime — including $trtllm +# (source + run scripts + wheel), $work_dir (cwd inside container), and +# $llm_models_path — must be reachable through one of these mounts. With +# $work_dir kept under $trtllm and $wheel_path under $trtllm/build, mounting +# $trtllm alone is enough for the test code paths, plus a second mount for +# the model weights. +mounts="$trtllm:$trtllm,$llm_models_path:$llm_models_path" + +# Test ID(s). Format: +# perf/test_perf_sanity.py::test_e2e[disagg--] +# The matches a file in tests/scripts/perf-sanity/disaggregated/. +# is e2e | gen_only | ctx_only. +# +# Two ways to declare tests — use ONE of these: +# +# (a) Single test (legacy): +# test_id="perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]" +# +# (b) Multiple tests — bash array. Each entry is submitted as its own SLURM job +# (parallel queueing). When >1 entry, each gets its own subdir under $work_dir +# (named '_'), so logs and outputs don't collide. +test_ids=( + # "perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]" + # Add more entries here, e.g.: + "perf/test_perf_sanity.py::test_e2e[disagg-e2e-h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT]" + "perf/test_perf_sanity.py::test_e2e[disagg-ctx_only-h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT]" +) + +# Install mode: 'wheel' (pip install ) or 'source' (pip install -e .) +install_mode="wheel" + +# Local wheel file (required when install_mode=wheel). +# Kept under $trtllm/build so the single $trtllm mount above covers it. +# +# The filename MUST follow PEP 427 wheel naming convention: +# {name}-{version}(+{local})-{python}-{abi}-{platform}.whl +# pip will reject the install if any field is malformed. Find the exact name +# with: ls $trtllm/build/tensorrt_llm-*.whl +wheel_path="$trtllm/build/tensorrt_llm-1.3.0rc16+ngcpytorch2602-cp312-cp312-linux_x86_64.whl" + +# Optional flags — uncomment to enable +# build_wheel_flag="--build-wheel" +# capture_nsys_flag="--capture-nsys" + +# Comma-separated list of #SBATCH options to strip from the generated +# slurm_launch.sh (because the local SLURM version / cluster doesn't accept them). +# Why these defaults: +# --segment : newer SLURM topology option, not in older versions. +# --gres : EOS doesn't register GPUs as gres at all. +# --gpus-per-node : SLURM translates this into a gres request internally, +# which also fails on EOS for the same reason. +# Set to "" to keep all; on clusters with a real GPU gres (B200/GB200 etc.), +# you typically want at most "--segment". +strip_sbatch_opts="--segment,--gres,--gpus-per-node" diff --git a/jenkins/scripts/perf/local/run_disagg.sh b/jenkins/scripts/perf/local/run_disagg.sh new file mode 100755 index 000000000000..74cdb191dcb1 --- /dev/null +++ b/jenkins/scripts/perf/local/run_disagg.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# Generate slurm_launch.sh for a local disaggregated perf-sanity run +# and submit it via sbatch. Run this on a SLURM login node. +# +# Usage: +# bash run_disagg.sh -c configs/.conf +# bash run_disagg.sh --config configs/h100.conf +# +# All user-tunable variables live in the config file. See configs/example.conf +# for the full list and documentation. Copy it, edit it, point -c at it. +# +# Watch the job with: squeue -u $USER ; logs are under $work_dir. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ---------------- Parse args ---------------- +config_file="" +while [[ $# -gt 0 ]]; do + case "$1" in + -c|--config) + config_file="$2" + shift 2 + ;; + -h|--help) + sed -n '2,12p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + echo + echo "Available configs:" + ls -1 "$script_dir/configs/"*.conf 2>/dev/null || echo " (none — copy configs/example.conf to get started)" + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + echo "Usage: $0 -c " >&2 + exit 1 + ;; + esac +done + +if [[ -z "$config_file" ]]; then + echo "ERROR: no config file given. Use -c ." >&2 + echo " See $script_dir/configs/example.conf for a template." >&2 + exit 1 +fi +if [[ ! -f "$config_file" ]]; then + echo "ERROR: config file not found: $config_file" >&2 + exit 1 +fi + +# ---------------- Load config ---------------- +# shellcheck disable=SC1090 +source "$config_file" + +# Apply defaults for anything still unset +: "${trtllm:=CHANGE_ME}" +: "${work_dir:=$trtllm/perf_runs/disagg_$(date +%Y%m%d_%H%M%S)}" +: "${partition:=CHANGE_ME}" +: "${account:=coreai_comparch_trtllm}" +: "${job_name:=disagg_test}" +: "${image:=}" +: "${image_var:=LLM_DOCKER_IMAGE}" +: "${mounts:=CHANGE_ME}" +: "${llm_models_path:=/path/to/models}" +: "${install_mode:=wheel}" +: "${wheel_path:=CHANGE_ME}" +: "${build_wheel_flag:=}" +: "${capture_nsys_flag:=}" +: "${time_limit:=02:00:00}" + +# Normalize test list: prefer 'test_ids' bash array if set, else fall back to +# legacy single 'test_id'. Either declares a non-empty list at this point. +if declare -p test_ids >/dev/null 2>&1 && [[ "$(declare -p test_ids)" == "declare -a"* ]]; then + : # test_ids already an array +elif [[ -n "${test_id:-}" ]]; then + test_ids=("$test_id") +else + test_ids=("perf/test_perf_sanity.py::test_e2e[disagg-e2e-CHANGE_ME]") +fi + +# ---------------- Sanity checks ---------------- +if [[ "$partition" == "CHANGE_ME" ]]; then + echo "ERROR: please set 'partition' in $config_file." >&2 + exit 1 +fi +if [[ ${#test_ids[@]} -eq 0 ]]; then + echo "ERROR: 'test_ids' is empty. Set 'test_id' or 'test_ids' in $config_file." >&2 + exit 1 +fi +for _tid in "${test_ids[@]}"; do + if [[ "$_tid" == *"CHANGE_ME"* ]]; then + echo "ERROR: please set 'test_id'/'test_ids' in $config_file (got placeholder: $_tid)." >&2 + exit 1 + fi +done +if [[ "$trtllm" == "CHANGE_ME" ]]; then + echo "ERROR: please set 'trtllm' (path to TensorRT-LLM source tree) in $config_file." >&2 + exit 1 +fi +if [[ "$mounts" == "CHANGE_ME" ]]; then + echo "ERROR: please set 'mounts' (cluster-specific enroot bind mounts) in $config_file." >&2 + exit 1 +fi +if [[ ! -d "$trtllm" ]]; then + echo "ERROR: trtllm directory not found: $trtllm" >&2 + exit 1 +fi +if [[ "$install_mode" == "wheel" ]]; then + if [[ "$wheel_path" == "CHANGE_ME" || -z "$wheel_path" ]]; then + echo "ERROR: install_mode=wheel but 'wheel_path' is not set." >&2 + echo " Either set wheel_path to a local .whl file, or use install_mode=source." >&2 + exit 1 + fi + if [[ ! -f "$wheel_path" ]]; then + echo "ERROR: wheel_path file not found: $wheel_path" >&2 + exit 1 + fi +fi + +mkdir -p "$work_dir" + +# ---------------- Resolve docker image ---------------- +# Two mutually-exclusive modes (image takes precedence if both set): +# (a) 'image' non-empty → used verbatim. 'image_var' is ignored. +# (b) 'image' empty/unset → look up 'image_var' as a key in +# $trtllm/jenkins/current_image_tags.properties (CI-updated file). +if [[ -n "$image" ]]; then + image_source="pinned in conf" +else + image_file="$trtllm/jenkins/current_image_tags.properties" + if [[ ! -f "$image_file" ]]; then + echo "ERROR: image properties file not found: $image_file" >&2 + exit 1 + fi + # NOTE: grep returning no match would exit 1 and set -o pipefail+set -e would + # silently kill the script before our error message — append '|| true' to absorb. + image=$(grep "^${image_var}=" "$image_file" | head -1 | awk -F"=" '{print $2}' || true) + if [[ -z "$image" ]]; then + echo "ERROR: failed to parse '${image_var}' from $image_file" >&2 + echo " (Either fix image_var to a key in that file, or set 'image' directly in the conf.)" >&2 + exit 1 + fi + image_source="resolved from \$image_var=${image_var} via current_image_tags.properties" +fi +# urm.nvidia.com/ → urm.nvidia.com# (enroot URI form). No-op if user already passed enroot form. +image=${image//urm.nvidia.com\//urm.nvidia.com#} + +echo "=== Configuration ===" +echo "config_file : $config_file" +echo "trtllm : $trtllm" +echo "work_dir : $work_dir" +echo "partition : $partition" +echo "account : $account" +echo "job_name : $job_name" +echo "image : $image" +echo "image_source : $image_source" +echo "mounts : $mounts" +echo "llm_models_path : $llm_models_path" +echo "install_mode : $install_mode" +echo "wheel_path : $wheel_path" +echo "time_limit : $time_limit" +echo "num_tests : ${#test_ids[@]}" +for _i in "${!test_ids[@]}"; do + printf " test[%02d] : %s\n" "$_i" "${test_ids[$_i]}" +done +echo "======================" + +# Build install-mode / wheel-path args for submit.py +install_args=(--install-mode "$install_mode") +if [[ "$install_mode" == "wheel" ]]; then + install_args+=(--wheel-path "$wheel_path") +fi + +# Default strip list — see note inside the loop. +: "${strip_sbatch_opts:=--segment}" + +# Per-test loop: each test gets its own subdir (when >1), its own slurm_launch.sh, +# and its own sbatch submission. Failures are collected, not fatal, so a bad +# test_id doesn't stop the rest of the batch. +num_tests=${#test_ids[@]} +submitted_count=0 +failed_tests=() + +for idx in "${!test_ids[@]}"; do + tid="${test_ids[$idx]}" + + # When running a single test, keep the flat layout (back-compat); for + # multi-test, drop each into its own subdir named by a slug of the test id. + if [[ $num_tests -gt 1 ]]; then + slug="${tid#*[}" # strip everything up to and including '[' + slug="${slug%]*}" # strip trailing ']' and beyond + slug="${slug//[^a-zA-Z0-9_.-]/_}" + test_work_dir="$work_dir/$(printf '%02d_%s' "$idx" "$slug")" + test_job_name="${job_name}_${idx}" + else + test_work_dir="$work_dir" + test_job_name="$job_name" + fi + mkdir -p "$test_work_dir" + + echo + echo "=== [$((idx+1))/$num_tests] $tid ===" + echo " work_dir : $test_work_dir" + echo " job_name : $test_job_name" + + # 1. Generate slurm_launch.sh for this test. + # NOTE: we deliberately do NOT pass --draft-launch-sh — submit.py picks the + # right draft (disaggregated/ vs aggregated/) based on the runtime_mode it + # derives from the test id. Hard-coding the disagg draft here breaks + # 'aggr-ctx_only-*' tests, which run in aggregated mode but would otherwise + # be launched with the disagg orchestration (and produce no report.xml). + if ! python3 "$trtllm/jenkins/scripts/perf/local/submit.py" \ + --test-list "$tid" \ + --launch-sh "$test_work_dir/slurm_launch.sh" \ + --install-sh "$trtllm/jenkins/scripts/perf/local/slurm_install.sh" \ + --run-sh "$trtllm/jenkins/scripts/perf/local/slurm_run.sh" \ + --llm-src "$trtllm" \ + --work-dir "$test_work_dir" \ + --partition "$partition" \ + --account "$account" \ + --job-name "$test_job_name" \ + --image "$image" \ + --mounts "$mounts" \ + --llm-models-root "$llm_models_path" \ + --time "$time_limit" \ + "${install_args[@]}" \ + $build_wheel_flag \ + $capture_nsys_flag; then + echo "ERROR: submit.py failed for $tid — skipping." >&2 + failed_tests+=("$tid (submit.py failed)") + continue + fi + + # Strip SBATCH directives unsupported by this cluster's SLURM. + # Default '--segment': newer SLURM topology feature, missing on older clusters (e.g. EOS). + if [[ -n "$strip_sbatch_opts" ]]; then + IFS=',' read -ra _strip_arr <<< "$strip_sbatch_opts" + for opt in "${_strip_arr[@]}"; do + opt_trimmed="${opt#"${opt%%[![:space:]]*}"}" + opt_trimmed="${opt_trimmed%"${opt_trimmed##*[![:space:]]}"}" + [[ -z "$opt_trimmed" ]] && continue + sed -i.bak "s|^#SBATCH \(${opt_trimmed}\)|# (stripped) #SBATCH \1|" "$test_work_dir/slurm_launch.sh" + done + rm -f "$test_work_dir/slurm_launch.sh.bak" + fi + + # 2. Submit + if ( cd "$test_work_dir" && sbatch slurm_launch.sh ); then + submitted_count=$((submitted_count + 1)) + else + echo "ERROR: sbatch failed for $tid" >&2 + failed_tests+=("$tid (sbatch failed)") + fi +done + +echo +echo "=== Submission summary ===" +echo "Submitted: $submitted_count / $num_tests" +if [[ ${#failed_tests[@]} -gt 0 ]]; then + echo "Failed:" + for ft in "${failed_tests[@]}"; do + echo " - $ft" + done +fi +echo "Check status with: squeue -u \$USER" +echo "Logs and outputs under: $work_dir" +[[ ${#failed_tests[@]} -gt 0 ]] && exit 1 +exit 0 From 0c44d2dafe148454f22939762b4779e42bc0692b Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Tue, 26 May 2026 16:13:20 +0800 Subject: [PATCH 033/308] [None][refactor] Update model path definitions in test_perf.py and clean up waives.txt (#14393) Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tests/integration/defs/perf/_model_paths.py | 146 ++++++++++++ tests/integration/defs/perf/test_perf.py | 223 +----------------- .../integration/defs/perf/test_perf_sanity.py | 25 +- tests/integration/test_lists/waives.txt | 12 - 4 files changed, 152 insertions(+), 254 deletions(-) create mode 100644 tests/integration/defs/perf/_model_paths.py diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py new file mode 100644 index 000000000000..30d28fe15d9b --- /dev/null +++ b/tests/integration/defs/perf/_model_paths.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared model path constants for perf and perf-sanity tests.""" + +# Model PATH of local dir synced from internal LLM models repo +MODEL_PATH_DICT = { + "llama_v3.1_8b": "llama-3.1-model/Meta-Llama-3.1-8B", + "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", + "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", + "llama_v3.1_8b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", + "llama_v3.1_70b": "llama-3.1-model/Meta-Llama-3.1-70B", + "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", + "llama_v3.1_70b_instruct_fp8": "llama-3.1-model/Llama-3.1-70B-Instruct-FP8", + "llama_v3.3_70b_instruct_fp8": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8", + "llama_v3.3_70b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4", + "llama_v3.1_405b_instruct_fp8": "llama-3.1-model/Llama-3.1-405B-Instruct-FP8", + "llama_v3.1_405b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.1-405B-Instruct-fp4", + "llama_v3.1_70b_instruct": "llama-3.1-model/Meta-Llama-3.1-70B-Instruct", + "llama_v3.2_1b": "llama-3.2-models/Llama-3.2-1B", + "llama_v3.1_nemotron_nano_8b": "Llama-3.1-Nemotron-Nano-8B-v1", + "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", + "llama_v3.3_nemotron_super_49b": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1", + "llama_v3.3_nemotron_super_49b_fp8": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8", + "llama_v3.3_nemotron_super_49b_v1.5_fp8": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1_5-FP8", + "llama_v3.1_nemotron_ultra_253b": "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1", + "llama_v3.1_nemotron_ultra_253b_fp8": "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", + "llama_v4_scout_17b_16e_instruct": "llama4-models/Llama-4-Scout-17B-16E-Instruct", + "llama_v4_scout_17b_16e_instruct_fp8": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", + "llama_v4_scout_17b_16e_instruct_fp4": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", + "llama_v4_maverick_17b_128e_instruct": "llama4-models/Llama-4-Maverick-17B-128E-Instruct", + "llama_v4_maverick_17b_128e_instruct_fp8": "llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8", + "deepseek_r1_distill_qwen_32b": "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B", + "deepseek_r1_distill_llama_70b": "DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B/", + "gemma_3_27b_it": "gemma/gemma-3-27b-it", + "gemma_3_27b_it_fp8": "gemma/gemma-3-27b-it-fp8", + "gemma_3_27b_it_fp4": "gemma/gemma-3-27b-it-FP4", + "gemma_3_12b_it": "gemma/gemma-3-12b-it", + "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", + "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", + "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", + "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", + "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", + "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", + "deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/", + "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8", + "deepseek_v3_lite_nvfp4": "DeepSeek-V3-Lite/nvfp4_moe_only", + "qwen2_7b_instruct": "Qwen2-7B-Instruct", + "qwen_14b_chat": "Qwen-14B-Chat", + "qwen3_0.6b": "Qwen3/Qwen3-0.6B", + "qwen3_4b_eagle3": "Qwen3/Qwen3-4B", + "qwen3_8b": "Qwen3/Qwen3-8B", + "qwen3_8b_fp8": "Qwen3/nvidia-Qwen3-8B-FP8", + "qwen3_8b_fp4": "Qwen3/nvidia-Qwen3-8B-NVFP4", + "qwen3_14b": "Qwen3/Qwen3-14B", + "qwen3_14b_fp8": "Qwen3/nvidia-Qwen3-14B-FP8", + "qwen3_14b_fp4": "Qwen3/nvidia-Qwen3-14B-NVFP4", + "qwen3_30b_a3b": "Qwen3/Qwen3-30B-A3B", + "qwen3_30b_a3b_fp4": "Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf", + "qwen3_32b": "Qwen3/Qwen3-32B", + "qwen3_32b_fp4": "Qwen3/nvidia-Qwen3-32B-NVFP4", + "qwen3_235b_a22b_fp8": "Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf", + "qwen3_235b_a22b_fp4": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", + "qwen3_235b_a22b_fp4_eagle3": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", + "qwen2_5_vl_7b_instruct": "Qwen2.5-VL-7B-Instruct", + "qwen2_5_vl_7b_instruct_fp8": "multimodals/Qwen2.5-VL-7B-Instruct-FP8", + "qwen2_5_vl_7b_instruct_fp4": "multimodals/Qwen2.5-VL-7B-Instruct-FP4", + "starcoder2_3b": "starcoder2-3b", + "phi_4_mini_instruct": "Phi-4-mini-instruct", + "phi_4_reasoning_plus": "Phi-4-reasoning-plus", + "phi_4_reasoning_plus_fp8": "nvidia-Phi-4-reasoning-plus-FP8", + "phi_4_reasoning_plus_fp4": "nvidia-Phi-4-reasoning-plus-NVFP4", + "phi_4_multimodal_instruct": "multimodals/Phi-4-multimodal-instruct", + "phi_4_multimodal_instruct_fp4": "multimodals/Phi-4-multimodal-instruct-FP4", + "phi_4_multimodal_instruct_fp8": "multimodals/Phi-4-multimodal-instruct-FP8", + "mistral_small_v3.1_24b": "Mistral-Small-3.1-24B-Instruct-2503", + "gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b", + "gpt_oss_20b_fp4": "gpt_oss/gpt-oss-20b", + "gpt_oss_120b_eagle3": "gpt_oss/gpt-oss-120b", + "gpt_oss_120b_eagle3_throughput": "gpt_oss/gpt-oss-120b", + "nemotron_nano_3_30b_fp8": "Nemotron-Nano-3-30B-A3.5B-FP8-KVFP8-dev", + "nemotron_nano_12b_v2": "NVIDIA-Nemotron-Nano-12B-v2", + "nvidia_nemotron_nano_9b_v2_nvfp4": "NVIDIA-Nemotron-Nano-9B-v2-NVFP4", + "nemotron_3_super_120b_nvfp4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", + "nemotron_3_super_120b_nvfp4_mtp": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", + # Nemotron-3-Nano-Omni-30B (text + image multimodal) + "nemotron_3_nano_omni_nvfp4": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", + "nemotron_3_nano_omni_nvfp4_image": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", + "kimi_k2_nvfp4": "Kimi-K2-Thinking-NVFP4", + # MiniMax M2.5 (FP8 block-scale, ~230B MoE) + "minimax_m2.5_fp8": "MiniMax-M2.5", + # Qwen3.5 dense + MoE + "qwen3.5_9b": "Qwen3.5-9B", + "qwen3.5_27b": "Qwen3.5-27B", + "qwen3.5_35b_a3b_fp8": "Qwen3.5-35B-A3B-FP8", + "qwen3.5_122b_a10b": "Qwen3.5-122B-A10B", + "qwen3.5_397b_a17b_fp8": "Qwen3.5-397B-A17B-FP8", + "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", + # DeepSeek V3.2 (671B MoE) + "deepseek_v3.2_fp8": "DeepSeek-V3.2-hf", + "deepseek_v3.2_fp4": "DeepSeek-V3.2-NVFP4", + # GLM-5 FP8 (MoE) + "glm_5_fp8": "GLM-5-FP8", + # Kimi K2.5 NVFP4 (~1T MoE multimodal) + "kimi_k2.5_fp4": "Kimi-K2.5-NVFP4", + # Keys below are sanity-side aliases; some point to the same weights as + # entries above but are kept under sanity's historical naming. + "deepseek_v32_fp4": "DeepSeek-V3.2-Exp-FP4-v2", + "k2_thinking_fp4": "Kimi-K2-Thinking-NVFP4", + "k25_thinking_fp4": "Kimi-K2.5-NVFP4", + "super_nvfp4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", + "super_fp8": "NVIDIA-Nemotron-3-Super-120B-A12B-FP8", + "super_bf16": "NVIDIA-Nemotron-3-Super-120B-A12B-BF16", + "qwen3_32b_fp8": "Qwen3/Qwen3-32B-FP8", + "glm_5_nvfp4": "GLM-5-NVFP4", +} + +# Model PATH of HuggingFace +HF_MODEL_PATH = { + "llama_v3.1_8b_hf": "meta-llama/Llama-3.1-8B", + "llama_v3.1_8b_instruct_hf": "nvidia/Llama-3.1-8B-Instruct-FP8", + "llama_v3.1_70b_instruct_hf": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "llama_v3.1_70b_hf": "meta-llama/Llama-3.1-70B", + "llama_v3.1_405b_hf": "meta-llama/Llama-3.1-405B", + "llama_v3.1_nemotron_nano_8b_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1", + "llama_v3.1_nemotron_nano_8b_fp8_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1-FP8", + "llama_v3.3_nemotron_super_49b_hf": "nvidia/Llama-3_3-Nemotron-Super-49B-v1", + "llama_v3.3_nemotron_super_49b_fp8_hf": "nvidia/Llama-3_3-Nemotron-Super-49B-v1-FP8", + "llama_v3.1_nemotron_ultra_253b_fp8_hf": "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", + "phi_4_mini_instruct_hf": "microsoft/Phi-4-mini-instruct", +} + +LORA_MODEL_PATH = { + "llama_v3.1_8b_instruct_fp8": "lora/llama-3-chinese-8b-instruct-v2-lora/", +} diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 33e3be93cb1d..df1b01128efb 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,6 +31,7 @@ from ..conftest import (get_device_count, get_llm_root, llm_models_root, trt_environment) +from ._model_paths import HF_MODEL_PATH, LORA_MODEL_PATH, MODEL_PATH_DICT from .pytorch_model_config import get_model_yaml_config from .sampler_options_config import get_sampler_options_config from .utils import (AbstractPerfScriptTestClass, PerfBenchScriptTestCmds, @@ -43,226 +44,6 @@ ALLOWED_CONFIGS_CACHE = None # Cache to avoid modifying sys.path many times. MAP_BY_SOCKET = None -# Model PATH of local dir synced from internal LLM models repo -MODEL_PATH_DICT = { - "llama_v2_7b": "llama-models-v2/llama-v2-7b-hf", # not safetensors repo - "llama_v2_13b": "llama-models-v2/llama-v2-13b-hf", # not safetensors repo - "llama_v2_70b": "llama-models-v2/llama-v2-70b-hf", # not safetensors repo - "llama_v3.1_8b": "llama-3.1-model/Meta-Llama-3.1-8B", - "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", - "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", - "llama_v3.1_8b_instruct_fp4": - "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", - "llama_v3.1_70b": "llama-3.1-model/Meta-Llama-3.1-70B", - "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", - "llama_v3.1_70b_instruct_fp8": "llama-3.1-model/Llama-3.1-70B-Instruct-FP8", - "llama_v3.3_70b_instruct_fp8": - "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8", - "llama_v3.3_70b_instruct_fp4": - "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4", - "llama_v3.1_405b_instruct_fp8": - "llama-3.1-model/Llama-3.1-405B-Instruct-FP8", - "llama_v3.1_405b_instruct_fp4": - "modelopt-hf-model-hub/Llama-3.1-405B-Instruct-fp4", - "llama_v3.1_70b_instruct": "llama-3.1-model/Meta-Llama-3.1-70B-Instruct", - "llama_v3.2_1b": "llama-3.2-models/Llama-3.2-1B", - "llama_v3.1_nemotron_nano_8b": "Llama-3.1-Nemotron-Nano-8B-v1", - "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", - "llama_v3.3_nemotron_super_49b": - "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1", - "llama_v3.3_nemotron_super_49b_fp8": - "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8", - "llama_v3.3_nemotron_super_49b_v1.5_fp8": - "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1_5-FP8", - "llama_v3.1_nemotron_ultra_253b": - "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1", - "llama_v3.1_nemotron_ultra_253b_fp8": - "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", - "llama_v4_scout_17b_16e_instruct": - "llama4-models/Llama-4-Scout-17B-16E-Instruct", - "llama_v4_scout_17b_16e_instruct_fp8": - "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", - "llama_v4_scout_17b_16e_instruct_fp4": - "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", - "llama_v4_maverick_17b_128e_instruct": - "llama4-models/Llama-4-Maverick-17B-128E-Instruct", - "llama_v4_maverick_17b_128e_instruct_fp8": - "llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8", - "mixtral_8x7b_v0.1": "Mixtral-8x7B-v0.1", - "mixtral_8x7b_v0.1_instruct": "Mixtral-8x7B-Instruct-v0.1", - "mixtral_8x7b_v0.1_instruct_fp8": "Mixtral-8x7B-Instruct-v0.1-fp8", - "mixtral_8x7b_v0.1_instruct_fp4": - "modelopt-hf-model-hub/Mixtral-8x7B-Instruct-v0.1-fp4", - "mistral_nemo_12b_base": "Mistral-Nemo-Base-2407", - "deepseek_r1_distill_qwen_32b": "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B", - "deepseek_r1_distill_llama_70b": - "DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B/", - "mixtral_8x22b_v0.1": "Mixtral-8x22B-v0.1", - "mistral_7b_v0.1": "mistral-7b-v0.1", - "ministral_8b": "Ministral-8B-Instruct-2410", - "ministral_8b_fp8": "Ministral-8B-Instruct-2410-FP8", - "gemma_3_1b_it": "gemma/gemma-3-1b-it", - "gemma_3_27b_it": "gemma/gemma-3-27b-it", - "gemma_3_27b_it_fp8": "gemma/gemma-3-27b-it-fp8", - "gemma_3_27b_it_fp4": "gemma/gemma-3-27b-it-FP4", - "gemma_3_12b_it": "gemma/gemma-3-12b-it", - "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", - "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", - "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", - "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", - "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", - "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", - "deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/", - "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8", - "deepseek_v3_lite_nvfp4": "DeepSeek-V3-Lite/nvfp4_moe_only", - "qwen2_7b_instruct": "Qwen2-7B-Instruct", - "qwen_14b_chat": "Qwen-14B-Chat", - "qwen3_0.6b": "Qwen3/Qwen3-0.6B", - "qwen3_4b_eagle3": "Qwen3/Qwen3-4B", - "qwen3_8b": "Qwen3/Qwen3-8B", - "qwen3_8b_fp8": "Qwen3/nvidia-Qwen3-8B-FP8", - "qwen3_8b_fp4": "Qwen3/nvidia-Qwen3-8B-NVFP4", - "qwen3_14b": "Qwen3/Qwen3-14B", - "qwen3_14b_fp8": "Qwen3/nvidia-Qwen3-14B-FP8", - "qwen3_14b_fp4": "Qwen3/nvidia-Qwen3-14B-NVFP4", - "qwen3_30b_a3b": "Qwen3/Qwen3-30B-A3B", - "qwen3_30b_a3b_fp4": "Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf", - "qwen3_32b": "Qwen3/Qwen3-32B", - "qwen3_32b_fp4": "Qwen3/nvidia-Qwen3-32B-NVFP4", - "qwen3_235b_a22b_fp8": "Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf", - "qwen3_235b_a22b_fp4": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", - "qwen3_235b_a22b_fp4_eagle3": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", - "qwen2_5_vl_7b_instruct": "Qwen2.5-VL-7B-Instruct", - "qwen2_5_vl_7b_instruct_fp8": "multimodals/Qwen2.5-VL-7B-Instruct-FP8", - "qwen2_5_vl_7b_instruct_fp4": "multimodals/Qwen2.5-VL-7B-Instruct-FP4", - "starcoder2_3b": "starcoder2-3b", - "starcoder2_7b": "starcoder2-7b", - "starcoder2_15b": "starcoder2-15b", - "t5": "t5-small", # not supported for trtllm-bench build config - "flan_t5_base": - "flan-t5-small", # not supported for trtllm-bench build config - "flan_t5_large": - "flan-t5-xl", # not supported for trtllm-bench build config - "whisper_large_v3": - "whisper-models/large-v3", # not supported for trtllm-bench tokenizer - "bart_large_cnn": "bart-large-cnn", # not safetensors repo - "mbart_large_50_many_to_one_mmt": "mbart-large-50-many-to-one-mmt", - "mamba_130m": "mamba/mamba-130m-hf", - "mamba_370m": "mamba/mamba-370m-hf", - "mamba_2.8b": "mamba/mamba-2.8b-hf", - "gpt_20b": "gpt-neox-20b", - "gpt_350m_moe": "gpt2-medium", - "phi_4_mini_instruct": "Phi-4-mini-instruct", - "phi_4_reasoning_plus": "Phi-4-reasoning-plus", - "phi_4_reasoning_plus_fp8": "nvidia-Phi-4-reasoning-plus-FP8", - "phi_4_reasoning_plus_fp4": "nvidia-Phi-4-reasoning-plus-NVFP4", - "phi_4_multimodal_instruct": "multimodals/Phi-4-multimodal-instruct", - "phi_4_multimodal_instruct_image": "multimodals/Phi-4-multimodal-instruct", - "phi_4_multimodal_instruct_audio": "multimodals/Phi-4-multimodal-instruct", - "phi_4_multimodal_instruct_fp4": - "multimodals/Phi-4-multimodal-instruct-FP4", - "phi_4_multimodal_instruct_fp4_image": - "multimodals/Phi-4-multimodal-instruct-FP4", - "phi_4_multimodal_instruct_fp4_audio": - "multimodals/Phi-4-multimodal-instruct-FP4", - "phi_4_multimodal_instruct_fp8_image": - "multimodals/Phi-4-multimodal-instruct-FP8", - "phi_4_multimodal_instruct_fp8_audio": - "multimodals/Phi-4-multimodal-instruct-FP8", - "phi_4_multimodal_instruct_fp8": - "multimodals/Phi-4-multimodal-instruct-FP8", - "bielik_11b_v2.2_instruct": "Bielik-11B-v2.2-Instruct", - "bielik_11b_v2.2_instruct_fp8": "Bielik-11B-v2.2-Instruct-FP8", - "mistral_small_v3.1_24b": "Mistral-Small-3.1-24B-Instruct-2503", - "gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b", - "gpt_oss_20b_fp4": "gpt_oss/gpt-oss-20b", - "gpt_oss_120b_eagle3": "gpt_oss/gpt-oss-120b", - "gpt_oss_120b_eagle3_throughput": "gpt_oss/gpt-oss-120b", - "nemotron_nano_3_30b_fp8": "Nemotron-Nano-3-30B-A3.5B-FP8-KVFP8-dev", - "nemotron_nano_12b_v2": "NVIDIA-Nemotron-Nano-12B-v2", - "nvidia_nemotron_nano_9b_v2_nvfp4": "NVIDIA-Nemotron-Nano-9B-v2-NVFP4", - "nemotron_3_super_120b_nvfp4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "nemotron_3_super_120b_nvfp4_mtp": - "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - # Nemotron-3-Nano-Omni-30B (text + image multimodal) - "nemotron_3_nano_omni_nvfp4": - "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "nemotron_3_nano_omni_nvfp4_image": - "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "kimi_k2_nvfp4": "Kimi-K2-Thinking-NVFP4", - # MiniMax M2.5 (FP8 block-scale, ~230B MoE) - "minimax_m2.5_fp8": "MiniMax-M2.5", - # Qwen3.5 dense + MoE - "qwen3.5_9b": "Qwen3.5-9B", - "qwen3.5_27b": "Qwen3.5-27B", - "qwen3.5_35b_a3b_fp8": "Qwen3.5-35B-A3B-FP8", - "qwen3.5_122b_a10b": "Qwen3.5-122B-A10B", - "qwen3.5_397b_a17b_fp8": "Qwen3.5-397B-A17B-FP8", - "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", - # DeepSeek V3.2 (671B MoE) - "deepseek_v3.2_fp8": "DeepSeek-V3.2-hf", - "deepseek_v3.2_fp4": "DeepSeek-V3.2-NVFP4", - "deepseek_v3.2_exp_fp4_v2": "DeepSeek-V3.2-Exp-FP4-v2", - # GLM-5 FP8 (MoE) - "glm_5_fp8": "GLM-5-FP8", - # Kimi K2.5 NVFP4 (~1T MoE multimodal) - "kimi_k2.5_fp4": "Kimi-K2.5-NVFP4", -} -# Model PATH of HuggingFace -HF_MODEL_PATH = { - "llama_v2_7b_hf": "meta-llama/Llama-2-7b-hf", - "llama_v2_70b_hf": "meta-llama/Llama-2-70b-hf", - "falcon_180b_hf": "tiiuae/falcon-180B", - "gptj_6b_hf": "EleutherAI/gpt-j-6b", - "llama_v3_8b_hf": "meta-llama/Meta-Llama-3-8B", - "llama_v3.1_8b_hf": "meta-llama/Llama-3.1-8B", - "llama_v3.1_8b_instruct_hf": "nvidia/Llama-3.1-8B-Instruct-FP8", - "llama_v3.1_70b_instruct_hf": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "llama_v3_70b_hf": "meta-llama/Meta-Llama-3-70B", - "llama_v3.1_70b_hf": "meta-llama/Llama-3.1-70B", - "llama_v3.1_405b_hf": "meta-llama/Llama-3.1-405B", - "llama_v3.1_nemotron_nano_8b_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1", - "llama_v3.1_nemotron_nano_8b_fp8_hf": - "nvidia/Llama-3.1-Nemotron-Nano-8B-v1-FP8", - "llama_v3.3_nemotron_super_49b_hf": - "nvidia/Llama-3_3-Nemotron-Super-49B-v1", - "llama_v3.3_nemotron_super_49b_fp8_hf": - "nvidia/Llama-3_3-Nemotron-Super-49B-v1-FP8", - "llama_v3.1_nemotron_ultra_253b_fp8_hf": - "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", - "mixtral_8x7b_v0.1_hf": "mistralai/Mixtral-8x7B-v0.1", - "mixtral_8x7b_v0.1_instruct_hf": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "mistral_7b_v0.1_hf": "mistralai/Mistral-7B-v0.1", - "ministral_8b_hf": "mistralai/Ministral-8B-Instruct-2410", - "flan_t5_base_hf": "google/flan-t5-small", - "phi_4_mini_instruct_hf": "microsoft/Phi-4-mini-instruct", - "gemma_3_1b_it_hf": "google/gemma-3-1b-it", -} -LORA_MODEL_PATH = { - "llama_v2_13b": - "llama-models-v2/chinese-llama-2-lora-13b", - "mixtral_8x7b_v0.1": - "chinese-mixtral-lora", - "llama_v3.1_8b_instruct_fp8": - "lora/llama-3-chinese-8b-instruct-v2-lora/", - "ministral_8b": - "lora/ministral/Ministral-8B-Instruct-2410-Loras-Dummy", # Dummy LoRA for Ministral - "gemma_3_1b_it": - "lora/gemma/gemma-3-1b-it-dummy-lora", # Dummy LoRA for Gemma-3-1B-Instruct - "phi_4_multimodal_instruct_image": - "multimodals/Phi-4-multimodal-instruct/vision-lora", - "phi_4_multimodal_instruct_audio": - "multimodals/Phi-4-multimodal-instruct/speech-lora", - "phi_4_multimodal_instruct_fp4_image": - "multimodals/Phi-4-multimodal-instruct-FP4/vision-lora", - "phi_4_multimodal_instruct_fp4_audio": - "multimodals/Phi-4-multimodal-instruct-FP4/speech-lora", - "phi_4_multimodal_instruct_fp8_image": - "multimodals/Phi-4-multimodal-instruct-FP8/vision-lora", - "phi_4_multimodal_instruct_fp8_audio": - "multimodals/Phi-4-multimodal-instruct-FP8/speech-lora", -} - TIMING_CACHE_DIR = os.environ.get("TIMING_CACHE_DIR", "") NEMOTRON_SUPER_MODELS = { diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 4d9882259f46..40deafd9b15f 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,30 +34,13 @@ from tensorrt_llm._utils import get_free_port from ..conftest import get_llm_root, llm_models_root +from ._model_paths import MODEL_PATH_DICT as _MODEL_PATH_DICT_BASE from .perf_regression_utils import process_and_upload_test_results -# Model PATH of local dir synced from internal LLM models repo +# Sanity-side path differs from test_perf for this key; preserve historical value. MODEL_PATH_DICT = { - "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", - "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", - "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", - "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", - "deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/", - "deepseek_v32_fp4": "DeepSeek-V3.2-Exp-FP4-v2", - "gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b", - "k2_thinking_fp4": "Kimi-K2-Thinking-NVFP4", - "k25_thinking_fp4": "Kimi-K2.5-NVFP4", - "qwen3_235b_a22b_fp4": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", # Qwen3-235B-A22B-FP4 - "super_nvfp4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", # Super (Nemotron-H SSM+MoE) NvFP4 - "super_fp8": "NVIDIA-Nemotron-3-Super-120B-A12B-FP8", - "super_bf16": "NVIDIA-Nemotron-3-Super-120B-A12B-BF16", - "qwen3_235b_a22b_fp8": "Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf", # Qwen3-235B-A22B-FP8 - "qwen3_32b_fp8": "Qwen3/Qwen3-32B-FP8", + **_MODEL_PATH_DICT_BASE, "llama_v3.3_70b_instruct_fp4": "llama-3.3-models/Llama-3.3-70B-Instruct-FP4", - "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8", - "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", - "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", - "glm_5_nvfp4": "GLM-5-NVFP4", } SUPPORTED_GPU_MAPPING = { diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 541a1e5a0a57..a53606807422 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -249,21 +249,9 @@ full:A10/unittest/kv_cache_manager_v2_tests/ SKIP (https://nvbugs/5841954) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6185480) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] SKIP (https://nvbugs/6185480) full:B200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) -full:B200/perf/test_perf.py::test_perf[bart_large_cnn] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[bert_large] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[flan_t5_base] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[flan_t5_large] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[flan_t5_xl] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[flan_t5_xxl] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[mbart_large_50_many_to_one_mmt] SKIP (bert_attention_plugin does not support SM >= 100) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) -full:B200/perf/test_perf.py::test_perf[roberta_base] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[t5_11b] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[t5_3b] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[t5_base] SKIP (bert_attention_plugin does not support SM >= 100) -full:B200/perf/test_perf.py::test_perf[t5_large] SKIP (bert_attention_plugin does not support SM >= 100) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:DGX_H100/kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked] SKIP (https://nvbugs/6136737) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) From a9c35f3c47b3324753c8ffce2ac541652aa28b02 Mon Sep 17 00:00:00 2001 From: YihuiLu512 <269394165+YihuiLu512@users.noreply.github.com> Date: Tue, 26 May 2026 17:06:53 +0800 Subject: [PATCH 034/308] [TRTLLM-12968][ci] Dedup executor unit tests on H100/B200 (#14556) Signed-off-by: Yihui Lu <269394165+YihuiLu512@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_b200.yml | 1 - tests/integration/test_lists/test-db/l0_h100.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index def40f798183..bf15e99b4db5 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -90,7 +90,6 @@ l0_b200: - unittest/_torch/attention - unittest/_torch/compilation - unittest/_torch/debugger - - unittest/_torch/executor # ------------- modules (non-MoE) --------------- - unittest/_torch/modules/test_mla_helix.py - unittest/_torch/modules/test_fused_add_rms_norm_quant.py diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index e55c5ae8363e..c09068b8a6d3 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -207,7 +207,7 @@ l0_h100: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0] - - unittest/_torch/executor + - unittest/_torch/executor/test_overlap_scheduler.py - unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py - unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py - unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m "part0" From 1f8312d5bfd42d445ee943ff741f972e7b1fd057 Mon Sep 17 00:00:00 2001 From: Yiyun Lu <55233584+luyiyun1021@users.noreply.github.com> Date: Tue, 26 May 2026 18:01:11 +0800 Subject: [PATCH 035/308] [TRTLLM-12949][refactor] visual_gen: unify fused QK-norm+rope dispatch (#14529) Signed-off-by: Yiyun Lu <55233584+luyiyun1021@users.noreply.github.com> --- ...bench_fused_dit_cross_head_qk_norm_rope.py | 218 -------------- .../kernels/fusedDiTQKNormRopeKernel.cu | 254 +---------------- .../kernels/fusedDiTQKNormRopeKernel.h | 28 +- .../kernels/fusedDiTSplitNormKernel.cu | 9 +- .../kernels/fusedDiTSplitQKNormRopeKernel.cu | 10 +- .../thop/fusedDiTQKNormRopeOp.cpp | 105 ++----- .../thop/fusedDiTSplitQKNormRopeOp.cpp | 39 ++- tensorrt_llm/_torch/compilation/utils.py | 3 - .../models/ltx2/transformer_ltx2.py | 17 +- .../visual_gen/models/wan/transformer_wan.py | 9 +- .../_torch/visual_gen/modules/attention.py | 122 ++------ .../defs/examples/test_visual_gen.py | 13 - .../test_fused_dit_qk_norm_rope.py | 267 ------------------ 13 files changed, 124 insertions(+), 970 deletions(-) delete mode 100644 benchmarks/bench_fused_dit_cross_head_qk_norm_rope.py diff --git a/benchmarks/bench_fused_dit_cross_head_qk_norm_rope.py b/benchmarks/bench_fused_dit_cross_head_qk_norm_rope.py deleted file mode 100644 index 70ae2d1e7af2..000000000000 --- a/benchmarks/bench_fused_dit_cross_head_qk_norm_rope.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Micro-benchmark: fused cross-head QK Norm + RoPE vs split (separate norm + RoPE). - -Three baselines are compared: - 1. Eager: PyTorch eager-mode split norm + RoPE (many small CUDA kernels - launched via PyTorch dispatch). - 2. Compile: `torch.compile`'d version of (1), where TorchInductor fuses the - elementwise + norm ops into a few large Triton kernels. - 3. Fused: Our single hand-written CUDA kernel that does cross-head RMSNorm - + RoPE in a single pass. - -Usage: - python benchmarks/bench_fused_dit_cross_head_qk_norm_rope.py -""" - -import torch -import torch._dynamo -import torch.nn.functional as F - -import tensorrt_llm # noqa: F401 # required to register torch.ops.trtllm.* - -# We benchmark many shape/mode combinations with a single torch.compile()'d -# function; each unique input shape + static arg combo forces Dynamo to -# recompile. Raise the cache limit so we don't silently fall back to eager. -torch._dynamo.config.cache_size_limit = 64 - - -def _apply_interleaved_rope(x, cos, sin): - x_rot = torch.empty_like(x, dtype=torch.float32) - x_rot[:, 0::2] = -x[:, 1::2].float() - x_rot[:, 1::2] = x[:, 0::2].float() - return (x.float() * cos + x_rot * sin).to(x.dtype) - - -def _apply_rotate_half_rope(x, cos, sin, head_dim, num_heads): - """rotate_half RoPE: pair (x[i], x[i + head_dim/2]) within each head.""" - T = x.shape[0] - x_4d = x.view(T, num_heads, head_dim).float() - half = head_dim // 2 - x1 = x_4d[..., :half] - x2 = x_4d[..., half:] - cos_4d = cos.view(T, num_heads, head_dim) - sin_4d = sin.view(T, num_heads, head_dim) - out = torch.empty_like(x_4d) - out[..., :half] = x1 * cos_4d[..., :half] - x2 * sin_4d[..., :half] - out[..., half:] = x2 * cos_4d[..., half:] + x1 * sin_4d[..., half:] - return out.reshape(T, -1).to(x.dtype) - - -def split_norm_rope( - qkv, num_heads, head_dim, eps, q_weight, k_weight, cos_emb, sin_emb, interleave -): - """Baseline: separate RMSNorm + RoPE (current WAN path).""" - q_size = num_heads * head_dim - q = qkv[:, :q_size] - k = qkv[:, q_size : 2 * q_size] - - q = F.rms_norm(q.float(), (q_size,), q_weight.float(), eps).to(qkv.dtype) - k = F.rms_norm(k.float(), (q_size,), k_weight.float(), eps).to(qkv.dtype) - - num_tokens = qkv.shape[0] - cos_q = cos_emb.unsqueeze(1).expand(-1, num_heads, -1).reshape(num_tokens, q_size) - sin_q = sin_emb.unsqueeze(1).expand(-1, num_heads, -1).reshape(num_tokens, q_size) - - if interleave: - q = _apply_interleaved_rope(q, cos_q, sin_q) - k = _apply_interleaved_rope(k, cos_q, sin_q) - else: - q = _apply_rotate_half_rope(q, cos_q, sin_q, head_dim, num_heads) - k = _apply_rotate_half_rope(k, cos_q, sin_q, head_dim, num_heads) - - qkv[:, :q_size] = q - qkv[:, q_size : 2 * q_size] = k - - -def fused_norm_rope( - qkv, num_heads, head_dim, eps, q_weight, k_weight, cos_emb, sin_emb, interleave -): - """Fused cross-head QK Norm + RoPE kernel.""" - torch.ops.trtllm.fused_dit_cross_head_qk_norm_rope( - qkv, - num_heads, - num_heads, - num_heads, - head_dim, - eps, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, - ) - - -def benchmark(fn, warmup=50, iters=200): - """Time GPU-side kernel work only using CUDA events. - - fn() should perform the kernel call(s) on pre-allocated buffers; caller is - responsible for ensuring any in-place mutation does not affect timing - semantics (we do iters back-to-back calls on the same buffer). - """ - for _ in range(warmup): - fn() - torch.cuda.synchronize() - - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(iters): - fn() - end.record() - torch.cuda.synchronize() - return start.elapsed_time(end) / iters - - -def main(): - device = "cuda" - configs = [ - {"name": "WAN_1.3B_256tok", "num_heads": 12, "head_dim": 128, "num_tokens": 256}, - {"name": "WAN_1.3B_4096tok", "num_heads": 12, "head_dim": 128, "num_tokens": 4096}, - {"name": "WAN_14B_256tok", "num_heads": 40, "head_dim": 128, "num_tokens": 256}, - {"name": "WAN_14B_4096tok", "num_heads": 40, "head_dim": 128, "num_tokens": 4096}, - {"name": "WAN_14B_16384tok", "num_heads": 40, "head_dim": 128, "num_tokens": 16384}, - ] - - # TorchInductor Triton-fused version of the eager split path. - # Separate torch.compile() call per shape avoids graph-break costs being - # counted in-between shapes (Inductor caches compiled kernels by shape). - split_compiled = torch.compile(split_norm_rope, mode="reduce-overhead", dynamic=False) - - # interleave=True -> WAN-1 family (interleaved RoPE) - # interleave=False -> WAN-2 family (rotate_half RoPE) - for mode_name, interleave in [("interleave (WAN-1)", True), ("rotate_half (WAN-2)", False)]: - print(f"\n=== RoPE mode: {mode_name} ===") - header = ( - f"{'Config':<22} {'Eager (ms)':>12} {'Compile (ms)':>14} {'Fused (ms)':>12} " - f"{'Fused/Eager':>12} {'Fused/Compile':>14}" - ) - print(header) - print("-" * len(header)) - - for cfg in configs: - num_heads = cfg["num_heads"] - head_dim = cfg["head_dim"] - num_tokens = cfg["num_tokens"] - q_dim = num_heads * head_dim - hidden_size = 3 * q_dim - - torch.random.manual_seed(42) - qkv_seed = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) - q_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) - k_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) - - half_dim = head_dim // 2 - freqs = torch.randn(num_tokens, half_dim, device=device, dtype=torch.float32) - freqs = freqs.repeat_interleave(2, dim=-1) - cos_emb = freqs.cos() - sin_emb = freqs.sin() - - # Pre-allocate one working buffer per path; benchmark() runs the - # kernel back-to-back on the same tensor (timing is independent of - # values). - qkv_eager_buf = qkv_seed.clone() - qkv_compile_buf = qkv_seed.clone() - qkv_fused_buf = qkv_seed.clone() - - eager_ms = benchmark( - lambda: split_norm_rope( - qkv_eager_buf, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, - ) - ) - compile_ms = benchmark( - lambda: split_compiled( - qkv_compile_buf, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, - ), - warmup=100, # first few calls trigger Inductor compilation - ) - fused_ms = benchmark( - lambda: fused_norm_rope( - qkv_fused_buf, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, - ) - ) - - speedup_eager = eager_ms / fused_ms - speedup_compile = compile_ms / fused_ms - print( - f"{cfg['name']:<22} {eager_ms:>12.4f} {compile_ms:>14.4f} {fused_ms:>12.4f} " - f"{speedup_eager:>11.2f}x {speedup_compile:>13.2f}x" - ) - - -if __name__ == "__main__": - main() diff --git a/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.cu b/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.cu index ebf88704e844..6d3d2d39c6d7 100644 --- a/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.cu +++ b/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.cu @@ -46,10 +46,11 @@ __global__ void fusedDiTQKNormRopeKernel(__nv_bfloat16* qkv, // [num_tokens, tot __nv_bfloat16 const* k_weight, // [head_dim] __nv_bfloat16 const* q_add_weight, // [head_dim] or nullptr __nv_bfloat16 const* k_add_weight, // [head_dim] or nullptr - float const* cos_emb, // [num_tokens, head_dim] - float const* sin_emb, // [num_tokens, head_dim] + float const* cos_emb, // [cos_rows, head_dim] + float const* sin_emb, // [cos_rows, head_dim] int const num_tokens, int const num_txt_tokens, - int const tokens_per_batch) // seq_len per batch element; 0 = flat (no batching) + int const tokens_per_batch, // seq_len per batch element; 0 = flat (no batching) + int const cos_seq_per_batch) // cos rows per batch for broadcast; 0 = no broadcast { int const warpsPerBlock = blockDim.x / 32; int const warpId = threadIdx.x / 32; @@ -138,7 +139,10 @@ __global__ void fusedDiTQKNormRopeKernel(__nv_bfloat16* qkv, // [num_tokens, tot } // ---- Step 3: Apply RoPE with precomputed cos/sin ---- - int64_t const embOffset = static_cast(tokenIdx) * head_dim; + // Broadcast over B: when cos_seq_per_batch > 0, cos rows are < num_tokens and + // each token mods its index into cos rows. Bit-identical to host-side .repeat(). + int const cos_tokenIdx = (cos_seq_per_batch > 0) ? (tokenIdx % cos_seq_per_batch) : tokenIdx; + int64_t const embOffset = static_cast(cos_tokenIdx) * head_dim; if constexpr (interleave) { @@ -207,7 +211,7 @@ __global__ void fusedDiTQKNormRopeKernel(__nv_bfloat16* qkv, // [num_tokens, tot void launchFusedDiTQKNormRope(void* qkv, int num_tokens, int num_heads_q, int num_heads_k, int num_heads_v, int head_dim, float eps, void const* q_weight, void const* k_weight, void const* q_add_weight, void const* k_add_weight, float const* cos_emb, float const* sin_emb, int num_txt_tokens, bool interleave, - int tokens_per_batch, cudaStream_t stream) + int tokens_per_batch, int cos_seq_per_batch, cudaStream_t stream) { constexpr int blockSize = 256; @@ -224,7 +228,7 @@ void launchFusedDiTQKNormRope(void* qkv, int num_tokens, int num_heads_q, int nu reinterpret_cast<__nv_bfloat16*>(qkv), num_heads_q, num_heads_k, num_heads_v, eps, \ reinterpret_cast<__nv_bfloat16 const*>(q_weight), reinterpret_cast<__nv_bfloat16 const*>(k_weight), \ reinterpret_cast<__nv_bfloat16 const*>(q_add_weight), reinterpret_cast<__nv_bfloat16 const*>(k_add_weight), \ - cos_emb, sin_emb, num_tokens, num_txt_tokens, tokens_per_batch) + cos_emb, sin_emb, num_tokens, num_txt_tokens, tokens_per_batch, cos_seq_per_batch) if (interleave) { @@ -249,189 +253,6 @@ void launchFusedDiTQKNormRope(void* qkv, int num_tokens, int num_heads_q, int nu #undef LAUNCH_PER_HEAD_KERNEL } -//////////////////////////////////////////////////////////////////////////////////////////////////// -// -// Cross-head QK Norm + RoPE kernel (WAN, LTX-2) -// -// One CTA per (token, Q-or-K). RMSNorm is computed across ALL heads combined -// (norm dim = num_heads * head_dim), then RoPE is applied in-register. -// -// Grid: (num_tokens, 2) — blockIdx.y==0 → Q, blockIdx.y==1 → K -// Block: up to 1024 threads; each thread handles multiple bf16x2 pairs -// at stride blockDim.x. -// -// The packed QKV layout is [num_tokens, (Hq + Hk + Hv) * head_dim]. -// Only Q and K regions are read/written; V is untouched. -// -template -__global__ void fusedDiTCrossHeadQKNormRopeKernel(__nv_bfloat16* qkv, // [num_tokens, (Hq+Hk+Hv)*head_dim], in-place - int const q_dim, // num_heads_q * head_dim - int const k_dim, // num_heads_k * head_dim - int const total_row, // (Hq+Hk+Hv) * head_dim - int const head_dim, float const eps, - __nv_bfloat16 const* q_weight, // [q_dim] - __nv_bfloat16 const* k_weight, // [k_dim] - float const* cos_emb, // [num_tokens, head_dim] - float const* sin_emb, // [num_tokens, head_dim] - int const num_tokens) -{ - int const tokenIdx = blockIdx.x; - if (tokenIdx >= num_tokens) - { - return; - } - bool const isQ = (blockIdx.y == 0); - int const norm_dim = isQ ? q_dim : k_dim; - __nv_bfloat16 const* weight = isQ ? q_weight : k_weight; - - // Base offset into packed QKV for this token's Q or K region - int64_t const baseOffset = static_cast(tokenIdx) * total_row + (isQ ? 0 : q_dim); - - // ---- Step 1: Load elements and accumulate sum-of-squares ---- - int const halfDim = norm_dim / 2; // number of bf16x2 pairs - float threadSumSq = 0.0f; - - for (int i = threadIdx.x; i < halfDim; i += blockDim.x) - { - int const elemIdx = i * 2; - __nv_bfloat162 val = *reinterpret_cast<__nv_bfloat162 const*>(&qkv[baseOffset + elemIdx]); - float2 valf = __bfloat1622float2(val); - threadSumSq += valf.x * valf.x + valf.y * valf.y; - } - - // ---- Step 2: CTA-level reduction for sum-of-squares ---- - float totalSumSq = tensorrt_llm::common::blockReduceSum(threadSumSq); - - __shared__ float s_rms_rcp; - if (threadIdx.x == 0) - { - s_rms_rcp = rsqrtf(totalSumSq / static_cast(norm_dim) + eps); - } - __syncthreads(); - float const rms_rcp = s_rms_rcp; - - // ---- Step 3: Apply norm weight and interleaved RoPE, then store ---- - // cos/sin embeddings are [num_tokens, head_dim] and broadcast across heads. - int64_t const embBase = static_cast(tokenIdx) * head_dim; - - // Reload from global memory rather than caching in registers: norm_dim can be - // very large (e.g. 40 heads * 128 = 5120), so hoarding per-thread register arrays - // would tank occupancy. The second load likely hits L1/L2 cache from Step 1. - if constexpr (interleave) - { - for (int i = threadIdx.x; i < halfDim; i += blockDim.x) - { - int const elemIdx = i * 2; - // - __nv_bfloat162 val = *reinterpret_cast<__nv_bfloat162 const*>(&qkv[baseOffset + elemIdx]); - __nv_bfloat162 w = *reinterpret_cast<__nv_bfloat162 const*>(&weight[elemIdx]); - float2 valf = __bfloat1622float2(val); - float2 wf = __bfloat1622float2(w); - - float x0 = valf.x * rms_rcp * wf.x; - float x1 = valf.y * rms_rcp * wf.y; - - // Interleaved RoPE: pair (x[2j], x[2j+1]) within each head - int const dimInHead0 = elemIdx % head_dim; - int const dimInHead1 = dimInHead0 + 1; - - float cos0 = cos_emb[embBase + dimInHead0]; - float sin0 = sin_emb[embBase + dimInHead0]; - float cos1 = cos_emb[embBase + dimInHead1]; - float sin1 = sin_emb[embBase + dimInHead1]; - - float out0 = x0 * cos0 - x1 * sin0; - float out1 = x1 * cos1 + x0 * sin1; - - *reinterpret_cast<__nv_bfloat162*>(&qkv[baseOffset + elemIdx]) - = __float22bfloat162_rn(make_float2(out0, out1)); - } - } - else - { - // rotate_half requires a two-pass approach: within the same head, element i - // is paired with element i + head_dim/2, which is processed by a different - // thread. We stage normalized values in shared memory so that Pass 2 can - // read partners without racing against other threads' RoPE-phase writes to - // global memory. - // - // Layout: s_norm[0..norm_dim) holds the post-normalize (pre-RoPE) values. - // Allocated as dynamic shared memory sized = norm_dim * sizeof(bf16). - extern __shared__ __nv_bfloat16 s_norm[]; - - // Pass 1: normalize all elements and store them to shared memory. - for (int i = threadIdx.x; i < halfDim; i += blockDim.x) - { - int const elemIdx = i * 2; - __nv_bfloat162 val = *reinterpret_cast<__nv_bfloat162 const*>(&qkv[baseOffset + elemIdx]); - __nv_bfloat162 w = *reinterpret_cast<__nv_bfloat162 const*>(&weight[elemIdx]); - float2 valf = __bfloat1622float2(val); - float2 wf = __bfloat1622float2(w); - - float x0 = valf.x * rms_rcp * wf.x; - float x1 = valf.y * rms_rcp * wf.y; - - *reinterpret_cast<__nv_bfloat162*>(&s_norm[elemIdx]) = __float22bfloat162_rn(make_float2(x0, x1)); - } - __syncthreads(); - - // Pass 2: apply rotate_half RoPE. Self + partner are both read from - // s_norm (read-only in this pass), and results are written to qkv in - // global memory. Since different threads write disjoint (elemIdx, - // elemIdx+1) pairs, and partner reads come from shared memory which is - // never written during Pass 2, there is no read/write race. - int const halfHead = head_dim / 2; - for (int i = threadIdx.x; i < halfDim; i += blockDim.x) - { - int const elemIdx = i * 2; - - __nv_bfloat162 selfVal = *reinterpret_cast<__nv_bfloat162 const*>(&s_norm[elemIdx]); - float2 selfF = __bfloat1622float2(selfVal); - float x0 = selfF.x; - float x1 = selfF.y; - - int const localDim0 = elemIdx % head_dim; - int const localDim1 = localDim0 + 1; - // elemIdx is always even (i*2) and head_dim is always even (it must be - // a power of 2: 64/128/256 for supported models), so head boundaries - // fall on even indices. Therefore elemIdx and elemIdx+1 always belong - // to the same head and share the same head offset. - int const headOff = elemIdx - localDim0; - - // Partner is head_dim/2 away within the same head - int pDim0 = (localDim0 < halfHead) ? (headOff + localDim0 + halfHead) : (headOff + localDim0 - halfHead); - - // pDim0 is always even: localDim0 is even (elemIdx = i*2), halfHead is - // even (head_dim is a power of 2), and headOff is a multiple of - // head_dim. So we can load the (pDim0, pDim0+1) partner pair as a - // single bf16x2 vector load from shared memory. - __nv_bfloat162 partnerVal = *reinterpret_cast<__nv_bfloat162 const*>(&s_norm[pDim0]); - float2 partnerF = __bfloat1622float2(partnerVal); - float p0 = partnerF.x; - float p1 = partnerF.y; - - // First half: result = self * cos - partner * sin - // Second half: result = self * cos + partner * sin - // A single sign suffices for both elements: localDim0 is always even - // (elemIdx = i*2) and halfHead is always even (head_dim is power of 2), - // so the half-boundary (halfHead-1 / halfHead = odd/even) can never - // fall between localDim0 and localDim1. Both are always in the same half. - float sign = (localDim0 < halfHead) ? -1.0f : 1.0f; - - float cos0 = cos_emb[embBase + localDim0]; - float sin0 = sin_emb[embBase + localDim0]; - float cos1 = cos_emb[embBase + localDim1]; - float sin1 = sin_emb[embBase + localDim1]; - - float out0 = x0 * cos0 + sign * p0 * sin0; - float out1 = x1 * cos1 + sign * p1 * sin1; - - *reinterpret_cast<__nv_bfloat162*>(&qkv[baseOffset + elemIdx]) - = __float22bfloat162_rn(make_float2(out0, out1)); - } - } -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // Fused full-dim RMSNorm + RoPE on packed QKV; Q and K share a single cos/sin pair. // Strategy: @@ -450,7 +271,10 @@ __global__ void fusedDiTQKNormFullDimRopeKernel(__nv_bfloat16* qkv, int const nu constexpr int THREADS_PER_ROW = BLOCK_SIZE / ROWS_PER_BLOCK; // 128 constexpr int WARPS_PER_ROW = THREADS_PER_ROW / 32; // 4 constexpr int CHUNK_ELEMS = 8; // uint4 = 8 bf16 - constexpr int MAX_N = 32 * HEAD_DIM; + // MAX_N = max(num_heads_q) * HEAD_DIM. 64 covers WAN-14B (40 heads) and + // future ≤64-head models. SMEM budget at 64h × 128 = 128 KB (bf16 cos) / 192 KB + // (fp32 cos), both within B200's 227 KB dynamic SMEM cap. + constexpr int MAX_N = 64 * HEAD_DIM; constexpr int MAX_CHUNKS = (MAX_N + THREADS_PER_ROW * CHUNK_ELEMS - 1) / (THREADS_PER_ROW * CHUNK_ELEMS); #if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) @@ -709,54 +533,6 @@ __global__ void fusedDiTQKNormFullDimRopeKernel(__nv_bfloat16* qkv, int const nu #endif } -//////////////////////////////////////////////////////////////////////////////////////////////////// - -void launchFusedDiTCrossHeadQKNormRope(void* qkv, int num_tokens, int num_heads_q, int num_heads_k, int num_heads_v, - int head_dim, float eps, void const* q_weight, void const* k_weight, float const* cos_emb, float const* sin_emb, - bool interleave, cudaStream_t stream) -{ - int const q_dim = num_heads_q * head_dim; - int const k_dim = num_heads_k * head_dim; - int const total_row = (num_heads_q + num_heads_k + num_heads_v) * head_dim; - int const max_dim = (q_dim > k_dim) ? q_dim : k_dim; - - // Block size: enough threads to cover max_dim/2 bf16x2 pairs, capped at 1024 - int const halfDim = max_dim / 2; - int blockSize = 256; - if (halfDim > 256) - { - blockSize = 512; - } - if (halfDim > 512) - { - blockSize = 1024; - } - - dim3 grid(num_tokens, 2); // y=0 → Q, y=1 → K - dim3 block(blockSize); - - // rotate_half requires shared memory to stage normalized values (avoids a - // read/write race on partner elements in global memory); the interleaved path - // computes each output pair from the thread's own values and needs no smem. - size_t const smem_bytes = interleave ? 0 : static_cast(max_dim) * sizeof(__nv_bfloat16); - -#define LAUNCH_CROSS_HEAD_KERNEL(INTERLEAVE) \ - fusedDiTCrossHeadQKNormRopeKernel \ - <<>>(reinterpret_cast<__nv_bfloat16*>(qkv), q_dim, k_dim, total_row, \ - head_dim, eps, reinterpret_cast<__nv_bfloat16 const*>(q_weight), \ - reinterpret_cast<__nv_bfloat16 const*>(k_weight), cos_emb, sin_emb, num_tokens) - - if (interleave) - { - LAUNCH_CROSS_HEAD_KERNEL(true); - } - else - { - LAUNCH_CROSS_HEAD_KERNEL(false); - } -#undef LAUNCH_CROSS_HEAD_KERNEL -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // Full-dim launch: norm range = num_heads_q * head_dim (LTX-2 mode). // Requires num_heads_q == num_heads_k (block_size identical for grid.y=0/1). @@ -767,7 +543,7 @@ void launchFusedDiTQKNormRopeFullDim(void* qkv, int num_tokens, int num_heads_q, { TLLM_CHECK_WITH_INFO(num_heads_q == num_heads_k, "fusedDiTQKNormRopeFullDim: requires num_heads_q == num_heads_k (got %d, %d)", num_heads_q, num_heads_k); - TLLM_CHECK_WITH_INFO(num_heads_q <= 32, "fusedDiTQKNormRopeFullDim: num_heads must be <= 32, got %d", num_heads_q); + TLLM_CHECK_WITH_INFO(num_heads_q <= 64, "fusedDiTQKNormRopeFullDim: num_heads must be <= 64, got %d", num_heads_q); int const N = num_heads_q * head_dim; constexpr int ROWS_PER_BLOCK = 2; diff --git a/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.h b/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.h index d937c512cc7d..d34dfba8a2db 100644 --- a/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.h +++ b/cpp/tensorrt_llm/kernels/fusedDiTQKNormRopeKernel.h @@ -46,35 +46,15 @@ void launchFusedDiTQKNormRope(void* qkv, // [num_tokens, (Hq+Hk+Hv)*head_dim], i void const* k_weight, // [head_dim] void const* q_add_weight, // [head_dim] or nullptr (dual-stream text norm) void const* k_add_weight, // [head_dim] or nullptr - float const* cos_emb, // [num_tokens, head_dim], float32 - float const* sin_emb, // [num_tokens, head_dim], float32 + float const* cos_emb, // [cos_rows, head_dim], float32 (cos_rows = num_tokens or num_tokens/B) + float const* sin_emb, // [cos_rows, head_dim], float32 int num_txt_tokens, // Text token boundary; -1 = no dual-stream bool interleave, // true = interleaved pairs, false = rotate_half int tokens_per_batch, // seq_len per batch element for dual-stream; 0 = flat + int cos_seq_per_batch, // cos rows per batch for broadcast; 0 = no broadcast cudaStream_t stream); -// Fused cross-head QK Normalization + RoPE for Diffusion Transformers (DiT). -// -// Cross-head norm: one CTA per (token, Q-or-K), CTA-level shared-memory reduction. -// For WAN, LTX-2, and other models using full-dim QK norm. -// -// Features: -// - Precomputed cos/sin embeddings (broadcast across heads) -// - Interleaved or rotate_half RoPE modes -// -// Operates in-place on the packed QKV tensor. Only Q and K are modified; -// V is left untouched. - -void launchFusedDiTCrossHeadQKNormRope(void* qkv, // [num_tokens, (Hq+Hk+Hv)*head_dim], in-place - int num_tokens, int num_heads_q, int num_heads_k, int num_heads_v, int head_dim, float eps, - void const* q_weight, // [num_heads_q * head_dim] - void const* k_weight, // [num_heads_k * head_dim] - float const* cos_emb, // [num_tokens, head_dim], float32 - float const* sin_emb, // [num_tokens, head_dim], float32 - bool interleave, // true = interleaved pairs, false = rotate_half - cudaStream_t stream); - -// Full-dim variant for LTX-2: RMSNorm range = num_heads_per_side * head_dim. +// Full-dim variant for LTX-2 / WAN: RMSNorm range = num_heads_per_side * head_dim. // Requires num_heads_q == num_heads_k. No dual-stream support. // per_head_cos=false: cos/sin shape [num_tokens, head_dim] (head broadcast). // per_head_cos=true: cos/sin shape [num_tokens, num_heads*head_dim] diff --git a/cpp/tensorrt_llm/kernels/fusedDiTSplitNormKernel.cu b/cpp/tensorrt_llm/kernels/fusedDiTSplitNormKernel.cu index 3c2fe0c87010..3af8f560bc77 100644 --- a/cpp/tensorrt_llm/kernels/fusedDiTSplitNormKernel.cu +++ b/cpp/tensorrt_llm/kernels/fusedDiTSplitNormKernel.cu @@ -45,7 +45,9 @@ __global__ void fusedDiTSplitNormFullDimKernel(__nv_bfloat16* __restrict__ tenso constexpr int THREADS_PER_ROW = BLOCK_SIZE / ROWS_PER_BLOCK; // 128 constexpr int WARPS_PER_ROW = THREADS_PER_ROW / 32; // 4 constexpr int CHUNK_ELEMS = 8; // uint4 = 8 bf16 - constexpr int MAX_N = 32 * HEAD_DIM; + // MAX_N = max(num_heads) * HEAD_DIM. 64 covers WAN-14B (40 heads); matches + // the cap of fused_dit_qk_norm_rope (full-dim) and fused_dit_split_norm_rope. + constexpr int MAX_N = 64 * HEAD_DIM; constexpr int MAX_CHUNKS_PER_ROW = (MAX_N + THREADS_PER_ROW * CHUNK_ELEMS - 1) / (THREADS_PER_ROW * CHUNK_ELEMS); #if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) @@ -175,8 +177,9 @@ __global__ void fusedDiTSplitNormFullDimKernel(__nv_bfloat16* __restrict__ tenso void launchFusedDiTSplitNormFullDim( void* tensor, int num_tokens, int num_heads, int head_dim, float eps, void const* weight, cudaStream_t stream) { - TLLM_CHECK_WITH_INFO(num_heads <= 32, - "fusedDiTSplitNormFullDim: num_heads (%d) must be <= 32 (block_size = num_heads*32 <= 1024)", num_heads); + // num_heads cap is SMEM/register driven (MAX_N = 64 * HEAD_DIM in the kernel template), + // NOT block_size — the launcher uses a fixed blockDim=256 with 2 rows/CTA. + TLLM_CHECK_WITH_INFO(num_heads <= 64, "fusedDiTSplitNormFullDim: num_heads (%d) must be <= 64", num_heads); TLLM_CHECK_WITH_INFO(num_heads >= 1, "fusedDiTSplitNormFullDim: num_heads must be >= 1, got %d", num_heads); int const N = num_heads * head_dim; diff --git a/cpp/tensorrt_llm/kernels/fusedDiTSplitQKNormRopeKernel.cu b/cpp/tensorrt_llm/kernels/fusedDiTSplitQKNormRopeKernel.cu index 11b7d4f849b4..ab65c95b284f 100644 --- a/cpp/tensorrt_llm/kernels/fusedDiTSplitQKNormRopeKernel.cu +++ b/cpp/tensorrt_llm/kernels/fusedDiTSplitQKNormRopeKernel.cu @@ -46,7 +46,10 @@ __global__ void fusedDiTSplitNormFullDimRopeKernel(__nv_bfloat16* __restrict__ t constexpr int THREADS_PER_ROW = BLOCK_SIZE / ROWS_PER_BLOCK; // 128 constexpr int WARPS_PER_ROW = THREADS_PER_ROW / 32; // 4 constexpr int CHUNK_ELEMS = 8; // uint4 = 8 bf16 - constexpr int MAX_N = 32 * HEAD_DIM; + // MAX_N = max(num_heads) * HEAD_DIM. 64 covers WAN-14B (40 heads) and any + // future ≤64-head model; matches the cap of the packed fused_dit_qk_norm_rope + // full-dim template. SMEM budget at 64h × 128 stays well within B200's 227 KB. + constexpr int MAX_N = 64 * HEAD_DIM; constexpr int MAX_CHUNKS = (MAX_N + THREADS_PER_ROW * CHUNK_ELEMS - 1) / (THREADS_PER_ROW * CHUNK_ELEMS); #if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) @@ -288,8 +291,9 @@ void launchFusedDiTSplitNormFullDimRope(void* tensor, int num_tokens, int num_he void const* weight, void const* cos_emb, void const* sin_emb, bool interleave, bool per_head_cos, bool cos_is_bf16, int cos_seq_per_batch, cudaStream_t stream) { - TLLM_CHECK_WITH_INFO(num_heads <= 32, - "fusedDiTSplitNormFullDimRope: num_heads (%d) must be <= 32 (block_size = num_heads*32 <= 1024)", num_heads); + // num_heads cap is SMEM/register driven (MAX_N = 64 * HEAD_DIM in the kernel template), + // NOT block_size — the launcher uses a fixed blockDim=256 with 2 rows/CTA. + TLLM_CHECK_WITH_INFO(num_heads <= 64, "fusedDiTSplitNormFullDimRope: num_heads (%d) must be <= 64", num_heads); TLLM_CHECK_WITH_INFO(num_heads >= 1, "fusedDiTSplitNormFullDimRope: num_heads must be >= 1, got %d", num_heads); int const N = num_heads * head_dim; diff --git a/cpp/tensorrt_llm/thop/fusedDiTQKNormRopeOp.cpp b/cpp/tensorrt_llm/thop/fusedDiTQKNormRopeOp.cpp index fb046f455530..16269d5ca8e1 100644 --- a/cpp/tensorrt_llm/thop/fusedDiTQKNormRopeOp.cpp +++ b/cpp/tensorrt_llm/thop/fusedDiTQKNormRopeOp.cpp @@ -46,35 +46,46 @@ void fused_dit_qk_norm_rope(torch::Tensor& qkv, // [num_tokens, (Hq+Hk+Hv)*head_ TORCH_CHECK(qkv.dim() == 2, "QKV tensor must be 2D: [num_tokens, total_heads*head_dim]"); TORCH_CHECK(q_weight.dim() == 1, "q_weight must be 1D"); TORCH_CHECK(k_weight.dim() == 1, "k_weight must be 1D"); - TORCH_CHECK(cos_emb.dim() == 2, "cos_emb must be 2D: [num_tokens, head_dim] or [num_tokens, num_heads*head_dim]"); - TORCH_CHECK(sin_emb.dim() == 2, "sin_emb must be 2D: [num_tokens, head_dim] or [num_tokens, num_heads*head_dim]"); + TORCH_CHECK(cos_emb.dim() >= 2 && cos_emb.dim() <= 4, "cos_emb must have rank in [2, 4]; got ", cos_emb.dim()); + TORCH_CHECK(sin_emb.sizes() == cos_emb.sizes(), + "sin_emb shape must match cos_emb exactly (raw, pre-flatten); got cos=", cos_emb.sizes(), + " sin=", sin_emb.sizes()); + + // Flatten cos/sin to 2D internally. Two supported layouts: + // shape (..., num_heads_q, head_dim) → per-head cos, last 2 dims fold together + // shape (..., head_dim) → shared cos, all leading dims fold + int64_t const cos_last_raw = cos_emb.size(-1); + bool const fold_last_two = (cos_emb.dim() >= 3 && cos_last_raw == head_dim && cos_emb.size(-2) == num_heads_q); + int64_t const cos_new_last = fold_last_two ? num_heads_q * head_dim : cos_last_raw; + torch::Tensor cos_2d = cos_emb.reshape({-1, cos_new_last}).contiguous(); + torch::Tensor sin_2d = sin_emb.reshape({-1, cos_new_last}).contiguous(); CHECK_INPUT(qkv, torch::kBFloat16); CHECK_INPUT(q_weight, torch::kBFloat16); CHECK_INPUT(k_weight, torch::kBFloat16); // Cos/sin may be fp32 (per-head FLUX path) or bf16 (B-2 full-dim LTX-2 path). // Per-head path requires fp32 (kernel has no bf16 branch); enforced below. - auto const cos_dtype = cos_emb.scalar_type(); + auto const cos_dtype = cos_2d.scalar_type(); TORCH_CHECK(cos_dtype == torch::kFloat32 || cos_dtype == torch::kBFloat16, "cos_emb dtype must be float32 or bfloat16, got ", cos_dtype); - TORCH_CHECK(sin_emb.scalar_type() == cos_dtype, "sin_emb dtype must match cos_emb"); + TORCH_CHECK(sin_2d.scalar_type() == cos_dtype, "sin_emb dtype must match cos_emb"); bool const cos_is_bf16 = (cos_dtype == torch::kBFloat16); if (cos_is_bf16) { - CHECK_INPUT(cos_emb, torch::kBFloat16); - CHECK_INPUT(sin_emb, torch::kBFloat16); + CHECK_INPUT(cos_2d, torch::kBFloat16); + CHECK_INPUT(sin_2d, torch::kBFloat16); } else { - CHECK_INPUT(cos_emb, torch::kFloat32); - CHECK_INPUT(sin_emb, torch::kFloat32); + CHECK_INPUT(cos_2d, torch::kFloat32); + CHECK_INPUT(sin_2d, torch::kFloat32); } int64_t num_tokens = qkv.size(0); int64_t total_heads = num_heads_q + num_heads_k + num_heads_v; TORCH_CHECK(qkv.size(1) == total_heads * head_dim, "QKV tensor size must match total_heads * head_dim"); // Auto-detect broadcast: cos rows == num_tokens (flat) or num_tokens / B (broadcast over B). - int64_t const cos_rows = cos_emb.size(0); + int64_t const cos_rows = cos_2d.size(0); int cos_seq_per_batch = 0; if (cos_rows != num_tokens) { @@ -82,10 +93,10 @@ void fused_dit_qk_norm_rope(torch::Tensor& qkv, // [num_tokens, (Hq+Hk+Hv)*head_ ") must equal num_tokens (", num_tokens, ") or evenly divide it (broadcast); got non-divisor count"); cos_seq_per_batch = static_cast(cos_rows); } - bool const per_head_cos = (cos_emb.size(1) == num_heads_q * head_dim); - TORCH_CHECK(per_head_cos || cos_emb.size(1) == head_dim, "cos_emb last dim must be head_dim (", head_dim, - ") or num_heads_q*head_dim (", num_heads_q * head_dim, "); got ", cos_emb.size(1)); - TORCH_CHECK(sin_emb.size(0) == cos_rows && sin_emb.size(1) == cos_emb.size(1), "sin_emb shape must match cos_emb"); + bool const per_head_cos = (cos_2d.size(1) == num_heads_q * head_dim); + TORCH_CHECK(per_head_cos || cos_2d.size(1) == head_dim, "cos_emb last dim must be head_dim (", head_dim, + ") or num_heads_q*head_dim (", num_heads_q * head_dim, "); got ", cos_2d.size(1)); + TORCH_CHECK(sin_2d.size(0) == cos_rows && sin_2d.size(1) == cos_2d.size(1), "sin_emb shape must match cos_emb"); // Auto-dispatch by weight shape: // weight.size(0) == head_dim → per-head norm (FLUX/Cosmos3, original kernel) @@ -109,15 +120,12 @@ void fused_dit_qk_norm_rope(torch::Tensor& qkv, // [num_tokens, (Hq+Hk+Hv)*head_ tensorrt_llm::kernels::launchFusedDiTQKNormRopeFullDim(qkv.data_ptr(), static_cast(num_tokens), static_cast(num_heads_q), static_cast(num_heads_k), static_cast(num_heads_v), static_cast(head_dim), static_cast(eps), q_weight.data_ptr(), k_weight.data_ptr(), - cos_emb.data_ptr(), sin_emb.data_ptr(), interleave, per_head_cos, cos_is_bf16, cos_seq_per_batch, stream); + cos_2d.data_ptr(), sin_2d.data_ptr(), interleave, per_head_cos, cos_is_bf16, cos_seq_per_batch, stream); return; } - // Per-head path (original FLUX/Cosmos3 kernel) — only fp32 cos supported here, no broadcast. - TORCH_CHECK(cos_seq_per_batch == 0, - "Per-head fused_dit_qk_norm_rope (FLUX/Cosmos) does not support cos broadcast over B; " - "got cos_emb rows = ", - cos_rows, ", num_tokens = ", num_tokens); + // Per-head path (original FLUX/Cosmos3 kernel) — only fp32 cos supported here. + // Broadcast over B is now supported by the kernel via cos_seq_per_batch. TORCH_CHECK(!cos_is_bf16, "Per-head fused_dit_qk_norm_rope (FLUX/Cosmos) requires fp32 cos/sin; bf16 cos is only supported " "by the full-dim path (LTX-2)"); @@ -143,58 +151,8 @@ void fused_dit_qk_norm_rope(torch::Tensor& qkv, // [num_tokens, (Hq+Hk+Hv)*head_ tensorrt_llm::kernels::launchFusedDiTQKNormRope(qkv.data_ptr(), static_cast(num_tokens), static_cast(num_heads_q), static_cast(num_heads_k), static_cast(num_heads_v), static_cast(head_dim), static_cast(eps), q_weight.data_ptr(), k_weight.data_ptr(), q_add_ptr, - k_add_ptr, reinterpret_cast(cos_emb.data_ptr()), - reinterpret_cast(sin_emb.data_ptr()), static_cast(num_txt_tokens), interleave, - static_cast(tokens_per_batch), stream); -} - -// Fused cross-head QK Norm + RoPE for Diffusion Transformers. -// Full-dim norm for WAN, LTX-2 — norm computed across all heads combined. -void fused_dit_cross_head_qk_norm_rope(torch::Tensor& qkv, // [num_tokens, (Hq+Hk+Hv)*head_dim] - int64_t num_heads_q, int64_t num_heads_k, int64_t num_heads_v, int64_t head_dim, double eps, - torch::Tensor& q_weight, // [num_heads_q * head_dim] - torch::Tensor& k_weight, // [num_heads_k * head_dim] - torch::Tensor& cos_emb, // [num_tokens, head_dim], float32 - torch::Tensor& sin_emb, // [num_tokens, head_dim], float32 - bool interleave) -{ - TORCH_CHECK(qkv.dim() == 2, "QKV tensor must be 2D: [num_tokens, total_heads*head_dim]"); - TORCH_CHECK(q_weight.dim() == 1, "q_weight must be 1D"); - TORCH_CHECK(k_weight.dim() == 1, "k_weight must be 1D"); - TORCH_CHECK(cos_emb.dim() == 2, "cos_emb must be 2D: [num_tokens, head_dim]"); - TORCH_CHECK(sin_emb.dim() == 2, "sin_emb must be 2D: [num_tokens, head_dim]"); - - CHECK_INPUT(qkv, torch::kBFloat16); - CHECK_INPUT(q_weight, torch::kBFloat16); - CHECK_INPUT(k_weight, torch::kBFloat16); - CHECK_INPUT(cos_emb, torch::kFloat32); - CHECK_INPUT(sin_emb, torch::kFloat32); - - int64_t num_tokens = qkv.size(0); - int64_t total_heads = num_heads_q + num_heads_k + num_heads_v; - TORCH_CHECK(qkv.size(1) == total_heads * head_dim, "QKV tensor size must match total_heads * head_dim"); - TORCH_CHECK(cos_emb.size(0) == num_tokens && cos_emb.size(1) == head_dim, "cos_emb must be [num_tokens, head_dim]"); - TORCH_CHECK(sin_emb.size(0) == num_tokens && sin_emb.size(1) == head_dim, "sin_emb must be [num_tokens, head_dim]"); - - int64_t q_dim = num_heads_q * head_dim; - int64_t k_dim = num_heads_k * head_dim; - TORCH_CHECK(q_weight.size(0) == q_dim, - "Cross-head norm requires q_weight of size [num_heads_q * head_dim]. Got: ", q_weight.size(0), - ", expected: ", q_dim); - TORCH_CHECK(k_weight.size(0) == k_dim, - "Cross-head norm requires k_weight of size [num_heads_k * head_dim]. Got: ", k_weight.size(0), - ", expected: ", k_dim); - - TORCH_CHECK(head_dim % 2 == 0, "head_dim must be even for RoPE rotate_half. Got: ", head_dim); - TORCH_CHECK(q_dim % 2 == 0 && k_dim % 2 == 0, "q_dim and k_dim must be even for bf16x2 vectorized access"); - - auto stream = at::cuda::getCurrentCUDAStream(qkv.get_device()); - - tensorrt_llm::kernels::launchFusedDiTCrossHeadQKNormRope(qkv.data_ptr(), static_cast(num_tokens), - static_cast(num_heads_q), static_cast(num_heads_k), static_cast(num_heads_v), - static_cast(head_dim), static_cast(eps), q_weight.data_ptr(), k_weight.data_ptr(), - reinterpret_cast(cos_emb.data_ptr()), reinterpret_cast(sin_emb.data_ptr()), - interleave, stream); + k_add_ptr, reinterpret_cast(cos_2d.data_ptr()), reinterpret_cast(sin_2d.data_ptr()), + static_cast(num_txt_tokens), interleave, static_cast(tokens_per_batch), cos_seq_per_batch, stream); } // Register the PyTorch operator schema @@ -206,17 +164,12 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) "Tensor? q_add_weight, Tensor? k_add_weight, " "Tensor cos_emb, Tensor sin_emb, SymInt num_txt_tokens, " "bool interleave, SymInt tokens_per_batch) -> ()"); - m.def( - "fused_dit_cross_head_qk_norm_rope(Tensor(a!) qkv, int num_heads_q, int num_heads_k, int num_heads_v, " - "int head_dim, float eps, Tensor q_weight, Tensor k_weight, " - "Tensor cos_emb, Tensor sin_emb, bool interleave) -> ()"); } // Register the CUDA implementation TORCH_LIBRARY_IMPL(trtllm, CUDA, m) { m.impl("fused_dit_qk_norm_rope", &fused_dit_qk_norm_rope); - m.impl("fused_dit_cross_head_qk_norm_rope", &fused_dit_cross_head_qk_norm_rope); } } // namespace torch_ext diff --git a/cpp/tensorrt_llm/thop/fusedDiTSplitQKNormRopeOp.cpp b/cpp/tensorrt_llm/thop/fusedDiTSplitQKNormRopeOp.cpp index e942ea10580b..b2967bd6e2d8 100644 --- a/cpp/tensorrt_llm/thop/fusedDiTSplitQKNormRopeOp.cpp +++ b/cpp/tensorrt_llm/thop/fusedDiTSplitQKNormRopeOp.cpp @@ -34,27 +34,38 @@ void fused_dit_split_norm_rope(torch::Tensor& tensor, int64_t num_heads, int64_t { TORCH_CHECK(tensor.dim() == 2, "tensor must be 2D: [num_tokens, num_heads*head_dim]"); TORCH_CHECK(weight.dim() == 1, "weight must be 1D"); - TORCH_CHECK(cos_emb.dim() == 2, "cos_emb must be 2D"); - TORCH_CHECK(sin_emb.dim() == 2, "sin_emb must be 2D"); + TORCH_CHECK(cos_emb.dim() >= 2 && cos_emb.dim() <= 4, "cos_emb must have rank in [2, 4]; got ", cos_emb.dim()); + TORCH_CHECK(sin_emb.sizes() == cos_emb.sizes(), + "sin_emb shape must match cos_emb exactly (raw, pre-flatten); got cos=", cos_emb.sizes(), + " sin=", sin_emb.sizes()); + + // Flatten cos/sin to 2D internally. Two supported layouts: + // shape (..., num_heads, head_dim) → per-head cos, fold last 2 dims together + // shape (..., head_dim) → shared cos, fold all leading dims + int64_t const cos_last_raw = cos_emb.size(-1); + bool const fold_last_two = (cos_emb.dim() >= 3 && cos_last_raw == head_dim && cos_emb.size(-2) == num_heads); + int64_t const cos_new_last = fold_last_two ? num_heads * head_dim : cos_last_raw; + torch::Tensor cos_2d = cos_emb.reshape({-1, cos_new_last}).contiguous(); + torch::Tensor sin_2d = sin_emb.reshape({-1, cos_new_last}).contiguous(); CHECK_INPUT(tensor, torch::kBFloat16); CHECK_INPUT(weight, torch::kBFloat16); // Cos/sin may be fp32 or bf16 (kernel upcasts bf16 to fp32 in registers, lossless). - auto const cos_dtype = cos_emb.scalar_type(); + auto const cos_dtype = cos_2d.scalar_type(); TORCH_CHECK(cos_dtype == torch::kFloat32 || cos_dtype == torch::kBFloat16, "cos_emb dtype must be float32 or bfloat16, got ", cos_dtype); - TORCH_CHECK(sin_emb.scalar_type() == cos_dtype, "sin_emb dtype must match cos_emb (", sin_emb.scalar_type(), " vs ", + TORCH_CHECK(sin_2d.scalar_type() == cos_dtype, "sin_emb dtype must match cos_emb (", sin_2d.scalar_type(), " vs ", cos_dtype, ")"); bool const cos_is_bf16 = (cos_dtype == torch::kBFloat16); if (cos_is_bf16) { - CHECK_INPUT(cos_emb, torch::kBFloat16); - CHECK_INPUT(sin_emb, torch::kBFloat16); + CHECK_INPUT(cos_2d, torch::kBFloat16); + CHECK_INPUT(sin_2d, torch::kBFloat16); } else { - CHECK_INPUT(cos_emb, torch::kFloat32); - CHECK_INPUT(sin_emb, torch::kFloat32); + CHECK_INPUT(cos_2d, torch::kFloat32); + CHECK_INPUT(sin_2d, torch::kFloat32); } int64_t const num_tokens = tensor.size(0); @@ -63,7 +74,7 @@ void fused_dit_split_norm_rope(torch::Tensor& tensor, int64_t num_heads, int64_t // Auto-detect broadcast: cos may carry one row per token (num_tokens) or one row // per token-in-batch (num_tokens / B); in the latter case the kernel broadcasts // cos across B via cos_tokenIdx = tokenIdx % cos_seq_per_batch. - int64_t const cos_rows = cos_emb.size(0); + int64_t const cos_rows = cos_2d.size(0); int cos_seq_per_batch = 0; if (cos_rows != num_tokens) { @@ -71,10 +82,10 @@ void fused_dit_split_norm_rope(torch::Tensor& tensor, int64_t num_heads, int64_t ") must equal num_tokens (", num_tokens, ") or evenly divide it (broadcast); got non-divisor count"); cos_seq_per_batch = static_cast(cos_rows); } - bool const per_head_cos = (cos_emb.size(1) == num_heads * head_dim); - TORCH_CHECK(per_head_cos || cos_emb.size(1) == head_dim, "cos_emb last dim must be head_dim (", head_dim, - ") or num_heads*head_dim (", num_heads * head_dim, "); got ", cos_emb.size(1)); - TORCH_CHECK(sin_emb.size(0) == cos_rows && sin_emb.size(1) == cos_emb.size(1), "sin_emb shape must match cos_emb"); + bool const per_head_cos = (cos_2d.size(1) == num_heads * head_dim); + TORCH_CHECK(per_head_cos || cos_2d.size(1) == head_dim, "cos_emb last dim must be head_dim (", head_dim, + ") or num_heads*head_dim (", num_heads * head_dim, "); got ", cos_2d.size(1)); + TORCH_CHECK(sin_2d.size(0) == cos_rows && sin_2d.size(1) == cos_2d.size(1), "sin_emb shape must match cos_emb"); TORCH_CHECK(weight.size(0) == num_heads * head_dim, "weight must be [num_heads*head_dim] (full-dim norm), got ", weight.size(0), " expected ", num_heads * head_dim); @@ -82,7 +93,7 @@ void fused_dit_split_norm_rope(torch::Tensor& tensor, int64_t num_heads, int64_t tensorrt_llm::kernels::launchFusedDiTSplitNormFullDimRope(tensor.data_ptr(), static_cast(num_tokens), static_cast(num_heads), static_cast(head_dim), static_cast(eps), weight.data_ptr(), - cos_emb.data_ptr(), sin_emb.data_ptr(), interleave, per_head_cos, cos_is_bf16, cos_seq_per_batch, stream); + cos_2d.data_ptr(), sin_2d.data_ptr(), interleave, per_head_cos, cos_is_bf16, cos_seq_per_batch, stream); } TORCH_LIBRARY_FRAGMENT(trtllm, m) diff --git a/tensorrt_llm/_torch/compilation/utils.py b/tensorrt_llm/_torch/compilation/utils.py index 83c912abda47..51c71e25ee2e 100644 --- a/tensorrt_llm/_torch/compilation/utils.py +++ b/tensorrt_llm/_torch/compilation/utils.py @@ -82,9 +82,6 @@ def inplace_info(): torch.ops.trtllm.fused_dit_qk_norm_rope.default: { 1: "qkv" }, - torch.ops.trtllm.fused_dit_cross_head_qk_norm_rope.default: { - 1: "qkv" - }, torch.ops.trtllm.fused_dit_split_norm_rope.default: { 1: "tensor" }, diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py index eb65911818bc..647e73a554a6 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py @@ -226,7 +226,9 @@ def project_kv( v = self.to_v(context) # All cross-attn K-norm paths (with or without RoPE) go through the - # split-fuse kernels. fallback only kicks in for unsupported head_dim. + # split-fuse kernels. Fallback only kicks in for unsupported head_dim + # — the fused kernel template covers {64, 128}; mini-config tests use + # head_dim=32 and must take the eager branch. if self.qk_norm and self.head_dim in (64, 128): self.apply_split_norm_or_norm_rope(k, self.norm_k.weight, self.num_key_value_heads, pe) else: @@ -255,7 +257,7 @@ def forward( # Fallback to the naive eager rope path when fusion is disabled or # the kernel doesn't support this head_dim. LTX-2 prod has # fuse_qk_norm_rope=True and head_dim ∈ {64, 128}, so this branch - # never fires in production. + # only fires under mini-config unit tests (head_dim=32). if not self.fuse_qk_norm_rope or self.head_dim not in (64, 128): return self._forward_unfused(x, context, pe, k_pe, pre_projected_kv) @@ -312,12 +314,13 @@ def _forward_unfused( k_pe: tuple[torch.Tensor, torch.Tensor] | None, pre_projected_kv: tuple[torch.Tensor, torch.Tensor] | None, ) -> torch.Tensor: - """Fallback path for unsupported configs (e.g. head_dim ∉ {64, 128} or - fuse_qk_norm_rope=False). + """Fallback path for unsupported configs (head_dim ∉ {64, 128} or + ``fuse_qk_norm_rope=False``). - LTX-2 prod uses fused (head_dim ∈ {64, 128} and fuse_qk_norm_rope=True - by default), so in practice this is never entered. Kept for safety in - case the class is reused for other models or fusion is explicitly off. + LTX-2 prod hardcodes ``fuse_qk_norm_rope=True`` and head_dim ∈ + {64, 128}, so in production this is never entered. Exercised by the + mini-config unit tests in ``test_ltx2_transformer.py`` (head_dim=32) + and by ablation tests that explicitly disable fusion. Contract: caller must pass *pe* / *k_pe* in 4D layout ([B, T, H, D] for SPLIT rope, [B, T, D] for INTERLEAVED). The fused diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py index b7af3f6d8464..8b5ddaaf84e2 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py @@ -281,11 +281,9 @@ def __init__( hidden_size=hidden_size, eps=eps, dtype=torch.float32, has_weights=False, has_bias=False ) - # Self-attention with fused QKV. - # WAN-14B is 40 heads × 128, which exceeds the default fused op's - # num_heads<=32 / head_dim in {64,128} envelope, so route to - # PR #13052's fused_dit_cross_head_qk_norm_rope op (validated for - # WAN-1.3B and WAN-14B sizes). + # Self-attention with fused QKV. All WAN variants (1.3B 12h, 5B 24h, 14B 40h) + # fit the default fused_dit_qk_norm_rope op's full-dim template now that + # the num_heads cap is 64 (post-survey 2026-05). self.attn1 = Attention( hidden_size=hidden_size, num_attention_heads=num_heads, @@ -294,7 +292,6 @@ def __init__( qk_norm=True, eps=eps, fuse_qk_norm_rope=True, - qk_norm_rope_kernel="fused_dit_cross_head_qk_norm_rope", config=model_config, layer_idx=_layer_idx, ) diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index e6f9f66a0a8c..676f3aec29fa 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -49,7 +49,6 @@ def __init__( bias: bool = True, interleave: bool = True, fuse_qk_norm_rope: Optional[bool] = None, - qk_norm_rope_kernel: str = "fused_dit_qk_norm_rope", config: Optional[DiffusionModelConfig] = None, layer_idx: Optional[int] = None, ): @@ -70,26 +69,11 @@ def __init__( self.bias = bias # Fused QK Norm + RoPE: each model class opts in via fuse_qk_norm_rope. - # Supported for both per_head (FLUX) and full/cross-head (WAN) norm modes. - # Defaults to False; models that want the fused kernel must pass True explicitly. + # Backed by torch.ops.trtllm.fused_dit_qk_norm_rope which auto-dispatches: + # - per-head template (FLUX/Cosmos): q/k_weight.shape == [head_dim] + # - full-dim template (LTX-2, WAN): q/k_weight.shape == [num_heads * head_dim] + # Full-dim template envelope: num_heads <= 64, head_dim in {64, 128}. self.fuse_qk_norm_rope = fuse_qk_norm_rope if fuse_qk_norm_rope is not None else False - # Which trtllm op backs the fused path. Value is the op name itself - # (1:1 with torch.ops.trtllm registrations): - # "fused_dit_qk_norm_rope" (default) - # — Per-head autodispatch for FLUX/Cosmos and full-dim path for - # LTX-2. Bounded to num_heads<=32 and head_dim in {64,128}. - # "fused_dit_cross_head_qk_norm_rope" - # — PR #13052 cross-head kernel for WAN sizes outside the default - # op's range (head_dim=256, or num_heads>32). fp32 head-broadcast - # cos only. - assert qk_norm_rope_kernel in ( - "fused_dit_qk_norm_rope", - "fused_dit_cross_head_qk_norm_rope", - ), ( - f"qk_norm_rope_kernel must be 'fused_dit_qk_norm_rope' or " - f"'fused_dit_cross_head_qk_norm_rope', got {qk_norm_rope_kernel}" - ) - self.qk_norm_rope_kernel = qk_norm_rope_kernel self.interleave = interleave # Select compute backend (orthogonal to parallelism) @@ -298,71 +282,19 @@ def apply_packed_qk_norm_rope( ) -> None: """Apply fused QK Norm + RoPE in-place on packed QKV tensor (FUSE_QKV self-attn). - cos/sin can be either shape (per-token total elements): - - [..., head_dim] : shared across heads (FLUX/Cosmos style) - - [..., num_heads*head_dim] : per-head freqs (LTX-2 3D RoPE style) - Op auto-detects via cos_emb.size(1) and dispatches the kernel template. + The op auto-dispatches by tensor shapes/dtypes: + - q_weight.shape = [head_dim] → per-head norm (FLUX/Cosmos, w/ dual-stream) + - q_weight.shape = [num_heads * head_dim] → full-dim norm (LTX-2, WAN, ≤64 heads) + - cos last dim = head_dim → per-token shared cos (FLUX, WAN) + - cos last dim = num_heads * head_dim → per-token per-head cos (LTX-2 3D RoPE) + - cos rows < num_tokens → kernel broadcasts via cos_seq_per_batch + + Caller passes raw freqs_cos / freqs_sin (any rank); the op reshapes internally. """ B, S, D = qkv.shape - - if self.qk_norm_rope_kernel == "fused_dit_cross_head_qk_norm_rope": - # PR #13052 cross-head op path (WAN sizes that exceed the default - # op's num_heads<=32 / head_dim in {64,128} envelope). Op requires - # head-broadcast fp32 cos: [num_tokens, head_dim]. - cos_2d = freqs_cos.reshape(-1, self.head_dim).float().contiguous() - sin_2d = freqs_sin.reshape(-1, self.head_dim).float().contiguous() - if cos_2d.shape[0] == S and B > 1: - cos_tiled = cos_2d.repeat(B, 1) - sin_tiled = sin_2d.repeat(B, 1) - else: - cos_tiled = cos_2d - sin_tiled = sin_2d - qkv_2d = qkv.view(B * S, D) - torch.ops.trtllm.fused_dit_cross_head_qk_norm_rope( - qkv_2d, - self.num_attention_heads, - self.num_key_value_heads, - self.num_key_value_heads, - self.head_dim, - self.eps, - self.norm_q.weight, - self.norm_k.weight, - cos_tiled, - sin_tiled, - self.interleave, - ) - return - - # cos last-dim is fixed by qk_norm_mode: - # "full" → num_heads * head_dim (LTX-2 / WAN per-head cos) - # "per_head" → head_dim (FLUX / Cosmos shared-across-heads cos) - cos_last = self.q_dim if self.qk_norm_mode == "full" else self.head_dim - # cos/sin are token-major [B, T, H, D] (or shared per-token [B, T, D]); - # reshape(-1, cos_last) yields the [B*T, cos_last] layout the kernel reads. - # Full-dim LTX-2 / WAN path accepts bf16 cos (kernel upcasts in registers, lossless); - # FLUX per-head path requires fp32 (and cos is already fp32 upstream there). - cos_2d = freqs_cos.reshape(-1, cos_last).contiguous() - sin_2d = freqs_sin.reshape(-1, cos_last).contiguous() - # LTX-2 / WAN full-dim path: kernel broadcasts cos over B internally - # (cos_tokenIdx = tokenIdx % cos_seq_per_batch in the fused kernel), so we - # pass cos as-is regardless of B. FLUX / Cosmos per-head path: kernel does - # not support broadcast, so host still has to tile when B > 1. - if self.qk_norm_mode == "full" or cos_2d.shape[0] != S or B == 1: - cos_tiled = cos_2d - sin_tiled = sin_2d - else: - cos_tiled = cos_2d.repeat(B, 1) - sin_tiled = sin_2d.repeat(B, 1) - qkv_2d = qkv.view(B * S, D) - - # Dual-stream batch correction: when B>1 and dual-stream is active, - # the kernel uses modulo (tokenIdx % tokens_per_batch) to find the - # local position within each batch element for the text/image boundary. - # 0 = no dual-stream (single-stream or batch=1). tokens_per_batch = S if num_txt_tokens > 0 else 0 - torch.ops.trtllm.fused_dit_qk_norm_rope( - qkv_2d, + qkv.view(B * S, D), self.num_attention_heads, self.num_key_value_heads, self.num_key_value_heads, @@ -372,8 +304,8 @@ def apply_packed_qk_norm_rope( self.norm_k.weight, q_add_weight, k_add_weight, - cos_tiled, - sin_tiled, + freqs_cos, + freqs_sin, num_txt_tokens, self.interleave, tokens_per_batch, @@ -389,25 +321,22 @@ def apply_split_norm_rope( ) -> None: """In-place fused RMSNorm + RoPE on a single Q or K tensor [B, T, H*D] (SEPARATE_QKV cross-attn). - Calls trtllm.fused_dit_split_norm_rope. Full-dim per-head cos in - [B, T, H, D] reshape(-1, H*D) -> [B*T, H*D] is what the kernel reads; - the kernel broadcasts cos over B internally - (cos_tokenIdx = tokenIdx % cos_seq_per_batch), so we pass cos as-is. - bf16 cos is also accepted (kernel upcasts to fp32 in registers). + The op auto-dispatches by shapes/dtypes (same as `apply_packed_qk_norm_rope`): + - cos last dim = head_dim → per-token shared cos + - cos last dim = num_heads * head_dim → per-token per-head cos (LTX-2 3D RoPE) + - cos rows < num_tokens → kernel broadcasts via cos_seq_per_batch + - cos dtype bf16 or fp32 → kernel upcasts bf16 to fp32 in registers + Caller passes raw cos/sin (any rank); the op reshapes internally. """ B, T, _ = tensor.shape - cos_last = num_heads * self.head_dim - cos_2d = cos.reshape(-1, cos_last).contiguous() - sin_2d = sin.reshape(-1, cos_last).contiguous() - tensor_2d = tensor.view(B * T, -1) torch.ops.trtllm.fused_dit_split_norm_rope( - tensor_2d, + tensor.view(B * T, -1), num_heads, self.head_dim, self.eps, weight, - cos_2d, - sin_2d, + cos, + sin, self.interleave, ) @@ -423,9 +352,8 @@ def apply_split_norm( no RoPE -- e.g. LTX-2 text cross-attn (Q-norm with pe=None). """ B, T, _ = tensor.shape - tensor_2d = tensor.view(B * T, -1) torch.ops.trtllm.fused_dit_split_norm( - tensor_2d, + tensor.view(B * T, -1), num_heads, self.head_dim, self.eps, diff --git a/tests/integration/defs/examples/test_visual_gen.py b/tests/integration/defs/examples/test_visual_gen.py index d71ade3853c6..b91839f51c2e 100644 --- a/tests/integration/defs/examples/test_visual_gen.py +++ b/tests/integration/defs/examples/test_visual_gen.py @@ -401,18 +401,6 @@ def _save_lpips_video_mp4(video, output_path, frame_rate): assert os.path.isfile(output_path), f"Visual gen did not produce {output_path}" -def _disable_wan_fused_qk_norm_rope_if_unavailable(pipeline): - try: - getattr(torch.ops.trtllm, "fused_dit_cross_head_qk_norm_rope") - return - except AttributeError: - pass - - for module in pipeline.modules(): - if hasattr(module, "fuse_qk_norm_rope"): - module.fuse_qk_norm_rope = False - - def _run_lpips_eval(tmp_path, sample_id, media_type, prompt, reference_path, generated_path): reference_key = "reference_video_path" if media_type == "video" else "reference_image_path" generated_key = "generated_video_path" if media_type == "video" else "generated_image_path" @@ -561,7 +549,6 @@ def _generate_wan_lpips_video( torch_compile_config=TorchCompileConfig(enable=False), ) pipeline = PipelineLoader(args).load(skip_warmup=True) - _disable_wan_fused_qk_norm_rope_if_unavailable(pipeline) try: with torch.no_grad(): result = pipeline.forward( diff --git a/tests/unittest/_torch/thop/parallel_hw_agnostic/test_fused_dit_qk_norm_rope.py b/tests/unittest/_torch/thop/parallel_hw_agnostic/test_fused_dit_qk_norm_rope.py index b5da528c3581..13a8d114d4a6 100644 --- a/tests/unittest/_torch/thop/parallel_hw_agnostic/test_fused_dit_qk_norm_rope.py +++ b/tests/unittest/_torch/thop/parallel_hw_agnostic/test_fused_dit_qk_norm_rope.py @@ -467,273 +467,6 @@ def test_per_head_v_unchanged(head_dim): torch.testing.assert_close(qkv[:, -v_size:], v_original, rtol=0, atol=0) -# ============================================================================ -# Cross-head norm reference implementation -# ============================================================================ - - -@torch.inference_mode() -def torch_ref_cross_head( - qkv, - num_heads_q, - num_heads_k, - num_heads_v, - head_dim, - eps, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, -): - """Reference: cross-head (full-dim) RMSNorm + RoPE (WAN pattern).""" - num_tokens = qkv.shape[0] - q_size = num_heads_q * head_dim - k_size = num_heads_k * head_dim - - q = qkv[:, :q_size].clone() - k = qkv[:, q_size : q_size + k_size].clone() - v = qkv[:, q_size + k_size :].clone() - - # Cross-head RMSNorm: norm over the full q_dim / k_dim dimension - q = F.rms_norm(q.float(), (q_size,), q_weight.float(), eps).to(qkv.dtype) - k = F.rms_norm(k.float(), (k_size,), k_weight.float(), eps).to(qkv.dtype) - - # Expand cos/sin per head for RoPE - cos_q = cos_emb.unsqueeze(1).expand(-1, num_heads_q, -1).reshape(num_tokens, q_size) - sin_q = sin_emb.unsqueeze(1).expand(-1, num_heads_q, -1).reshape(num_tokens, q_size) - cos_k = cos_emb.unsqueeze(1).expand(-1, num_heads_k, -1).reshape(num_tokens, k_size) - sin_k = sin_emb.unsqueeze(1).expand(-1, num_heads_k, -1).reshape(num_tokens, k_size) - - if interleave: - q = _apply_interleaved_rope(q, cos_q, sin_q) - k = _apply_interleaved_rope(k, cos_k, sin_k) - else: - q = _apply_rotate_half_rope(q, cos_q, sin_q, head_dim, num_heads_q) - k = _apply_rotate_half_rope(k, cos_k, sin_k, head_dim, num_heads_k) - - return torch.cat([q.to(qkv.dtype), k.to(qkv.dtype), v], dim=1) - - -def _call_cross_head_kernel( - qkv, - num_heads_q, - num_heads_k, - num_heads_v, - head_dim, - eps, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave=True, -): - """Call the fused DiT cross-head QK Norm + RoPE kernel.""" - torch.ops.trtllm.fused_dit_cross_head_qk_norm_rope( - qkv, - num_heads_q, - num_heads_k, - num_heads_v, - head_dim, - eps, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave, - ) - - -# ============================================================================ -# Cross-head norm tests (WAN pattern) -# ============================================================================ - - -@pytest.mark.parametrize( - "num_heads,head_dim", - [ - (12, 128), # WAN 1.3B: hidden_size=1536 - (40, 128), # WAN 14B: hidden_size=5120 - ], -) -@pytest.mark.parametrize("num_tokens", [1, 64, 512]) -def test_cross_head_interleaved(num_heads, head_dim, num_tokens): - """Cross-head norm + interleaved RoPE (WAN pattern).""" - device = "cuda" - torch.random.manual_seed(42) - - hidden_size = 3 * num_heads * head_dim - qkv = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) - qkv_copy = qkv.clone() - - q_dim = num_heads * head_dim - q_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - k_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - cos_emb, sin_emb = _generate_cos_sin(num_tokens, head_dim, device) - - _call_cross_head_kernel( - qkv, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave=True, - ) - - ref = torch_ref_cross_head( - qkv_copy, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - True, - ) - torch.testing.assert_close(qkv, ref, rtol=5e-2, atol=1e-1) - - -@pytest.mark.parametrize("num_heads", [12, 40]) -@pytest.mark.parametrize("num_tokens", [1, 64, 512]) -def test_cross_head_rotate_half(num_heads, num_tokens): - """Cross-head norm + rotate_half RoPE.""" - device = "cuda" - head_dim = 128 - torch.random.manual_seed(42) - - hidden_size = 3 * num_heads * head_dim - qkv = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) - qkv_copy = qkv.clone() - - q_dim = num_heads * head_dim - q_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - k_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - cos_emb, sin_emb = _generate_cos_sin(num_tokens, head_dim, device) - - _call_cross_head_kernel( - qkv, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - interleave=False, - ) - - ref = torch_ref_cross_head( - qkv_copy, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - False, - ) - torch.testing.assert_close(qkv, ref, rtol=5e-2, atol=1e-1) - - -@pytest.mark.parametrize("num_heads", [12, 40]) -def test_cross_head_v_unchanged(num_heads): - """Verify V is not modified by cross-head kernel.""" - device = "cuda" - head_dim = 128 - num_tokens = 64 - hidden_size = 3 * num_heads * head_dim - - torch.random.manual_seed(0) - qkv = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) - v_size = num_heads * head_dim - v_original = qkv[:, -v_size:].clone() - - q_dim = num_heads * head_dim - q_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) - k_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) - cos_emb, sin_emb = _generate_cos_sin(num_tokens, head_dim, device) - - _call_cross_head_kernel( - qkv, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - ) - torch.testing.assert_close(qkv[:, -v_size:], v_original, rtol=0, atol=0) - - -@pytest.mark.parametrize("batch_size", [2, 4]) -def test_cross_head_batched(batch_size): - """Cross-head norm + interleaved RoPE with batch_size > 1. - - Simulates the batch-flattening done by Attention.apply_qk_norm_rope. - """ - device = "cuda" - num_heads = 12 - head_dim = 128 - num_tokens = 256 - - torch.random.manual_seed(42) - total_tokens = batch_size * num_tokens - hidden_size = 3 * num_heads * head_dim - qkv = torch.randn(total_tokens, hidden_size, dtype=torch.bfloat16, device=device) - qkv_copy = qkv.clone() - - q_dim = num_heads * head_dim - q_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - k_weight = torch.randn(q_dim, dtype=torch.bfloat16, device=device) * 5.0 - cos_single, sin_single = _generate_cos_sin(num_tokens, head_dim, device) - cos_emb = cos_single.repeat(batch_size, 1) - sin_emb = sin_single.repeat(batch_size, 1) - - _call_cross_head_kernel( - qkv, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - ) - - ref = torch_ref_cross_head( - qkv_copy, - num_heads, - num_heads, - num_heads, - head_dim, - 1e-6, - q_weight, - k_weight, - cos_emb, - sin_emb, - True, - ) - torch.testing.assert_close(qkv, ref, rtol=5e-2, atol=1e-1) - - # ============================================================================ # Full-dim norm tests (LTX-2 / WAN packed FUSE_QKV) # weight is [num_heads*head_dim] (full-dim per-token norm), no dual-stream, From 6130b2a30e0acd9a4ca3c3da0128635a01acdbb2 Mon Sep 17 00:00:00 2001 From: Joyjit Daw <1127155+tijyojwad@users.noreply.github.com> Date: Tue, 26 May 2026 13:39:55 -0400 Subject: [PATCH 036/308] [https://nvbugs/6143579][fix] Allow content: null in CustomChatCompletionMessageParam (#14368) Signed-off-by: tijyojwad <1127155+tijyojwad@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 2 +- tests/unittest/llmapi/apps/test_chat_utils.py | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 95b6c1270505..e9282795c34f 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -565,7 +565,7 @@ class CustomChatCompletionMessageParam(TypedDict, total=False): role: Required[str] """The role of the message's author.""" - content: Union[str, List[ChatCompletionContentPartParam]] + content: Union[str, List[ChatCompletionContentPartParam], None] """The contents of the message.""" name: str diff --git a/tests/unittest/llmapi/apps/test_chat_utils.py b/tests/unittest/llmapi/apps/test_chat_utils.py index 53f282524e77..74b7123129a2 100644 --- a/tests/unittest/llmapi/apps/test_chat_utils.py +++ b/tests/unittest/llmapi/apps/test_chat_utils.py @@ -591,3 +591,97 @@ async def test_input_audio_synthesizes_data_url(self, mm_tracker_factory): result = parse_chat_message_content_part(part, tracker) await result["data"] mock_load.assert_called_once_with("data:audio/wav;base64,AAAA") + + +class TestChatCompletionRequestSchemaValidation: + """Tests that ChatCompletionRequest pydantic schema accepts valid payloads. + + Regression tests for NVBUG 6143579: multi-turn messages with + reasoning_content and content: null were rejected by the schema. + """ + + def test_reasoning_content_with_null_content_and_tool_calls(self): + """Round-tripping an assistant message with reasoning_content. + + content: null must be accepted (OpenAI spec allows null content + when tool_calls is present). + """ + from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + + payload = { + "model": "test-model", + "messages": [ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "List files."}, + { + "role": "assistant", + "content": None, + "reasoning_content": "User wants me to list files.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "read", "arguments": '{"path":"/tmp"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "a.txt"}, + {"role": "user", "content": "thanks"}, + ], + "max_tokens": 16, + } + + request = ChatCompletionRequest(**payload) + assert len(request.messages) == 5 + + def test_reasoning_field_with_null_content(self): + """The 'reasoning' field name (OpenAI SDK style) with content: null. + + Should be accepted since CustomChatCompletionMessageParam allows + extra fields and content: null. + """ + from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + + payload = { + "model": "test-model", + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": None, + "reasoning": "Let me think about this.", + }, + ], + } + + request = ChatCompletionRequest(**payload) + assert len(request.messages) == 2 + + def test_null_content_without_reasoning_fields(self): + """Plain assistant message with content: null and no reasoning fields. + + Valid per OpenAI spec for tool_calls messages. + """ + from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + + payload = { + "model": "test-model", + "messages": [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":"NYC"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "72F sunny"}, + ], + } + + request = ChatCompletionRequest(**payload) + assert len(request.messages) == 3 From 46532472e327e7542601ff5b5cc82d8b9cf78f60 Mon Sep 17 00:00:00 2001 From: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Date: Tue, 26 May 2026 11:32:25 -0700 Subject: [PATCH 037/308] [None][chore] log KV cache utilization and context tokens per iter (#14206) Signed-off-by: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Co-authored-by: Iman Tabrizian <10105175+Tabrizian@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index c6cf937ce755..91484ee96a20 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1104,16 +1104,22 @@ def profile_step(): host_step_time = (end_time - start_time) * 1000 # milliseconds formatted_timestamp = datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + kv_util_str = "N/A" + if self.kv_cache_manager is not None: + kv_stats = self.kv_cache_manager.get_kv_cache_stats() + if kv_stats.max_num_blocks > 0: + kv_util_str = f"{1.0 - kv_stats.free_num_blocks / kv_stats.max_num_blocks:.3f}" logger.info( f"iter = {self.iter_counter}, " f"global_rank = {self.global_rank}, " f"rank = {self.dist.rank}, " + f"num_scheduled_requests = {self.num_scheduled_requests}, " + f"kv_cache_util = {kv_util_str}, " f"currank_total_requests = {self.num_fetch_requests_cur_rank}/" f"{self.num_fetch_requests}, " f"host_step_time = {host_step_time}ms, " f"prev_device_step_time = {prev_device_step_time}, " f"timestamp = {formatted_timestamp}, " - f"num_scheduled_requests: {self.num_scheduled_requests}, " f"states = {self.model_engine.iter_states}") it += 1 @@ -3940,8 +3946,11 @@ def _forward_step( num_accepted_tokens_device: Optional[torch.Tensor] = None): ExpertStatistic.set_iter(self.iter_counter) + num_ctx_tokens = sum(req.context_chunk_size + for req in scheduled_requests.context_requests) + @nvtx_range( - f"[Executor] _forward_step {self.iter_counter}: {scheduled_requests.num_context_requests} ctx reqs, {scheduled_requests.num_generation_requests} gen reqs" + f"[Executor] _forward_step {self.iter_counter}: {scheduled_requests.num_context_requests} ctx reqs, {num_ctx_tokens} ctx tokens, {scheduled_requests.num_generation_requests} gen reqs" ) def forward(scheduled_requests, resource_manager, new_tensors_device, gather_context_logits, cache_indirection_buffer, From ba25b6afae08376ace07205f67be7a069a931f77 Mon Sep 17 00:00:00 2001 From: dongfengy <99041270+dongfengy@users.noreply.github.com> Date: Tue, 26 May 2026 12:37:11 -0700 Subject: [PATCH 038/308] [https://nvbugs/6168859][fix] move tinygemm PDL release after reduction (#14537) Signed-off-by: Dongfeng Yu --- cpp/tensorrt_llm/kernels/tinygemm2/tinygemm2_kernel.cuh | 6 +++++- tests/integration/test_lists/waives.txt | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/tinygemm2/tinygemm2_kernel.cuh b/cpp/tensorrt_llm/kernels/tinygemm2/tinygemm2_kernel.cuh index ca95f6849bc0..4592fe1d110d 100644 --- a/cpp/tensorrt_llm/kernels/tinygemm2/tinygemm2_kernel.cuh +++ b/cpp/tensorrt_llm/kernels/tinygemm2/tinygemm2_kernel.cuh @@ -236,7 +236,6 @@ __global__ __launch_bounds__(384, 1) void tinygemm_kernel(__nv_bfloat16* output, if (!weight_warp) { cudaGridDependencySynchronize(); - cudaTriggerProgrammaticLaunchCompletion(); } for (int ki = 0; ki < K_LOOPS_DMA; ki++) @@ -422,6 +421,11 @@ __global__ __launch_bounds__(384, 1) void tinygemm_kernel(__nv_bfloat16* output, __syncthreads(); + if (threadIdx.x == 0) // one thread per block suffices according to official code examples + { + cudaTriggerProgrammaticLaunchCompletion(); + } + if (warp_id == 0) { diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index a53606807422..b26944b57de3 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -346,7 +346,6 @@ test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https: test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6114608) test_e2e.py::test_multi_nodes_eval[nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-tp16-mmlu] SKIP (https://nvbugs/6114608) test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) -test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] SKIP (https://nvbugs/6168859) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] SKIP (https://nvbugs/6190759) test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] SKIP (https://nvbugs/6115562) From a23207f9ba334ed24593ab5a7508b8227ace6510 Mon Sep 17 00:00:00 2001 From: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> Date: Tue, 26 May 2026 13:46:22 -0700 Subject: [PATCH 039/308] [None][chore] Unwaive `test_cp_tp_broadcast_object` (#14328) Signed-off-by: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index b26944b57de3..54c1ccf11a95 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -397,7 +397,6 @@ unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_t unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens16-_hidden512] SKIP (https://nvbugs/5940460) unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens256-_hidden32] SKIP (https://nvbugs/5940460) unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens256-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py::test_cp_tp_broadcast_object[tp_cp_broadcast-list] SKIP (https://nvbugs/6132301) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingDSv3-swiglu-1024-1024-1] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_qwen_next-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_topk_4-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) From 92e601cd76958e1af8abe52a50eb8fcd53302cf4 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Tue, 26 May 2026 23:44:30 +0100 Subject: [PATCH 040/308] [https://nvbugs/6211185][fix] Fix failed GSM8K accuracy tests for LagunaXS on B200/GB200/B300 (#14580) Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- tensorrt_llm/_torch/models/modeling_laguna.py | 9 ++++++++- tests/integration/test_lists/test-db/l0_b200.yml | 1 + tests/integration/test_lists/waives.txt | 3 --- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_laguna.py b/tensorrt_llm/_torch/models/modeling_laguna.py index 03c82f67a9fb..6d48944a3170 100644 --- a/tensorrt_llm/_torch/models/modeling_laguna.py +++ b/tensorrt_llm/_torch/models/modeling_laguna.py @@ -21,6 +21,7 @@ from torch import nn from transformers import PretrainedConfig +from tensorrt_llm._utils import get_sm_version from tensorrt_llm.functional import PositionEmbeddingType, RotaryScalingType from ..attention_backend import AttentionMetadata @@ -247,6 +248,12 @@ def __init__( self._use_gating = bool(gating) self._gate_per_head = gating == "per-head" or gating is True + # Temporary workaround: Hopper fails without unfused RoPE for Laguna + # While Blackwell has issues when RoPE is unfused. + # This check is to unblock Blackwell on main while we find proper fixes + # https://nvbugs/6211185 + rope_fusion = get_sm_version() in (100, 103) + # fuse_qk_norm_rope=False is required: the fused kernel reads # partial_rotary_factor and yarn params from pretrained_config # globally, ignoring per-layer RopeParams. Laguna has different @@ -259,7 +266,7 @@ def __init__( bias=getattr(config, "qkv_bias", False) or getattr(config, "attention_bias", False), pos_embd_params=pos_embd_params, fuse_qk_norm_rope=False, - rope_fusion=False, + rope_fusion=rope_fusion, layer_idx=layer_idx, dtype=config.torch_dtype, dense_bias=False, diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index bf15e99b4db5..3b04ba55c708 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -21,6 +21,7 @@ l0_b200: - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=True-attn_backend=TRTLLM-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 54c1ccf11a95..923063a16176 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -97,9 +97,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-au accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) -accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_bf16 SKIP (https://nvbugs/6211185) -accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 SKIP (https://nvbugs/6211185) -accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_nvfp4 SKIP (https://nvbugs/6211185) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) From 28718e5ba7571e40bc63768eba7a5a2110e23064 Mon Sep 17 00:00:00 2001 From: Matt Lefebvre Date: Tue, 26 May 2026 16:24:51 -0700 Subject: [PATCH 041/308] [TRTLLMINF-106][infra] Use B300 frontend platforms (#14581) Signed-off-by: Matt Lefebvre --- jenkins/L0_Test.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 69906657c1c2..ecc967d59aca 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3921,10 +3921,10 @@ def launchTestJobs(pipeline, testFilter) "DGX_B200-8_GPUs-PyTorch-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 2, 8, 1, true], "DGX_B200-8_GPUs-AutoDeploy-Post-Merge-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 1, 8, 1, true], "DGX_B200-4_GPUs-Verl-Post-Merge-1": ["auto:dgx-b200-flex", "l0_verl", 1, 1, 4, 1, true], - "B300-PyTorch-1": ["b300-single", "l0_b300", 1, 1], - "DGX_B300-4_GPUs-PyTorch-1": ["b300-x4", "l0_dgx_b300", 1, 1, 4], - "DGX_B300-4_GPUs-PyTorch-Post-Merge-1": ["b300-x4", "l0_dgx_b300", 1, 2, 4], - "DGX_B300-4_GPUs-PyTorch-Post-Merge-2": ["b300-x4", "l0_dgx_b300", 2, 2, 4], + "B300-PyTorch-1": ["auto:dgx-b300-flex", "l0_b300", 1, 1, 1, 1, true], + "DGX_B300-4_GPUs-PyTorch-1": ["auto:dgx-b300-flex", "l0_dgx_b300", 1, 1, 4, 1, true], + "DGX_B300-4_GPUs-PyTorch-Post-Merge-1": ["auto:dgx-b300-flex", "l0_dgx_b300", 1, 2, 4, 1, true], + "DGX_B300-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b300-flex", "l0_dgx_b300", 2, 2, 4, 1, true], // VisualGen PerfSanity post-merge test "DGX_B200-8_GPUs-PyTorch-VisualGen-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_perf_sanity", 1, 1, 8, 1, true], // PerfSanity post-merge tests From c7e7fc5cdc351a7bccdda91a9627d46980372d26 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Wed, 27 May 2026 00:25:27 +0100 Subject: [PATCH 042/308] [None] [refactor] Unify compressed-tensors quant config parsing (#14468) Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 76 +----- tensorrt_llm/llmapi/llm_utils.py | 88 +------ tensorrt_llm/models/quant_config_utils.py | 98 +++++++ .../integration/test_lists/test-db/l0_a10.yml | 1 + .../llmapi/test_kv_cache_dtype_override.py | 60 +++++ .../models/test_quant_config_utils.py | 239 ++++++++++++++++++ 6 files changed, 406 insertions(+), 156 deletions(-) create mode 100644 tensorrt_llm/models/quant_config_utils.py create mode 100644 tests/unittest/models/test_quant_config_utils.py diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 5df3a09a188e..f2a15151d781 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -23,6 +23,8 @@ from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.models.quant_config_utils import \ + update_quant_config_from_compressed_tensors from tensorrt_llm.quantization.mode import QuantAlgo from tensorrt_llm.quantization.modelopt_config import ( is_modelopt_quant_config, read_modelopt_quant_config, @@ -477,78 +479,8 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None): # NOTE: This is for llm-compressor's quantized checkpoints. elif hf_quant_config.get("quant_method") == "compressed-tensors": - config_groups = hf_quant_config.get("config_groups") - if config_groups is None: - raise ValueError( - f"config_groups is not set in {hf_quant_config}.") - - weights_quant_config = config_groups["group_0"]["weights"] - inputs_quant_config = config_groups["group_0"]["input_activations"] - weights_quant_strategy = weights_quant_config["strategy"] - inputs_quant_strategy = inputs_quant_config["strategy"] - - if weights_quant_config["num_bits"] == 8: - if weights_quant_strategy == "channel": - if inputs_quant_strategy != "token": - raise ValueError( - f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}." - ) - quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN - elif weights_quant_strategy == "block": - if inputs_quant_strategy != "group": - raise ValueError( - f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}." - ) - quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES - group_size = inputs_quant_config["group_size"] - - # NOTE: TRT-LLM only supports group_size=128 for FP8_BLOCK_SCALES. - if group_size != 128: - raise ValueError( - f"Unsupported group_size: {group_size}. Supported: 128." - ) - quant_config.group_size = group_size - - else: - raise ValueError( - f"Unsupported weights_quant_strategy: {weights_quant_strategy}. " - "Supported strategies: 'channel', 'block'.") - elif (weights_quant_config["num_bits"] == 4 - and weights_quant_config.get("type") == "float" - and weights_quant_strategy == "tensor_group"): - # llm-compressor NVFP4: weights FP4 with FP8 per-group scales - # (group_size=16), scaled by an FP32 global scale. - if inputs_quant_strategy != "tensor_group": - raise ValueError( - f"Unsupported inputs_quant_strategy for NVFP4: {inputs_quant_strategy}." - ) - group_size = weights_quant_config["group_size"] - if group_size != 16: - raise ValueError( - f"Unsupported group_size: {group_size}. Supported: 16 for NVFP4." - ) - quant_config.quant_algo = QuantAlgo.NVFP4 - quant_config.group_size = group_size - else: - raise ValueError( - f"Unsupported quant_bits: {weights_quant_config['num_bits']}. " - "Supported: 8 (FP8) or 4 (NVFP4).") - - # kv_cache_scheme (llm-compressor): FP8 per-tensor KV cache. - kv_cache_scheme = hf_quant_config.get("kv_cache_scheme") - if kv_cache_scheme is not None: - if (kv_cache_scheme.get("num_bits") == 8 - and kv_cache_scheme.get("type") == "float"): - quant_config.kv_cache_quant_algo = QuantAlgo.FP8 - else: - raise ValueError( - f"Unsupported kv_cache_scheme: {kv_cache_scheme}.") - - if hf_exclude_modules is not None: - quant_config.exclude_modules = list( - set(hf_exclude_modules + hf_quant_config.get("ignore", []))) - else: - quant_config.exclude_modules = hf_quant_config.get("ignore", []) + update_quant_config_from_compressed_tensors(quant_config, + hf_quant_config) elif hf_quant_config.get("quant_method") == "nvfp4": quant_config.quant_algo = QuantAlgo.NVFP4 group_size = hf_quant_config.get("group_size", 16) diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index 3984722184e7..89bfdab0e60d 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -26,6 +26,8 @@ from ..mapping import Mapping from ..models.automodel import MODEL_MAP, AutoConfig, AutoModelForCausalLM from ..models.modeling_utils import PretrainedConfig, QuantAlgo, QuantConfig +from ..models.quant_config_utils import \ + update_quant_config_from_compressed_tensors from ..module import Module from ..quantization.modelopt_config import (is_modelopt_quant_config, read_modelopt_quant_config, @@ -470,90 +472,8 @@ def _update_from_hf_quant_config(self) -> bool: ] # NOTE: This is for llm-compressor's quantized checkpoints. elif hf_quant_config.get("quant_method") == "compressed-tensors": - config_groups = hf_quant_config.get("config_groups") - if config_groups is None: - raise ValueError( - f"config_groups is not set in {hf_quant_config}.") - - weights_quant_config = config_groups["group_0"]["weights"] - inputs_quant_config = config_groups["group_0"][ - "input_activations"] - weights_quant_strategy = weights_quant_config["strategy"] - inputs_quant_strategy = inputs_quant_config["strategy"] - - if weights_quant_config["num_bits"] == 8: - if weights_quant_strategy == "channel": - if inputs_quant_strategy != "token": - raise ValueError( - f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}." - ) - quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN - elif weights_quant_strategy == "block": - if inputs_quant_strategy != "group": - raise ValueError( - f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}." - ) - quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES - group_size = inputs_quant_config["group_size"] - - # NOTE: TRT-LLM only supports group_size=128 for FP8_BLOCK_SCALES. - if group_size != 128: - raise ValueError( - f"Unsupported group_size: {group_size}. Supported: 128." - ) - quant_config.group_size = group_size - - else: - raise ValueError( - f"Unsupported weights_quant_strategy: {weights_quant_strategy}. " - "Supported strategies: 'channel', 'block'.") - elif (weights_quant_config["num_bits"] == 4 - and weights_quant_config.get("type") == "float" - and weights_quant_strategy == "tensor_group"): - # llm-compressor NVFP4: weights FP4 with FP8 per-group - # scales (group_size=16), scaled by an FP32 global scale. - if inputs_quant_strategy != "tensor_group": - raise ValueError( - f"Unsupported inputs_quant_strategy for NVFP4: {inputs_quant_strategy}." - ) - group_size = weights_quant_config["group_size"] - if group_size != 16: - raise ValueError( - f"Unsupported group_size: {group_size}. Supported: 16 for NVFP4." - ) - quant_config.quant_algo = QuantAlgo.NVFP4 - quant_config.group_size = group_size - else: - raise ValueError( - f"Unsupported quant_bits: {weights_quant_config['num_bits']}. " - "Supported: 8 (FP8) or 4 (NVFP4).") - - # kv_cache_scheme (llm-compressor): FP8 per-tensor KV cache. - kv_cache_scheme = hf_quant_config.get("kv_cache_scheme") - if kv_cache_scheme is not None: - if (kv_cache_scheme.get("num_bits") == 8 - and kv_cache_scheme.get("type") == "float"): - if quant_config.kv_cache_quant_algo in (None, - QuantAlgo.FP8): - quant_config.kv_cache_quant_algo = QuantAlgo.FP8 - else: - raise ValueError( - f"Specified kv_cache_quant_algo={quant_config.kv_cache_quant_algo}, " - f"conflicting with FP8 KV cache from HF quant config." - ) - else: - raise ValueError( - f"Unsupported kv_cache_scheme: {kv_cache_scheme}.") - - hf_exclude_modules = hf_quant_config.get( - "modules_to_not_convert", None) - if hf_exclude_modules is not None: - quant_config.exclude_modules = list( - set(hf_exclude_modules + - hf_quant_config.get("ignore", []))) - else: - quant_config.exclude_modules = hf_quant_config.get( - "ignore", []) + update_quant_config_from_compressed_tensors( + quant_config, hf_quant_config) elif hf_quant_config.get("quant_method") == "nvfp4": quant_config.quant_algo = QuantAlgo.NVFP4 group_size = hf_quant_config.get("group_size", 16) diff --git a/tensorrt_llm/models/quant_config_utils.py b/tensorrt_llm/models/quant_config_utils.py new file mode 100644 index 000000000000..ffebe91ec786 --- /dev/null +++ b/tensorrt_llm/models/quant_config_utils.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Mapping + +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + + +def update_quant_config_from_compressed_tensors( + quant_config: QuantConfig, hf_quant_config: Mapping[str, Any] +) -> None: + """Mutate QuantConfig from an llm-compressor compressed-tensors config.""" + config_groups = hf_quant_config.get("config_groups") + if config_groups is None: + raise ValueError(f"config_groups is not set in {hf_quant_config}.") + + weights_quant_config = config_groups["group_0"]["weights"] + inputs_quant_config = config_groups["group_0"]["input_activations"] + weights_quant_strategy = weights_quant_config["strategy"] + inputs_quant_strategy = inputs_quant_config["strategy"] + + if weights_quant_config["num_bits"] == 8: + if weights_quant_strategy == "channel": + if inputs_quant_strategy != "token": + raise ValueError(f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}.") + quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN + elif weights_quant_strategy == "block": + if inputs_quant_strategy != "group": + raise ValueError(f"Unsupported inputs_quant_strategy: {inputs_quant_strategy}.") + quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES + group_size = inputs_quant_config["group_size"] + + # TRT-LLM only supports group_size=128 for FP8_BLOCK_SCALES. + if group_size != 128: + raise ValueError(f"Unsupported group_size: {group_size}. Supported: 128.") + quant_config.group_size = group_size + + else: + raise ValueError( + f"Unsupported weights_quant_strategy: {weights_quant_strategy}. " + "Supported strategies: 'channel', 'block'." + ) + elif ( + weights_quant_config["num_bits"] == 4 + and weights_quant_config.get("type") == "float" + and weights_quant_strategy == "tensor_group" + ): + # llm-compressor NVFP4: weights FP4 with FP8 per-group scales + # (group_size=16), scaled by an FP32 global scale. + if inputs_quant_strategy != "tensor_group": + raise ValueError( + f"Unsupported inputs_quant_strategy for NVFP4: {inputs_quant_strategy}." + ) + group_size = weights_quant_config["group_size"] + if group_size != 16: + raise ValueError(f"Unsupported group_size: {group_size}. Supported: 16 for NVFP4.") + quant_config.quant_algo = QuantAlgo.NVFP4 + quant_config.group_size = group_size + else: + raise ValueError( + f"Unsupported quant_bits: {weights_quant_config['num_bits']}. " + "Supported: 8 (FP8) or 4 (NVFP4)." + ) + + # kv_cache_scheme (llm-compressor): FP8 per-tensor KV cache. + kv_cache_scheme = hf_quant_config.get("kv_cache_scheme") + if kv_cache_scheme is not None: + if kv_cache_scheme.get("num_bits") == 8 and kv_cache_scheme.get("type") == "float": + if quant_config.kv_cache_quant_algo in (None, QuantAlgo.FP8): + quant_config.kv_cache_quant_algo = QuantAlgo.FP8 + else: + raise ValueError( + f"Specified kv_cache_quant_algo={quant_config.kv_cache_quant_algo}, " + "conflicting with FP8 KV cache from HF quant config." + ) + else: + raise ValueError(f"Unsupported kv_cache_scheme: {kv_cache_scheme}.") + + hf_exclude_modules = hf_quant_config.get("modules_to_not_convert", None) + if hf_exclude_modules is not None: + quant_config.exclude_modules = list( + set(hf_exclude_modules + hf_quant_config.get("ignore", [])) + ) + else: + quant_config.exclude_modules = hf_quant_config.get("ignore", []) diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 9a4867492fd6..0b3cceab299f 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -19,6 +19,7 @@ l0_a10: - unittest/utils/test_util.py - unittest/utils/test_logger.py - unittest/_torch/test_model_config.py + - unittest/models/test_quant_config_utils.py - unittest/_torch/modeling/test_modeling_mistral.py - unittest/_torch/modeling/test_modeling_pixtral.py - unittest/_torch/modeling/test_modeling_cohere2.py diff --git a/tests/unittest/llmapi/test_kv_cache_dtype_override.py b/tests/unittest/llmapi/test_kv_cache_dtype_override.py index 7c1741cdcbb1..b5fed337c769 100644 --- a/tests/unittest/llmapi/test_kv_cache_dtype_override.py +++ b/tests/unittest/llmapi/test_kv_cache_dtype_override.py @@ -22,6 +22,27 @@ def _write_hf_quant_config(model_dir, kv_cache_quant_algo: str = "FP8"): ) +def _compressed_tensors_nvfp4_config(**overrides): + config = { + "quant_method": "compressed-tensors", + "config_groups": { + "group_0": { + "weights": { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + }, + "input_activations": { + "strategy": "tensor_group", + }, + }, + }, + } + config.update(overrides) + return config + + def test_get_llm_args_plumbs_kv_cache_dtype(): llm_args, _ = get_llm_args(model="dummy", kv_cache_dtype="nvfp4") assert llm_args["kv_cache_config"].dtype == "nvfp4" @@ -65,3 +86,42 @@ def test_update_from_hf_quant_config_explicit_dtype_overrides(tmp_path): assert model_loader._update_from_hf_quant_config() is True assert llm_args.quant_config.kv_cache_quant_algo == QuantAlgo.NVFP4 + + +def test_update_from_hf_quant_config_parses_compressed_tensors_model_kwargs(tmp_path): + llm_args = TorchLlmArgs( + model=str(tmp_path), + model_kwargs={ + "quantization_config": _compressed_tensors_nvfp4_config( + kv_cache_scheme={ + "num_bits": 8, + "type": "float", + } + ), + }, + ) + model_loader = ModelLoader(llm_args) + + assert model_loader._update_from_hf_quant_config() is True + assert llm_args.quant_config.quant_algo == QuantAlgo.NVFP4 + assert llm_args.quant_config.group_size == 16 + assert llm_args.quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + + +def test_update_from_hf_quant_config_rejects_compressed_tensors_kv_conflict(tmp_path): + llm_args = TorchLlmArgs( + model=str(tmp_path), + model_kwargs={ + "quantization_config": _compressed_tensors_nvfp4_config( + kv_cache_scheme={ + "num_bits": 8, + "type": "float", + } + ), + }, + ) + llm_args.quant_config = QuantConfig(kv_cache_quant_algo=QuantAlgo.NVFP4) + model_loader = ModelLoader(llm_args) + + with pytest.raises(ValueError, match="conflicting with FP8 KV cache"): + model_loader._update_from_hf_quant_config() diff --git a/tests/unittest/models/test_quant_config_utils.py b/tests/unittest/models/test_quant_config_utils.py new file mode 100644 index 000000000000..9b2860f58114 --- /dev/null +++ b/tests/unittest/models/test_quant_config_utils.py @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.models.quant_config_utils import update_quant_config_from_compressed_tensors +from tensorrt_llm.quantization.mode import QuantAlgo + + +def _compressed_tensors_config(weights=None, input_activations=None, **overrides): + config = { + "quant_method": "compressed-tensors", + "config_groups": { + "group_0": { + "weights": weights + or { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + }, + "input_activations": input_activations + or { + "strategy": "tensor_group", + }, + }, + }, + } + config.update(overrides) + return config + + +def test_update_quant_config_from_compressed_tensors_parses_nvfp4(): + gate_exclude = "re:model\\.layers\\.\\d+\\.mlp\\.gate" + quant_config = QuantConfig() + update_quant_config_from_compressed_tensors( + quant_config, + _compressed_tensors_config( + kv_cache_scheme={ + "num_bits": 8, + "type": "float", + }, + modules_to_not_convert=[gate_exclude], + ignore=["lm_head"], + ), + ) + + assert quant_config.quant_algo == QuantAlgo.NVFP4 + assert quant_config.group_size == 16 + assert quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + assert set(quant_config.exclude_modules) == {gate_exclude, "lm_head"} + + +def test_update_quant_config_from_compressed_tensors_parses_fp8_block_scales(): + quant_config = QuantConfig() + update_quant_config_from_compressed_tensors( + quant_config, + _compressed_tensors_config( + weights={ + "num_bits": 8, + "strategy": "block", + }, + input_activations={ + "num_bits": 8, + "strategy": "group", + "group_size": 128, + }, + ignore=["lm_head"], + ), + ) + + assert quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES + assert quant_config.group_size == 128 + assert quant_config.exclude_modules == ["lm_head"] + + +def test_update_quant_config_from_compressed_tensors_parses_fp8_channel(): + quant_config = QuantConfig() + update_quant_config_from_compressed_tensors( + quant_config, + _compressed_tensors_config( + weights={ + "num_bits": 8, + "strategy": "channel", + }, + input_activations={ + "num_bits": 8, + "strategy": "token", + }, + ignore=["lm_head"], + ), + ) + + assert quant_config.quant_algo == QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN + assert quant_config.exclude_modules == ["lm_head"] + + +@pytest.mark.parametrize( + "weights,input_activations,error_match", + [ + ( + { + "num_bits": 8, + "strategy": "block", + }, + { + "num_bits": 8, + "strategy": "group", + "group_size": 64, + }, + "Supported: 128", + ), + ( + { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 32, + }, + { + "strategy": "tensor_group", + }, + "Supported: 16 for NVFP4", + ), + ], +) +def test_update_quant_config_from_compressed_tensors_rejects_group_sizes( + weights, input_activations, error_match +): + with pytest.raises(ValueError, match=error_match): + update_quant_config_from_compressed_tensors( + QuantConfig(), + _compressed_tensors_config(weights=weights, input_activations=input_activations), + ) + + +@pytest.mark.parametrize( + "weights,input_activations,error_match", + [ + ( + { + "num_bits": 8, + "strategy": "tensor", + }, + { + "num_bits": 8, + "strategy": "token", + }, + "Unsupported weights_quant_strategy", + ), + ( + { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + }, + { + "strategy": "token", + }, + "Unsupported inputs_quant_strategy for NVFP4", + ), + ], +) +def test_update_quant_config_from_compressed_tensors_rejects_strategies( + weights, input_activations, error_match +): + with pytest.raises(ValueError, match=error_match): + update_quant_config_from_compressed_tensors( + QuantConfig(), + _compressed_tensors_config(weights=weights, input_activations=input_activations), + ) + + +def test_update_quant_config_from_compressed_tensors_requires_config_groups(): + with pytest.raises(ValueError, match="config_groups is not set"): + update_quant_config_from_compressed_tensors( + QuantConfig(), + { + "quant_method": "compressed-tensors", + }, + ) + + +def test_update_quant_config_from_compressed_tensors_rejects_kv_cache_scheme(): + with pytest.raises(ValueError, match="Unsupported kv_cache_scheme"): + update_quant_config_from_compressed_tensors( + QuantConfig(), + _compressed_tensors_config( + kv_cache_scheme={ + "num_bits": 4, + "type": "float", + } + ), + ) + + +def test_update_quant_config_from_compressed_tensors_rejects_weight_num_bits(): + with pytest.raises(ValueError, match="Unsupported quant_bits"): + update_quant_config_from_compressed_tensors( + QuantConfig(), + _compressed_tensors_config( + weights={ + "num_bits": 3, + "strategy": "block", + }, + input_activations={ + "num_bits": 8, + "strategy": "group", + "group_size": 128, + }, + ), + ) + + +def test_update_quant_config_from_compressed_tensors_rejects_kv_cache_conflict(): + with pytest.raises(ValueError, match="conflicting with FP8 KV cache"): + update_quant_config_from_compressed_tensors( + QuantConfig(kv_cache_quant_algo=QuantAlgo.NVFP4), + _compressed_tensors_config( + kv_cache_scheme={ + "num_bits": 8, + "type": "float", + } + ), + ) From 5dd96d6c5f0030b09edecfc6f2f60b6243d1d224 Mon Sep 17 00:00:00 2001 From: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Date: Tue, 26 May 2026 19:39:47 -0700 Subject: [PATCH 043/308] [None][feat] AutoDeploy push the rope buffer to later stage (#13859) Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- .../_torch/auto_deploy/config/default.yaml | 10 +- .../mlir/fusion/subgraph_replace.py | 4 +- .../_torch/auto_deploy/mlir/mlir_to_fx.py | 33 +++- .../models/custom/modeling_deepseek.py | 28 +--- .../models/custom/modeling_gemma3n.py | 76 ++++++---- .../models/custom/modeling_gemma4.py | 32 ++-- .../models/custom/modeling_glm4_moe_lite.py | 32 +--- .../models/custom/modeling_kimi_k2.py | 25 +-- .../models/custom/modeling_llama3_ir.py | 23 ++- .../models/custom/modeling_llama4.py | 24 ++- .../models/custom/modeling_minimax_m2.py | 19 ++- .../models/custom/modeling_mistral3.py | 22 +-- .../models/custom/modeling_qwen3.py | 25 +-- .../models/custom/modeling_qwen3_moe.py | 25 +-- .../models/custom/modeling_starcoder2.py | 20 +-- .../auto_deploy/models/custom/rotary_utils.py | 73 +++++++++ .../transform/library/fuse_rope_mla.py | 21 +-- .../test_lists/test-db/l0_dgx_b200.yml | 3 +- .../mlir/test_elementwise_fusion_e2e.py | 48 ++++++ .../singlegpu/mlir/test_fx_mlir_roundtrip.py | 129 ++++++++++++++++ .../singlegpu/models/test_deepseek_custom.py | 45 ++++++ .../singlegpu/models/test_mla_rope_utils.py | 78 ++++++++++ .../library/test_rope_transformation.py | 142 +++++++++++++++++- 23 files changed, 697 insertions(+), 240 deletions(-) create mode 100644 tensorrt_llm/_torch/auto_deploy/models/custom/rotary_utils.py diff --git a/tensorrt_llm/_torch/auto_deploy/config/default.yaml b/tensorrt_llm/_torch/auto_deploy/config/default.yaml index 166ff6b965f4..337a9f4ef85f 100644 --- a/tensorrt_llm/_torch/auto_deploy/config/default.yaml +++ b/tensorrt_llm/_torch/auto_deploy/config/default.yaml @@ -235,11 +235,6 @@ transforms: fuse_rmsnorm_quant_fp8: stage: post_load_fusion enabled: false - mlir_elementwise_fusion: - stage: post_load_fusion - enabled: false - run_shape_prop: false - bypass_ops: [] gather_logits_before_lm_head: stage: post_load_fusion # TODO: fix https://github.com/NVIDIA/TensorRT-LLM/issues/9878 to enable by default @@ -257,6 +252,11 @@ transforms: enabled: false optimize_rope: stage: post_load_fusion + mlir_elementwise_fusion: + stage: post_load_fusion + enabled: false + run_shape_prop: false + bypass_ops: [] ############################################################################################ # VISUALIZE GRAPH ############################################################################################ diff --git a/tensorrt_llm/_torch/auto_deploy/mlir/fusion/subgraph_replace.py b/tensorrt_llm/_torch/auto_deploy/mlir/fusion/subgraph_replace.py index d5004289203a..73de63e3bf00 100644 --- a/tensorrt_llm/_torch/auto_deploy/mlir/fusion/subgraph_replace.py +++ b/tensorrt_llm/_torch/auto_deploy/mlir/fusion/subgraph_replace.py @@ -66,7 +66,9 @@ def replace_subgraph_with_fused_op( shape = [s if s >= 0 else 2 for s in out.type.get_shape()] dtype = mlir_to_torch_dtype(out.type.element_type) fake_vals.append(torch.empty(shape, dtype=dtype, device="meta")) - val_meta = tuple(fake_vals) if len(fake_vals) > 1 else fake_vals[0] if fake_vals else None + # Generated fused kernels return a tuple even for a single output, so keep + # metadata in the same structure for MLIR -> FX getitem propagation. + val_meta = tuple(fake_vals) if fake_vals else None # Store synthetic metadata for the MLIR→FX converter metadata[node_key] = { diff --git a/tensorrt_llm/_torch/auto_deploy/mlir/mlir_to_fx.py b/tensorrt_llm/_torch/auto_deploy/mlir/mlir_to_fx.py index 81f86389b84a..08615bc70796 100644 --- a/tensorrt_llm/_torch/auto_deploy/mlir/mlir_to_fx.py +++ b/tensorrt_llm/_torch/auto_deploy/mlir/mlir_to_fx.py @@ -25,7 +25,7 @@ import torch from torch.fx import Graph, GraphModule, Node -from xdsl.dialects.builtin import ModuleOp +from xdsl.dialects.builtin import ModuleOp, TensorType from xdsl.ir import Operation, SSAValue from .dialect import ( @@ -512,14 +512,24 @@ def _restore_meta(node: Node, key: str, metadata: Dict[str, Dict[str, Any]]) -> def _restore_meta_from_op( node: Node, mlir_op: Operation, metadata: Dict[str, Dict[str, Any]] ) -> None: - """No-op: metadata for precisely-lowered ops is recomputed by shape propagation. + """Restore metadata for precisely-lowered ops from MLIR result types. Precisely-lowered ops (ad.add, ad.mul, etc.) don't carry a node_key back to the original FX metadata side-table, so there is no key to look up. - The transform framework runs shape_prop after MLIR→FX conversion, which - repopulates node.meta["val"] for all nodes. + Most runs recover metadata through shape propagation, but some AutoDeploy + graph partitions do not have fake input tensors available. In that case, + downstream transforms still require shape and dtype metadata from + node.meta["val"]. """ - pass + del metadata + + output = getattr(mlir_op, "output", None) + if output is None: + return + + fake_val = _fake_tensor_from_mlir_type(output.type) + if fake_val is not None: + node.meta["val"] = fake_val @staticmethod def _resolve_target(op_name: str) -> Optional[Callable]: @@ -543,3 +553,16 @@ def _opaque_placeholder(*args, **kwargs): "Opaque op placeholder was called. This indicates an MLIR → FX " "conversion issue where the original op target could not be resolved." ) + + +def _fake_tensor_from_mlir_type(result_type: Any) -> torch.Tensor | None: + """Build minimal tensor metadata from an MLIR tensor result type.""" + if not isinstance(result_type, TensorType): + return None + + shape = [s if s >= 0 else 2 for s in result_type.get_shape()] + try: + dtype = mlir_to_torch_dtype(result_type.element_type) + except ValueError: + return None + return torch.empty(shape, dtype=dtype, device="meta") diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py index 1bf0c7d5ca78..39614903f11e 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py @@ -71,6 +71,7 @@ from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory from . import mla_rope_utils +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class DeepSeekV3RMSNorm(nn.Module): @@ -87,11 +88,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ).to(hidden_states.dtype) -class DeepSeekV3RotaryEmbedding(nn.Module): +class DeepSeekV3RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for DeepSeekV3. - Simplified version that precomputes and caches cos/sin values. - Returns full cached values (not sliced by seq_len) to enable export. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__(self, dim: int, max_position_embeddings: int = 2048, base: float = 10000.0): @@ -102,25 +103,18 @@ def __init__(self, dim: int, max_position_embeddings: int = 2048, base: float = inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) + self.attention_scaling = 1.0 - # Build cos/sin cache self._set_cos_sin_cache(max_position_embeddings) def _set_cos_sin_cache(self, seq_len: int): self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos(), persistent=False) - self.register_buffer("sin_cached", emb.sin(), persistent=False) def forward( self, x: torch.Tensor, seq_len: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: - # Return full cached cos/sin (not sliced) for export compatibility - return ( - self.cos_cached.to(dtype=x.dtype, device=x.device), - self.sin_cached.to(dtype=x.dtype, device=x.device), + return build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling ) @@ -167,17 +161,11 @@ def _set_cos_sin_cache(self, seq_len: int): inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask self.register_buffer("inv_freq", inv_freq, persistent=False) - t = torch.arange(seq_len, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - _mscale = float( self._yarn_get_mscale(self.scaling_factor, self.mscale) / self._yarn_get_mscale(self.scaling_factor, self.mscale_all_dim) ) - - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", (emb.cos() * _mscale), persistent=False) - self.register_buffer("sin_cached", (emb.sin() * _mscale), persistent=False) + self.attention_scaling = _mscale @staticmethod def _yarn_find_correction_dim( diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma3n.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma3n.py index 6e1ddd1e7a51..7201694c65dc 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma3n.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma3n.py @@ -36,6 +36,7 @@ from transformers import AutoTokenizer from transformers.activations import ACT2FN from transformers.generation import GenerationMixin +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS from transformers.modeling_utils import PreTrainedModel from transformers.models.gemma3n.configuration_gemma3n import ( Gemma3nAudioConfig, @@ -47,6 +48,7 @@ from ..factory import ModelFactoryRegistry from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache def _get_rope_theta(config: Gemma3nTextConfig, layer_type: str = "full_attention") -> float: @@ -57,21 +59,22 @@ def _get_rope_theta(config: Gemma3nTextConfig, layer_type: str = "full_attention ``config.rope_parameters[layer_type]["rope_theta"]`` or ``config.default_theta``. """ - # Explicit attribute (set e.g. by deepcopy + manual override) - if hasattr(config, "rope_theta") and config.rope_theta is not None: - return float(config.rope_theta) - - # transformers 5.x: per-layer-type parameters rope_params = getattr(config, "rope_parameters", None) if isinstance(rope_params, dict) and layer_type in rope_params: theta = rope_params[layer_type].get("rope_theta") if theta is not None: return float(theta) - # Fallback: default_theta dict + is_local_layer = "sliding" in layer_type or "local" in layer_type + if is_local_layer and hasattr(config, "rope_local_base_freq"): + return float(config.rope_local_base_freq) + + if hasattr(config, "rope_theta") and config.rope_theta is not None: + return float(config.rope_theta) + default_theta = getattr(config, "default_theta", None) if isinstance(default_theta, dict): - key = "local" if "sliding" in layer_type or "local" in layer_type else "global" + key = "local" if is_local_layer else "global" return float(default_theta.get(key, 10000.0)) if default_theta is not None: return float(default_theta) @@ -79,24 +82,39 @@ def _get_rope_theta(config: Gemma3nTextConfig, layer_type: str = "full_attention return 10000.0 -def _build_rope_cache( +def _get_rope_type(config: Gemma3nTextConfig, layer_type: str = "full_attention") -> str: + rope_params = getattr(config, "rope_parameters", None) + if isinstance(rope_params, dict): + layer_params = rope_params.get(layer_type) + if isinstance(layer_params, dict): + return layer_params.get("rope_type", layer_params.get("type", "default")) + return rope_params.get("rope_type", rope_params.get("type", "default")) + + rope_scaling = getattr(config, "rope_scaling", None) + if isinstance(rope_scaling, dict): + return rope_scaling.get("rope_type", rope_scaling.get("type", "default")) + + return "default" + + +def _build_rope_inv_freq( config: Gemma3nTextConfig, layer_type: str = "full_attention", -) -> Tuple[torch.Tensor, torch.Tensor, float]: - """Build RoPE cos/sin cache with default (no-scaling) computation. +) -> tuple[torch.Tensor, float]: + rope_type = _get_rope_type(config, layer_type) + if rope_type == "default": + head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + base = _get_rope_theta(config, layer_type) + inv_freq = 1.0 / (base ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)) + return inv_freq, 1.0 - In transformers 5.x the ``"default"`` key was removed from - ``ROPE_INIT_FUNCTIONS``, so we inline the standard computation. - """ - head_dim = config.head_dim - base = _get_rope_theta(config, layer_type) - inv_freq = 1.0 / (base ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)) - attention_scaling = 1.0 - - positions = torch.arange(config.max_position_embeddings, dtype=inv_freq.dtype) - freqs = torch.outer(positions, inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - return emb.cos(), emb.sin(), attention_scaling + try: + inv_freq, attention_scaling = ROPE_INIT_FUNCTIONS[rope_type]( + config, device=None, layer_type=layer_type + ) + except TypeError: + inv_freq, attention_scaling = ROPE_INIT_FUNCTIONS[rope_type](config, device=None) + return inv_freq, attention_scaling class Gemma3nRMSNorm(nn.Module): @@ -227,20 +245,20 @@ def scale_corrected_output(self, corrected: torch.Tensor) -> torch.Tensor: ) -class Gemma3nTextRotaryEmbedding(nn.Module): +class Gemma3nTextRotaryEmbedding(RotaryEmbeddingBase): def __init__(self, config: Gemma3nTextConfig, layer_type: str = "full_attention"): super().__init__() - cos, sin, attention_scaling = _build_rope_cache(config, layer_type=layer_type) - self.register_buffer("_ad_cos_cached", cos * attention_scaling, persistent=False) - self.register_buffer("_ad_sin_cached", sin * attention_scaling, persistent=False) + inv_freq, self.attention_scaling = _build_rope_inv_freq(config, layer_type=layer_type) + self.max_position_embeddings = config.max_position_embeddings + self.register_buffer("inv_freq", inv_freq, persistent=False) def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: del position_ids - cos = self._ad_cos_cached.to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached.to(dtype=x.dtype, device=x.device) - return cos, sin + return build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling + ) def _slice_rope_cache( diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py index b8e56d42c107..43e9ee396fea 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py @@ -78,6 +78,7 @@ AutoModelForImageTextToTextFactory, TextModelExportInfo, ) +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache # --------------------------------------------------------------------------- # Multimodal semantic mask custom ops — prefill-only and cached variants. @@ -448,11 +449,11 @@ def __init__( # --------------------------------------------------------------------------- -def _build_rope_cache( +def _build_rope_inv_freq( config: Gemma4TextConfig, layer_type: str, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Pre-compute cos/sin RoPE cache for the given layer type.""" +) -> tuple[torch.Tensor, float]: + """Build RoPE inverse frequencies for the given layer type.""" rope_params = config.rope_parameters[layer_type] rope_type = rope_params.get("rope_type", "default") base = rope_params["rope_theta"] @@ -485,10 +486,7 @@ def _build_rope_cache( rope_init_fn = ROPE_INIT_FUNCTIONS[rope_type] inv_freq, attention_scaling = rope_init_fn(config, device=None, layer_type=layer_type) - positions = torch.arange(config.max_position_embeddings, dtype=inv_freq.dtype) - freqs = torch.outer(positions, inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - return emb.cos() * attention_scaling, emb.sin() * attention_scaling + return inv_freq, attention_scaling # --------------------------------------------------------------------------- @@ -538,24 +536,22 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.linear(hidden_states) -class Gemma4RotaryEmbedding(nn.Module): - """Pre-computed RoPE cache for a single layer type (global or local).""" +class Gemma4RotaryEmbedding(RotaryEmbeddingBase): + """RoPE for a single layer type (global or local).""" def __init__(self, config: Gemma4TextConfig, layer_type: str): super().__init__() - ( - cos, - sin, - ) = _build_rope_cache(config, layer_type) - self.register_buffer("_ad_cos_cached", cos, persistent=False) - self.register_buffer("_ad_sin_cached", sin, persistent=False) + inv_freq, self.attention_scaling = _build_rope_inv_freq(config, layer_type) + self.max_position_embeddings = config.max_position_embeddings + self.register_buffer("inv_freq", inv_freq, persistent=False) def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - cos = self._ad_cos_cached[position_ids].to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached[position_ids].to(dtype=x.dtype, device=x.device) - return cos, sin + cos, sin = build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling + ) + return cos[position_ids], sin[position_ids] # --------------------------------------------------------------------------- diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_glm4_moe_lite.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_glm4_moe_lite.py index 2f73d479b36c..869cb4468d6a 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_glm4_moe_lite.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_glm4_moe_lite.py @@ -38,6 +38,7 @@ from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory from . import mla_rope_utils +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class Glm4MoeLiteConfig(PretrainedConfig): @@ -165,13 +166,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.weight * hidden_states.to(input_dtype) -class Glm4MoeLiteRotaryEmbedding(nn.Module): +class Glm4MoeLiteRotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for GLM4 MoE Lite. - Simplified version that precomputes and caches cos/sin values. - Returns full cached values (not sliced by seq_len) to enable export. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__( @@ -190,25 +189,16 @@ def __init__( inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) - # Build cos/sin cache with AD-specific naming self._set_cos_sin_cache(max_position_embeddings) def _set_cos_sin_cache(self, seq_len: int): self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - # Use _ad_ prefix for AutoDeploy compatibility with lift_to_meta - self.register_buffer("_ad_cos_cached", emb.cos() * self.attention_scaling, persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin() * self.attention_scaling, persistent=False) def forward( self, x: torch.Tensor, seq_len: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: - # Return full cached cos/sin (not sliced) for export compatibility - return ( - self._ad_cos_cached.to(dtype=x.dtype, device=x.device), - self._ad_sin_cached.to(dtype=x.dtype, device=x.device), + return build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling ) @@ -256,19 +246,11 @@ def _set_cos_sin_cache(self, seq_len: int): inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask self.register_buffer("inv_freq", inv_freq, persistent=False) - t = torch.arange(seq_len, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - _mscale = float( self._yarn_get_mscale(self.scaling_factor, self.mscale) / self._yarn_get_mscale(self.scaling_factor, self.mscale_all_dim) ) - - emb = torch.cat((freqs, freqs), dim=-1) - # Use _ad_ prefix for AutoDeploy compatibility with lift_to_meta - # Note: attention_scaling is already incorporated in _mscale for YaRN - self.register_buffer("_ad_cos_cached", (emb.cos() * _mscale), persistent=False) - self.register_buffer("_ad_sin_cached", (emb.sin() * _mscale), persistent=False) + self.attention_scaling = _mscale @staticmethod def _yarn_find_correction_dim( diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_kimi_k2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_kimi_k2.py index dcbf869c4d4d..8da79b802dd0 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_kimi_k2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_kimi_k2.py @@ -45,6 +45,7 @@ from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache # ============================================================================= # Configuration @@ -222,11 +223,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.weight * hidden_states.to(input_dtype) -class KimiK2RotaryEmbedding(nn.Module): +class KimiK2RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Kimi-K2. - Returns full cached cos/sin (not sliced by seq_len) to enable export. - Uses _ad_ prefix for buffer names for AutoDeploy lift_to_meta compatibility. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__( @@ -249,18 +250,12 @@ def __init__( def _set_cos_sin_cache(self, seq_len: int): self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", emb.cos() * self.attention_scaling, persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin() * self.attention_scaling, persistent=False) def forward( self, x: torch.Tensor, seq_len: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: - return ( - self._ad_cos_cached.to(dtype=x.dtype, device=x.device), - self._ad_sin_cached.to(dtype=x.dtype, device=x.device), + return build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling ) @@ -308,17 +303,11 @@ def _set_cos_sin_cache(self, seq_len: int): inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask self.register_buffer("inv_freq", inv_freq, persistent=False) - t = torch.arange(seq_len, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - _mscale = float( self._yarn_get_mscale(self.scaling_factor, self.mscale) / self._yarn_get_mscale(self.scaling_factor, self.mscale_all_dim) ) - - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", (emb.cos() * _mscale), persistent=False) - self.register_buffer("_ad_sin_cached", (emb.sin() * _mscale), persistent=False) + self.attention_scaling = _mscale @staticmethod def _yarn_find_correction_dim( diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama3_ir.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama3_ir.py index 3c44530e32de..5ae0af50c6fc 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama3_ir.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama3_ir.py @@ -45,6 +45,7 @@ from ... import custom_ops # noqa: F401 -- register all ops from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class Llama3RMSNorm(nn.Module): @@ -61,14 +62,12 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) -class Llama3RotaryEmbedding(nn.Module): +class Llama3RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Llama 3 family. Supports all rope types (default, llama3, linear, dynamic, etc.) via - transformers ROPE_INIT_FUNCTIONS. Precomputes and caches cos/sin values. - Slices by position_ids once and returns pre-sliced cos/sin to all layers. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + transformers ROPE_INIT_FUNCTIONS. Keeps only the small inv_freq buffer + before graph-cache transforms. """ def __init__(self, config: LlamaConfig): @@ -81,19 +80,15 @@ def __init__(self, config: LlamaConfig): rope_type = "default" inv_freq, self.attention_scaling = ROPE_INIT_FUNCTIONS[rope_type](config, device=None) - - max_pos = config.max_position_embeddings - t = torch.arange(max_pos, dtype=inv_freq.dtype) - freqs = torch.outer(t, inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", emb.cos() * self.attention_scaling, persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin() * self.attention_scaling, persistent=False) + self.max_position_embeddings = config.max_position_embeddings + self.register_buffer("inv_freq", inv_freq, persistent=False) def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - cos = self._ad_cos_cached.to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached.to(dtype=x.dtype, device=x.device) + cos, sin = build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling + ) return cos[position_ids], sin[position_ids] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama4.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama4.py index b330a5f467aa..62b76dd64f12 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama4.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_llama4.py @@ -29,7 +29,6 @@ * Removed gradient checkpointing and training code paths * Removed attention dropout (inference only) * No repeat_kv — AD attention ops handle GQA natively -* Complex RoPE frequencies precomputed once at model level The Llama 4 family features: * GQA with complex-frequency RoPE (with llama3-style scaling) @@ -52,6 +51,7 @@ from transformers.utils import ModelOutput from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_complex_cache # ========================================================================= # Normalization @@ -94,14 +94,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # ========================================================================= -class Llama4RotaryEmbedding(nn.Module): +class Llama4RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Llama 4 using complex frequencies. Supports llama3-style rope scaling via transformers ROPE_INIT_FUNCTIONS. - Precomputes and caches complex freqs_cis values. Slices by position_ids - once and returns pre-sliced freqs to all layers. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + Keeps only the small inv_freq buffer before graph-cache transforms. """ def __init__(self, config: Llama4TextConfig): @@ -126,16 +123,11 @@ def __init__(self, config: Llama4TextConfig): # Use HF's ROPE_INIT_FUNCTIONS to compute inv_freq with proper scaling inv_freq, self.attention_scaling = ROPE_INIT_FUNCTIONS[rope_type](config, device=None) - # Precompute complex frequencies for all positions - max_pos = config.max_position_embeddings - t = torch.arange(max_pos, dtype=inv_freq.dtype) - freqs = torch.outer(t, inv_freq) # [max_pos, head_dim//2] - freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex [max_pos, D//2] - freqs_cis = freqs_cis * self.attention_scaling - self.register_buffer("_ad_freqs_cis", freqs_cis, persistent=False) + self.max_position_embeddings = config.max_position_embeddings + self.register_buffer("inv_freq", inv_freq, persistent=False) def forward(self, x: torch.Tensor) -> torch.Tensor: - """Return the full precomputed complex freqs_cis table. + """Return the full graph-computed complex freqs_cis table. Args: x: Input tensor (used only for device). @@ -143,7 +135,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Returns: freqs_cis: [max_pos, head_dim//2] complex tensor (full table). """ - return self._ad_freqs_cis.to(device=x.device) + return build_rope_complex_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling + ) # ========================================================================= diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py index cfa257c6bed1..18e509ebd91d 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py @@ -40,6 +40,7 @@ from tensorrt_llm._torch.utils import ActivationType from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache # --------------------------------------------------------------------------- # Output dataclasses @@ -93,27 +94,25 @@ def apply_rotary_pos_emb( # --------------------------------------------------------------------------- -# Rotary Embedding (pre-cached with _ad_ prefix) +# Rotary Embedding # --------------------------------------------------------------------------- -class MiniMaxM2RotaryEmbedding(nn.Module): - """Pre-computed RoPE cache for partial rotation (rotary_dim < head_dim).""" +class MiniMaxM2RotaryEmbedding(RotaryEmbeddingBase): + """RoPE for partial rotation (rotary_dim < head_dim).""" def __init__(self, dim: int, max_position_embeddings: int, base: float): super().__init__() + self.max_position_embeddings = max_position_embeddings inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) - t = torch.arange(max_position_embeddings, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) + self.register_buffer("inv_freq", inv_freq, persistent=False) def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - cos = self._ad_cos_cached[position_ids].to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached[position_ids].to(dtype=x.dtype, device=x.device) + cos, sin = build_rope_cos_sin_cache(self.inv_freq, self.max_position_embeddings, x) + cos = cos[position_ids] + sin = sin[position_ids] return cos, sin diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_mistral3.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_mistral3.py index 61c58694a37b..c99928756ae6 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_mistral3.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_mistral3.py @@ -50,6 +50,7 @@ from ..factory import ModelFactoryRegistry from ..hf import AutoModelForCausalLMFactory, AutoModelForImageTextToTextFactory from . import mla_rope_utils +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class Mistral4TextConfig(PretrainedConfig): @@ -201,7 +202,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) -class Mistral4RotaryEmbedding(nn.Module): +class Mistral4RotaryEmbedding(RotaryEmbeddingBase): def __init__(self, dim: int, max_position_embeddings: int, base: float): super().__init__() self.dim = dim @@ -209,21 +210,17 @@ def __init__(self, dim: int, max_position_embeddings: int, base: float): self.base = base inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) + self.attention_scaling = 1.0 self._build_cache(max_position_embeddings) def _build_cache(self, seq_len: int) -> None: - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) + self.max_seq_len_cached = seq_len def forward( self, x: torch.Tensor, seq_len: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: - return ( - self._ad_cos_cached.to(dtype=x.dtype, device=x.device), - self._ad_sin_cached.to(dtype=x.dtype, device=x.device), + return build_rope_cos_sin_cache( + self.inv_freq, self.max_position_embeddings, x, self.attention_scaling ) @@ -280,6 +277,7 @@ def _yarn_linear_ramp_mask(min_val: float, max_val: float, dim: int) -> torch.Te return torch.clamp(linear_func, 0, 1) def _build_cache(self, seq_len: int) -> None: + self.max_seq_len_cached = seq_len dim = self.dim freq_extra = 1.0 / (self.base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) freq_inter = 1.0 / ( @@ -295,15 +293,11 @@ def _build_cache(self, seq_len: int) -> None: inv_freq_mask = 1.0 - self._yarn_linear_ramp_mask(low, high, dim // 2) inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask self.register_buffer("inv_freq", inv_freq, persistent=False) - t = torch.arange(seq_len, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) mscale = float( self._yarn_get_mscale(self.scaling_factor, self.mscale) / self._yarn_get_mscale(self.scaling_factor, self.mscale_all_dim) ) - self.register_buffer("_ad_cos_cached", emb.cos() * mscale, persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin() * mscale, persistent=False) + self.attention_scaling = mscale class Mistral4MLP(nn.Module): diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py index 37fda3c1bcd5..40656fa9bc09 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py @@ -59,6 +59,7 @@ from ... import custom_ops # noqa: F401 -- register all ops from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class Qwen3RMSNorm(nn.Module): @@ -75,13 +76,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) -class Qwen3RotaryEmbedding(nn.Module): +class Qwen3RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Qwen3. - Simplified version that precomputes and caches cos/sin values. - Returns full cached values (not sliced by seq_len) to enable export. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__( @@ -98,24 +97,10 @@ def __init__( inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) - # Build cos/sin cache with AD-specific naming - self._set_cos_sin_cache(max_position_embeddings) - - def _set_cos_sin_cache(self, seq_len: int): - self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - # Use _ad_ prefix for AutoDeploy compatibility with lift_to_meta - self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) - def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - # Slice cos/sin by position_ids here (once) instead of in every attention layer - cos = self._ad_cos_cached.to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached.to(dtype=x.dtype, device=x.device) + cos, sin = build_rope_cos_sin_cache(self.inv_freq, self.max_position_embeddings, x) return cos[position_ids], sin[position_ids] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_moe.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_moe.py index 50e6830d5604..c36daefc4387 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_moe.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_moe.py @@ -49,6 +49,7 @@ from transformers.utils import ModelOutput from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache class Qwen3MoeRMSNorm(nn.Module): @@ -65,13 +66,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) -class Qwen3MoeRotaryEmbedding(nn.Module): +class Qwen3MoeRotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Qwen3 MoE. - Simplified version that precomputes and caches cos/sin values. - Returns pre-sliced values indexed by position_ids. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__( @@ -88,24 +87,10 @@ def __init__( inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) - # Build cos/sin cache with AD-specific naming - self._set_cos_sin_cache(max_position_embeddings) - - def _set_cos_sin_cache(self, seq_len: int): - self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - # Use _ad_ prefix for AutoDeploy compatibility with lift_to_meta - self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) - def forward( self, x: torch.Tensor, position_ids: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - # Slice cos/sin by position_ids here (once) instead of in every attention layer - cos = self._ad_cos_cached.to(dtype=x.dtype, device=x.device) - sin = self._ad_sin_cached.to(dtype=x.dtype, device=x.device) + cos, sin = build_rope_cos_sin_cache(self.inv_freq, self.max_position_embeddings, x) return cos[position_ids], sin[position_ids] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_starcoder2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_starcoder2.py index 3dfc5e701066..4487fc94022b 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_starcoder2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_starcoder2.py @@ -46,15 +46,14 @@ from transformers.utils import ModelOutput from ..hf import AutoModelForCausalLMFactory +from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache -class Starcoder2RotaryEmbedding(nn.Module): +class Starcoder2RotaryEmbedding(RotaryEmbeddingBase): """Rotary Position Embedding for Starcoder2. - Precomputes and caches cos/sin values for the full max_position_embeddings. - Returns values indexed by position_ids to enable export. - - Uses _ad_ prefix for buffer names to work with AutoDeploy's lift_to_meta. + Keeps only the small inv_freq buffer before graph-cache transforms. The full + cos/sin table is graph-computed and materialized by later RoPE transforms. """ def __init__( @@ -75,18 +74,9 @@ def __init__( def _set_cos_sin_cache(self, seq_len: int): self.max_seq_len_cached = seq_len - t = torch.arange(seq_len, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("_ad_cos_cached", emb.cos(), persistent=False) - self.register_buffer("_ad_sin_cached", emb.sin(), persistent=False) def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - # Return full cached tables — slicing by position_ids happens downstream in attention. - return ( - self._ad_cos_cached.to(dtype=x.dtype, device=x.device), - self._ad_sin_cached.to(dtype=x.dtype, device=x.device), - ) + return build_rope_cos_sin_cache(self.inv_freq, self.max_position_embeddings, x) class Starcoder2MLP(nn.Module): diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/rotary_utils.py b/tensorrt_llm/_torch/auto_deploy/models/custom/rotary_utils.py new file mode 100644 index 000000000000..9cdb4056367a --- /dev/null +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/rotary_utils.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RoPE table helpers for AutoDeploy custom model implementations.""" + +import torch +from torch import nn + + +class RotaryEmbeddingBase(nn.Module): + """Base class for RoPE modules that keep inv_freq in FP32.""" + + def _apply(self, fn): + super()._apply(fn) + inv_freq = getattr(self, "inv_freq", None) + if isinstance(inv_freq, torch.Tensor) and inv_freq.is_floating_point(): + self.inv_freq = inv_freq.float() + return self + + +def build_rope_cos_sin_cache( + inv_freq: torch.Tensor, + max_position_embeddings: int, + target: torch.Tensor, + attention_scaling: float = 1.0, +) -> tuple[torch.Tensor, torch.Tensor]: + """Build full RoPE cos/sin tables from a small inv_freq buffer. + + This intentionally returns graph-computed tensors instead of registering large + module buffers. ``optimize_rope`` recognizes this pattern after the + pipeline-cache boundary and materializes a fused ``_prefused_rope_cache`` on + the exported ``GraphModule`` when possible. + """ + inv_freq = inv_freq.to(device=target.device) + positions = torch.arange( + max_position_embeddings, + dtype=inv_freq.dtype, + device=inv_freq.device, + ) + freqs = torch.outer(positions, inv_freq) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * attention_scaling + sin = emb.sin() * attention_scaling + return cos.to(dtype=target.dtype), sin.to(dtype=target.dtype) + + +def build_rope_complex_cache( + inv_freq: torch.Tensor, + max_position_embeddings: int, + target: torch.Tensor, + attention_scaling: float = 1.0, +) -> torch.Tensor: + """Build full complex RoPE frequencies from a small inv_freq buffer.""" + inv_freq = inv_freq.to(device=target.device) + positions = torch.arange( + max_position_embeddings, + dtype=inv_freq.dtype, + device=inv_freq.device, + ) + freqs = torch.outer(positions, inv_freq) + return torch.polar(torch.ones_like(freqs), freqs) * attention_scaling diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py index 4bb20b5b3b0d..d3d1563ed444 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py @@ -238,7 +238,9 @@ def _compute_rotary_cos_sin_from_config( rope_scaling = getattr(model_config, "rope_scaling", None) mscale = 1.0 if rope_scaling is not None: - scaling_type = rope_scaling.get("type", "") + scaling_type = rope_scaling.get( + "type", rope_scaling.get("rope_type", "yarn" if "factor" in rope_scaling else "") + ) if scaling_type == "yarn": factor = rope_scaling.get("factor", 1.0) mscale_factor = rope_scaling.get("mscale", 1.0) @@ -271,14 +273,15 @@ def _yarn_find_correction_dim(num_rotations, dim, base, max_pos): _yarn_find_correction_dim(beta_slow, half_dim * 2, rope_theta, original_max) ) low = max(low, 0) - high = min(high, half_dim - 1) - - if low < high: - smooth = (torch.arange(half_dim, dtype=torch.float32) - low) / (high - low) - smooth = smooth.clamp(0, 1) - inv_freq = inv_freq / factor * (1 - smooth) + inv_freq * smooth - else: - inv_freq = inv_freq / factor + high = min(high, qk_rope_head_dim - 1) + if low == high: + high += 0.001 + + smooth = (torch.arange(half_dim, dtype=torch.float32) - low) / (high - low) + smooth = smooth.clamp(0, 1) + freq_extra = inv_freq + freq_inter = inv_freq / factor + inv_freq = freq_inter * smooth + freq_extra * (1 - smooth) t = torch.arange(max_position, dtype=torch.float32) freqs = torch.outer(t, inv_freq) diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 6eec1c676946..10adf8fdbd40 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -356,7 +356,6 @@ l0_dgx_b200: - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-4] - accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-trtllm] - accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[fp8-4-attn_dp_off-trtllm] - - accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] - accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4] - accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] # ------------- AutoDeploy Perf Sanity --------------- @@ -376,6 +375,8 @@ l0_dgx_b200: backend: autodeploy orchestrator: mpi tests: + # Move to post-merge due to https://nvbugspro.nvidia.com/bug/6221483 + - accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] - accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4] - accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-4-trtllm] - accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[bf16-4-attn_dp_off-trtllm] diff --git a/tests/unittest/auto_deploy/singlegpu/mlir/test_elementwise_fusion_e2e.py b/tests/unittest/auto_deploy/singlegpu/mlir/test_elementwise_fusion_e2e.py index 946c1758d39f..398950853709 100644 --- a/tests/unittest/auto_deploy/singlegpu/mlir/test_elementwise_fusion_e2e.py +++ b/tests/unittest/auto_deploy/singlegpu/mlir/test_elementwise_fusion_e2e.py @@ -33,6 +33,7 @@ ) from xdsl.ir import Block, Region # noqa: E402 +from tensorrt_llm._torch.auto_deploy.mlir.codegen.kernel_cache import KernelCache # noqa: E402 from tensorrt_llm._torch.auto_deploy.mlir.codegen.triton_emitter import ( # noqa: E402 generate_kernel_from_subgraph, ) @@ -49,6 +50,9 @@ from tensorrt_llm._torch.auto_deploy.mlir.fusion.subgraph_discovery import ( # noqa: E402 discover_fusible_subgraphs, ) +from tensorrt_llm._torch.auto_deploy.mlir.fusion.subgraph_replace import ( # noqa: E402 + replace_subgraph_with_fused_op, +) def _build_add_rmsnorm_module(hidden: int = 128): @@ -80,6 +84,27 @@ def _build_add_rmsnorm_module(hidden: int = 128): return ModuleOp(Region([block])) +def _build_single_output_add_mul_module(hidden: int = 128): + """Build a minimal MLIR module whose largest fusible subgraph has one output.""" + t = TensorType(BFloat16Type(), [2, hidden]) + + block = Block() + x = block.insert_arg(t, 0) + y = block.insert_arg(t, 1) + z = block.insert_arg(t, 2) + + add_op = AdAdd.build(operands=[x, y], result_types=[t]) + block.add_op(add_op) + + mul_op = AdMul.build(operands=[add_op.output, z], result_types=[t]) + block.add_op(mul_op) + + out_op = AdGraphOutput.build(operands=[[mul_op.output]]) + block.add_op(out_op) + + return ModuleOp(Region([block])) + + def test_decompose_discover_codegen_pipeline(): """Full pipeline: decompose add+rmsnorm -> discover subgraph -> generate kernel.""" hidden = 128 @@ -102,6 +127,29 @@ def test_decompose_discover_codegen_pipeline(): assert callable(kernel_fn), "Expected generate_kernel_from_subgraph to return a callable" +def test_single_output_fused_metadata_uses_tuple_contract(): + """Generated fused kernels return tuples, including the one-output case.""" + hidden = 128 + mlir_mod = _build_single_output_add_mul_module(hidden) + + subgraphs = discover_fusible_subgraphs(mlir_mod) + assert len(subgraphs) == 1 + + sg = subgraphs[0] + kernel_fn = generate_kernel_from_subgraph(sg) + sg_hash = KernelCache.hash_subgraph(sg) + metadata = {} + + replace_subgraph_with_fused_op(sg, kernel_fn, sg_hash, metadata) + + node_key = f"mlir_fused_{sg_hash}" + val_meta = metadata[node_key]["val"] + + assert isinstance(val_meta, tuple) + assert len(val_meta) == 1 + assert tuple(val_meta[0].shape) == (2, hidden) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") def test_decompose_discover_codegen_numerical_correctness(): """Generated kernel from add+rmsnorm decomposition matches torch reference.""" diff --git a/tests/unittest/auto_deploy/singlegpu/mlir/test_fx_mlir_roundtrip.py b/tests/unittest/auto_deploy/singlegpu/mlir/test_fx_mlir_roundtrip.py index aaf6208f4671..797e91f4d96f 100644 --- a/tests/unittest/auto_deploy/singlegpu/mlir/test_fx_mlir_roundtrip.py +++ b/tests/unittest/auto_deploy/singlegpu/mlir/test_fx_mlir_roundtrip.py @@ -27,6 +27,11 @@ from tensorrt_llm._torch.auto_deploy.mlir.fx_to_mlir import FXToMLIRConverter # noqa: E402 from tensorrt_llm._torch.auto_deploy.mlir.mlir_to_fx import MLIRToFXConverter # noqa: E402 + +def _single_output_fused_stub(x): + return (x,) + + # --------------------------------------------------------------------------- # Test models # --------------------------------------------------------------------------- @@ -81,11 +86,135 @@ def _export_model(model, *inputs): return torch_export_to_gm(model, args=inputs, dynamic_shapes=ds, clone=True) +def _build_single_output_fused_mlir_module(): + from xdsl.dialects.builtin import BFloat16Type, ModuleOp, StringAttr, TensorType + from xdsl.ir import Block, Region + + from tensorrt_llm._torch.auto_deploy.mlir.dialect import AdGraphInput, AdGraphOutput, AdOpaque + + tensor_type = TensorType(BFloat16Type(), [2, 4]) + block = Block() + + input_op = AdGraphInput.build( + attributes={"input_name": StringAttr("x")}, + result_types=[tensor_type], + ) + block.add_op(input_op) + + fused_op = AdOpaque.build( + operands=[[input_op.output]], + attributes={ + "op_name": StringAttr("mlir_fused_single"), + "node_key": StringAttr("mlir_fused_single"), + }, + result_types=[[tensor_type]], + ) + block.add_op(fused_op) + + output_op = AdGraphOutput.build(operands=[[fused_op.outputs[0]]]) + block.add_op(output_op) + + return ModuleOp(Region([block])) + + +def _build_precise_to_dtype_mlir_module(): + from xdsl.dialects.builtin import ( + BFloat16Type, + Float32Type, + IntegerAttr, + IntegerType, + ModuleOp, + StringAttr, + TensorType, + ) + from xdsl.ir import Block, Region + + from tensorrt_llm._torch.auto_deploy.mlir.dialect import AdGraphInput, AdGraphOutput, AdToDtype + + input_type = TensorType(Float32Type(), [2, 4]) + output_type = TensorType(BFloat16Type(), [2, 4]) + block = Block() + + input_op = AdGraphInput.build( + attributes={"input_name": StringAttr("x")}, + result_types=[input_type], + ) + block.add_op(input_op) + + to_dtype_op = AdToDtype.build( + operands=[input_op.output], + attributes={"target_dtype": IntegerAttr(15, IntegerType(64))}, + result_types=[output_type], + ) + block.add_op(to_dtype_op) + + output_op = AdGraphOutput.build(operands=[[to_dtype_op.output]]) + block.add_op(output_op) + + return ModuleOp(Region([block])) + + # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- +def test_mlir_to_fx_propagates_single_output_fused_getitem_meta(): + import operator + + graph = torch.fx.Graph() + x = graph.placeholder("x") + graph.output(x) + original_gm = torch.fx.GraphModule(torch.nn.Module(), graph) + + fake_val = torch.empty((2, 4), dtype=torch.bfloat16, device="meta") + metadata = { + "x": {"val": fake_val}, + "mlir_fused_single": { + "_original_target": _single_output_fused_stub, + "_args_template": (("__mlir_operand__", 0),), + "_kwargs_template": {}, + "val": (fake_val,), + }, + } + + new_gm = MLIRToFXConverter(original_gm).convert( + _build_single_output_fused_mlir_module(), metadata + ) + + getitem_nodes = [ + n for n in new_gm.graph.nodes if n.op == "call_function" and n.target is operator.getitem + ] + + assert len(getitem_nodes) == 1 + assert "val" in getitem_nodes[0].meta + assert tuple(getitem_nodes[0].meta["val"].shape) == tuple(fake_val.shape) + assert getitem_nodes[0].meta["val"].dtype == fake_val.dtype + + +def test_mlir_to_fx_synthesizes_precise_op_meta_from_result_type(): + graph = torch.fx.Graph() + x = graph.placeholder("x") + graph.output(x) + original_gm = torch.fx.GraphModule(torch.nn.Module(), graph) + + input_val = torch.empty((2, 4), dtype=torch.float32, device="meta") + new_gm = MLIRToFXConverter(original_gm).convert( + _build_precise_to_dtype_mlir_module(), {"x": {"val": input_val}} + ) + + to_dtype_nodes = [ + n + for n in new_gm.graph.nodes + if n.op == "call_function" and n.target == torch.ops.aten.to.dtype + ] + + assert len(to_dtype_nodes) == 1 + assert "val" in to_dtype_nodes[0].meta + assert tuple(to_dtype_nodes[0].meta["val"].shape) == (2, 4) + assert to_dtype_nodes[0].meta["val"].dtype == torch.bfloat16 + + def test_fx_to_mlir_basic(): """FX → MLIR conversion produces expected ops.""" model = AddNormModel() diff --git a/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py b/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py index 775d43163f62..91ab98297b6e 100644 --- a/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py +++ b/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py @@ -30,6 +30,9 @@ DeepSeekV3RotaryEmbedding, DeepSeekV3YarnRotaryEmbedding, ) +from tensorrt_llm._torch.auto_deploy.transform.library.fuse_rope_mla import ( + _compute_rotary_cos_sin_from_config, +) class MockDeepSeekConfig(PretrainedConfig): @@ -147,6 +150,48 @@ def test_yarn_rope_shape(self): assert cos.shape == (max_pos, dim) assert sin.shape == (max_pos, dim) + def test_fused_mla_yarn_config_fallback_matches_model_rope(self): + """The fused MLA RoPE fallback must match the model's YaRN table.""" + dim = 64 + max_pos = 128 + config = MockDeepSeekConfig() + config.qk_rope_head_dim = dim + config.max_position_embeddings = max_pos + config.rope_scaling = { + "type": "yarn", + "factor": 40.0, + "original_max_position_embeddings": 4096, + "beta_fast": 32, + "beta_slow": 1, + "mscale": 1.0, + "mscale_all_dim": 1.0, + } + + rope = DeepSeekV3YarnRotaryEmbedding( + dim, + max_pos, + base=config.rope_theta, + scaling_factor=config.rope_scaling["factor"], + original_max_position_embeddings=config.rope_scaling[ + "original_max_position_embeddings" + ], + beta_fast=config.rope_scaling["beta_fast"], + beta_slow=config.rope_scaling["beta_slow"], + mscale=config.rope_scaling["mscale"], + mscale_all_dim=config.rope_scaling["mscale_all_dim"], + ) + + x = torch.empty(1, 1, 1, dim, dtype=torch.float32) + cos, sin = rope(x) + expected = torch.stack([cos.float(), sin.float()], dim=-1).reshape(1, -1) + + class Factory: + def _get_model_config(self): + return config, None + + actual = _compute_rotary_cos_sin_from_config(Factory()).cpu() + torch.testing.assert_close(actual, expected, atol=3e-7, rtol=1e-4) + class TestDeepSeekV3MLP: """Test DeepSeekV3MLP implementation.""" diff --git a/tests/unittest/auto_deploy/singlegpu/models/test_mla_rope_utils.py b/tests/unittest/auto_deploy/singlegpu/models/test_mla_rope_utils.py index 26cb53e712bc..041d4fbf7a36 100644 --- a/tests/unittest/auto_deploy/singlegpu/models/test_mla_rope_utils.py +++ b/tests/unittest/auto_deploy/singlegpu/models/test_mla_rope_utils.py @@ -1,12 +1,90 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from types import SimpleNamespace + import pytest import torch from tensorrt_llm._torch.auto_deploy.models.custom.mla_rope_utils import ( _rope_deinterleave_load_hook, ) +from tensorrt_llm._torch.auto_deploy.models.custom.modeling_deepseek import ( + DeepSeekV3YarnRotaryEmbedding, +) +from tensorrt_llm._torch.auto_deploy.models.custom.modeling_deepseek_v2 import ( + DeepSeekV2YarnRotaryEmbedding, +) +from tensorrt_llm._torch.auto_deploy.models.custom.modeling_glm4_moe_lite import ( + Glm4MoeLiteYarnRotaryEmbedding, +) +from tensorrt_llm._torch.auto_deploy.models.custom.modeling_kimi_k2 import KimiK2YarnRotaryEmbedding +from tensorrt_llm._torch.auto_deploy.transform.library.fuse_rope_mla import ( + _compute_rotary_cos_sin_from_config, +) + + +def _flatten_cos_sin(cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + return torch.stack([cos.float().cpu(), sin.float().cpu()], dim=-1).reshape(1, -1) + + +class _Factory: + def __init__(self, config): + self.config = config + + def _get_model_config(self): + return self.config, None + + +@pytest.mark.parametrize( + "rope_cls", + [ + DeepSeekV2YarnRotaryEmbedding, + DeepSeekV3YarnRotaryEmbedding, + Glm4MoeLiteYarnRotaryEmbedding, + KimiK2YarnRotaryEmbedding, + ], +) +@pytest.mark.parametrize("rope_type_key", ["type", "rope_type"]) +def test_fused_mla_yarn_config_fallback_matches_model_rope_variants(rope_cls, rope_type_key): + dim = 64 + max_pos = 128 + rope_scaling = { + rope_type_key: "yarn", + "factor": 40.0, + "original_max_position_embeddings": 4096, + "beta_fast": 32, + "beta_slow": 1, + "mscale": 1.0, + "mscale_all_dim": 1.0, + } + config = SimpleNamespace( + rope_theta=10000.0, + qk_rope_head_dim=dim, + max_position_embeddings=max_pos, + rope_scaling=rope_scaling, + ) + + rope = rope_cls( + dim, + max_pos, + base=config.rope_theta, + scaling_factor=rope_scaling["factor"], + original_max_position_embeddings=rope_scaling["original_max_position_embeddings"], + beta_fast=rope_scaling["beta_fast"], + beta_slow=rope_scaling["beta_slow"], + mscale=rope_scaling["mscale"], + mscale_all_dim=rope_scaling["mscale_all_dim"], + ) + if hasattr(rope, "_ad_cos_cached"): + expected = _flatten_cos_sin(rope._ad_cos_cached, rope._ad_sin_cached) + else: + x = torch.empty(1, 1, 1, dim, dtype=torch.float32, device="cuda") + cos, sin = rope.to("cuda")(x) + expected = _flatten_cos_sin(cos, sin) + + actual = _compute_rotary_cos_sin_from_config(_Factory(config)).cpu() + torch.testing.assert_close(actual, expected, atol=3e-7, rtol=1e-4) @pytest.mark.parametrize("dtype", [torch.float32, torch.float8_e4m3fn]) diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_rope_transformation.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_rope_transformation.py index 402f694092ac..ee37dcbc7c8c 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_rope_transformation.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_rope_transformation.py @@ -16,7 +16,7 @@ import pytest import torch -from _graph_test_helpers import run_test_transformed_gm +from _graph_test_helpers import FakeFactory, run_test_transformed_gm from _model_test_utils import ( apply_rotary_pos_emb_complex, apply_rotary_pos_emb_ds, @@ -28,6 +28,10 @@ from tensorrt_llm._torch.auto_deploy.models.custom.mla_rope_utils import ( _rope_deinterleave_load_hook, ) +from tensorrt_llm._torch.auto_deploy.models.custom.rotary_utils import ( + RotaryEmbeddingBase, + build_rope_cos_sin_cache, +) from tensorrt_llm._torch.auto_deploy.transform.optimizer import InferenceOptimizer from tensorrt_llm._torch.auto_deploy.utils.node_utils import extract_output_tuple, is_op @@ -572,6 +576,142 @@ def checker(gm): ) +class DynamicCosSinRotaryEmbedding(RotaryEmbeddingBase): + def __init__( + self, + dim, + max_position_embeddings=2048, + base=10000, + attention_scaling=1.0, + device=None, + ): + super().__init__() + self.max_position_embeddings = max_position_embeddings + self.attention_scaling = attention_scaling + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, device=device).float() / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, x): + return build_rope_cos_sin_cache( + self.inv_freq, + self.max_position_embeddings, + x, + self.attention_scaling, + ) + + +class DynamicCosSinExplicitRoPEModel(torch.nn.Module): + def __init__( + self, + hidden_size, + max_seq, + num_heads, + num_kv_heads, + attention_scaling=1.0, + ): + super().__init__() + self.head_dim = hidden_size // num_heads + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.q_lin = torch.nn.Linear(hidden_size, num_heads * self.head_dim) + self.k_lin = torch.nn.Linear(hidden_size, num_kv_heads * self.head_dim) + self.rotary = DynamicCosSinRotaryEmbedding( + self.head_dim, + max_seq, + base=10000, + attention_scaling=attention_scaling, + device="cuda", + ) + + def forward(self, x): + b, s, _ = x.shape + q = self.q_lin(x).view(b, s, self.num_heads, self.head_dim) + k = self.k_lin(x).view(b, s, self.num_kv_heads, self.head_dim) + cos_table, sin_table = self.rotary(x) + position_ids = torch.arange(s, device=x.device).unsqueeze(0).expand(b, s) + cos = cos_table[position_ids] + sin = sin_table[position_ids] + q_out, k_out = torch.ops.auto_deploy.torch_rope_with_explicit_cos_sin(q, k, cos, sin, 2) + return torch.cat([q_out.reshape(b, s, -1), k_out.reshape(b, s, -1)], dim=-1) + + def get_dynamic_shapes(self): + return {0: Dim.DYNAMIC, 1: Dim.DYNAMIC} + + +@torch.inference_mode() +def test_optimize_rope_prefuses_dynamic_cos_sin_cache_from_rotary_util(): + batch, seq, hidden_size = 4, 12, 256 + max_seq = 16 + num_heads = 4 + num_kv_heads = 2 + attention_scaling = 1.25 + model = DynamicCosSinExplicitRoPEModel( + hidden_size, + max_seq, + num_heads, + num_kv_heads, + attention_scaling=attention_scaling, + ).to("cuda", torch.float16) + + x = torch.randn(batch, seq, hidden_size, device="cuda", dtype=torch.float16) + dynamic_shapes = model.get_dynamic_shapes() + gm = torch_export_to_gm(model, args=(x,), dynamic_shapes=(dynamic_shapes,), clone=True) + + assert any( + is_op(n, torch.ops.auto_deploy.torch_rope_with_explicit_cos_sin) for n in gm.graph.nodes + ), "Expected torch_rope_with_explicit_cos_sin in graph before optimization" + + factory = FakeFactory() + gm_transformed = InferenceOptimizer( + factory, + {"optimize_rope": {"stage": "pattern_matcher"}}, + )(None, gm) + gm_transformed.to("cuda") + + cache_names = [ + name + for name, _ in gm_transformed.named_buffers() + if name.startswith("_prefused_rope_cache") + ] + assert len(cache_names) == 1 + + cache = gm_transformed.get_buffer(cache_names[0]) + inv_freq = model.rotary.inv_freq + positions = torch.arange(factory.max_seq_len, dtype=inv_freq.dtype, device=inv_freq.device) + freqs = torch.outer(positions, inv_freq) + expected_cache = torch.cat( + [freqs.cos() * attention_scaling, freqs.sin() * attention_scaling], + dim=-1, + ).to(torch.float32) + torch.testing.assert_close(cache, expected_cache) + + def checker(gm): + flash_nodes = [n for n in gm.graph.nodes if is_op(n, torch.ops.auto_deploy.flashinfer_rope)] + if len(flash_nodes) != 1: + return False + + cache_arg = flash_nodes[0].args[3] + return ( + isinstance(cache_arg, torch.fx.Node) + and cache_arg.op == "get_attr" + and cache_arg.target == cache_names[0] + ) + + run_test_transformed_gm( + model, + x, + gm_transformed, + checker, + lambda num_p: num_p, + atol=1e-2, + rtol=1e-2, + test_load_hook=True, + strict_loading=True, + dynamic_shapes=dynamic_shapes, + skip_output_assert=False, + ) + + class DSExplicitRotaryEmbedding(torch.nn.Module): """Rotary embedding that stores cos/sin as buffers (full tables). From 276ccd64f2a7e7061d78f72b3d9c52e2098405f8 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Wed, 27 May 2026 03:12:39 +0000 Subject: [PATCH 044/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/auto_deploy/poetry.lock | 13 +-- .../examples/draft_target_model/poetry.lock | 9 +- security_scanning/examples/eagle/poetry.lock | 9 +- .../llm-eval/lm-eval-harness/poetry.lock | 13 +-- .../examples/lookahead/poetry.lock | 9 +- security_scanning/examples/medusa/poetry.lock | 9 +- .../models/contrib/baichuan/poetry.lock | 9 +- .../examples/models/contrib/bloom/poetry.lock | 9 +- .../models/contrib/chatglm-6b/poetry.lock | 9 +- .../models/contrib/chatglm2-6b/poetry.lock | 9 +- .../contrib/chatglm3-6b-32k/poetry.lock | 9 +- .../examples/models/contrib/dbrx/poetry.lock | 9 +- .../models/contrib/deepseek_v1/poetry.lock | 9 +- .../models/contrib/deepseek_v2/poetry.lock | 9 +- .../models/contrib/falcon/poetry.lock | 9 +- .../examples/models/contrib/gptj/poetry.lock | 9 +- .../models/contrib/gptneox/poetry.lock | 9 +- .../examples/models/contrib/grok/poetry.lock | 9 +- .../models/contrib/hyperclovax/poetry.lock | 13 +-- .../models/contrib/internlm/poetry.lock | 9 +- .../examples/models/contrib/jais/poetry.lock | 9 +- .../examples/models/contrib/mmdit/poetry.lock | 9 +- .../examples/models/contrib/mpt/poetry.lock | 9 +- .../examples/models/contrib/opt/poetry.lock | 9 +- .../models/contrib/skywork/poetry.lock | 9 +- .../examples/models/contrib/smaug/poetry.lock | 9 +- .../examples/models/contrib/stdit/poetry.lock | 13 +-- .../examples/models/core/commandr/poetry.lock | 9 +- .../examples/models/core/gemma/poetry.lock | 9 +- .../examples/models/core/glm-4-9b/poetry.lock | 9 +- .../examples/models/core/gpt/poetry.lock | 9 +- .../examples/models/core/llama/poetry.lock | 9 +- .../examples/models/core/mamba/poetry.lock | 9 +- .../examples/models/core/mixtral/poetry.lock | 4 +- .../examples/models/core/mllama/poetry.lock | 4 +- .../examples/models/core/nemotron/poetry.lock | 9 +- .../examples/models/core/phi/poetry.lock | 9 +- .../examples/models/core/qwen/poetry.lock | 9 +- .../models/core/qwen2audio/poetry.lock | 9 +- .../examples/models/core/qwenvl/poetry.lock | 13 +-- .../models/core/recurrentgemma/poetry.lock | 9 +- .../examples/models/core/whisper/poetry.lock | 13 +-- security_scanning/examples/ngram/poetry.lock | 9 +- .../examples/quantization/poetry.lock | 9 +- .../examples/redrafter/poetry.lock | 9 +- security_scanning/examples/serve/poetry.lock | 9 +- .../examples/trtllm-eval/poetry.lock | 13 +-- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 15 +-- security_scanning/triton_backend/poetry.lock | 91 ++++++++++--------- 50 files changed, 302 insertions(+), 249 deletions(-) diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index e5227d9a6e1c..779e49242e2f 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -579,14 +579,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -1137,17 +1137,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1155,7 +1156,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/draft_target_model/poetry.lock b/security_scanning/examples/draft_target_model/poetry.lock index 2bbb5304abb8..181c780b1fff 100644 --- a/security_scanning/examples/draft_target_model/poetry.lock +++ b/security_scanning/examples/draft_target_model/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/eagle/poetry.lock b/security_scanning/examples/eagle/poetry.lock index 273d5a54ed11..bed68fee3f9a 100644 --- a/security_scanning/examples/eagle/poetry.lock +++ b/security_scanning/examples/eagle/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index b306b28b3c9e..e81117d89a49 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -500,14 +500,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -992,17 +992,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1010,7 +1011,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/lookahead/poetry.lock b/security_scanning/examples/lookahead/poetry.lock index 2bbb5304abb8..181c780b1fff 100644 --- a/security_scanning/examples/lookahead/poetry.lock +++ b/security_scanning/examples/lookahead/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/medusa/poetry.lock b/security_scanning/examples/medusa/poetry.lock index 2bbb5304abb8..181c780b1fff 100644 --- a/security_scanning/examples/medusa/poetry.lock +++ b/security_scanning/examples/medusa/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/baichuan/poetry.lock b/security_scanning/examples/models/contrib/baichuan/poetry.lock index c56e7094fd58..697bdb39fdeb 100644 --- a/security_scanning/examples/models/contrib/baichuan/poetry.lock +++ b/security_scanning/examples/models/contrib/baichuan/poetry.lock @@ -839,17 +839,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -857,7 +858,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/bloom/poetry.lock b/security_scanning/examples/models/contrib/bloom/poetry.lock index 34ceff26ca50..d44bb3c7605b 100644 --- a/security_scanning/examples/models/contrib/bloom/poetry.lock +++ b/security_scanning/examples/models/contrib/bloom/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock index cfc925053ecd..c32a0fbf6d9b 100644 --- a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock index cfc925053ecd..c32a0fbf6d9b 100644 --- a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock index cfc925053ecd..c32a0fbf6d9b 100644 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/dbrx/poetry.lock b/security_scanning/examples/models/contrib/dbrx/poetry.lock index 546fd6a12bd9..c5ed9ef17aeb 100644 --- a/security_scanning/examples/models/contrib/dbrx/poetry.lock +++ b/security_scanning/examples/models/contrib/dbrx/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock index e7b61db50233..f878729b3a47 100644 --- a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/falcon/poetry.lock b/security_scanning/examples/models/contrib/falcon/poetry.lock index d1fafb8b7252..d5fe9085af98 100644 --- a/security_scanning/examples/models/contrib/falcon/poetry.lock +++ b/security_scanning/examples/models/contrib/falcon/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/gptj/poetry.lock b/security_scanning/examples/models/contrib/gptj/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/contrib/gptj/poetry.lock +++ b/security_scanning/examples/models/contrib/gptj/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/gptneox/poetry.lock b/security_scanning/examples/models/contrib/gptneox/poetry.lock index 273d5a54ed11..bed68fee3f9a 100644 --- a/security_scanning/examples/models/contrib/gptneox/poetry.lock +++ b/security_scanning/examples/models/contrib/gptneox/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/grok/poetry.lock b/security_scanning/examples/models/contrib/grok/poetry.lock index 595b9bd8cabe..087b1c6d8610 100644 --- a/security_scanning/examples/models/contrib/grok/poetry.lock +++ b/security_scanning/examples/models/contrib/grok/poetry.lock @@ -981,17 +981,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -999,7 +1000,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index 32bc1f53f421..ab0009f30043 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -143,14 +143,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -394,17 +394,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -412,7 +413,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/internlm/poetry.lock b/security_scanning/examples/models/contrib/internlm/poetry.lock index 2bbb5304abb8..181c780b1fff 100644 --- a/security_scanning/examples/models/contrib/internlm/poetry.lock +++ b/security_scanning/examples/models/contrib/internlm/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/jais/poetry.lock b/security_scanning/examples/models/contrib/jais/poetry.lock index 34ceff26ca50..d44bb3c7605b 100644 --- a/security_scanning/examples/models/contrib/jais/poetry.lock +++ b/security_scanning/examples/models/contrib/jais/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/mmdit/poetry.lock b/security_scanning/examples/models/contrib/mmdit/poetry.lock index d329cfe4b462..dfbe1bd0fcfb 100644 --- a/security_scanning/examples/models/contrib/mmdit/poetry.lock +++ b/security_scanning/examples/models/contrib/mmdit/poetry.lock @@ -420,17 +420,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -438,7 +439,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/mpt/poetry.lock b/security_scanning/examples/models/contrib/mpt/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/contrib/mpt/poetry.lock +++ b/security_scanning/examples/models/contrib/mpt/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/opt/poetry.lock b/security_scanning/examples/models/contrib/opt/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/contrib/opt/poetry.lock +++ b/security_scanning/examples/models/contrib/opt/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/skywork/poetry.lock b/security_scanning/examples/models/contrib/skywork/poetry.lock index 34ceff26ca50..d44bb3c7605b 100644 --- a/security_scanning/examples/models/contrib/skywork/poetry.lock +++ b/security_scanning/examples/models/contrib/skywork/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/smaug/poetry.lock b/security_scanning/examples/models/contrib/smaug/poetry.lock index 34ceff26ca50..d44bb3c7605b 100644 --- a/security_scanning/examples/models/contrib/smaug/poetry.lock +++ b/security_scanning/examples/models/contrib/smaug/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/contrib/stdit/poetry.lock b/security_scanning/examples/models/contrib/stdit/poetry.lock index c1455311e1ea..74c5acb363b1 100644 --- a/security_scanning/examples/models/contrib/stdit/poetry.lock +++ b/security_scanning/examples/models/contrib/stdit/poetry.lock @@ -616,14 +616,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -944,17 +944,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -962,7 +963,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/commandr/poetry.lock b/security_scanning/examples/models/core/commandr/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/core/commandr/poetry.lock +++ b/security_scanning/examples/models/core/commandr/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/gemma/poetry.lock b/security_scanning/examples/models/core/gemma/poetry.lock index 34abf3551b80..d40358cfdd00 100644 --- a/security_scanning/examples/models/core/gemma/poetry.lock +++ b/security_scanning/examples/models/core/gemma/poetry.lock @@ -951,17 +951,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -969,7 +970,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/glm-4-9b/poetry.lock b/security_scanning/examples/models/core/glm-4-9b/poetry.lock index cfc925053ecd..c32a0fbf6d9b 100644 --- a/security_scanning/examples/models/core/glm-4-9b/poetry.lock +++ b/security_scanning/examples/models/core/glm-4-9b/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/gpt/poetry.lock b/security_scanning/examples/models/core/gpt/poetry.lock index 34ceff26ca50..d44bb3c7605b 100644 --- a/security_scanning/examples/models/core/gpt/poetry.lock +++ b/security_scanning/examples/models/core/gpt/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/llama/poetry.lock b/security_scanning/examples/models/core/llama/poetry.lock index adf59334e511..5c12350d21cb 100644 --- a/security_scanning/examples/models/core/llama/poetry.lock +++ b/security_scanning/examples/models/core/llama/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/mamba/poetry.lock b/security_scanning/examples/models/core/mamba/poetry.lock index 75945e62a633..67392b7a282e 100644 --- a/security_scanning/examples/models/core/mamba/poetry.lock +++ b/security_scanning/examples/models/core/mamba/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock index d1516a94dcf3..d4dbf489f2be 100644 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ b/security_scanning/examples/models/core/mixtral/poetry.lock @@ -232,14 +232,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] diff --git a/security_scanning/examples/models/core/mllama/poetry.lock b/security_scanning/examples/models/core/mllama/poetry.lock index d7320df7f6c2..61e2b681c6b9 100644 --- a/security_scanning/examples/models/core/mllama/poetry.lock +++ b/security_scanning/examples/models/core/mllama/poetry.lock @@ -225,14 +225,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] diff --git a/security_scanning/examples/models/core/nemotron/poetry.lock b/security_scanning/examples/models/core/nemotron/poetry.lock index 9fb09b5ccfac..4bc5cf76045c 100644 --- a/security_scanning/examples/models/core/nemotron/poetry.lock +++ b/security_scanning/examples/models/core/nemotron/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/phi/poetry.lock b/security_scanning/examples/models/core/phi/poetry.lock index 3a25a09f7378..bb3b6bc9b6d9 100644 --- a/security_scanning/examples/models/core/phi/poetry.lock +++ b/security_scanning/examples/models/core/phi/poetry.lock @@ -840,17 +840,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -858,7 +859,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/qwen/poetry.lock b/security_scanning/examples/models/core/qwen/poetry.lock index 2376be8c32d7..db818eb8ca56 100644 --- a/security_scanning/examples/models/core/qwen/poetry.lock +++ b/security_scanning/examples/models/core/qwen/poetry.lock @@ -996,17 +996,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1014,7 +1015,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/qwen2audio/poetry.lock b/security_scanning/examples/models/core/qwen2audio/poetry.lock index 8ad78e325f89..e5295d68ae75 100644 --- a/security_scanning/examples/models/core/qwen2audio/poetry.lock +++ b/security_scanning/examples/models/core/qwen2audio/poetry.lock @@ -840,17 +840,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -858,7 +859,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 46065dc38154..58282ffa4bf1 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -627,14 +627,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -1199,17 +1199,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1217,7 +1218,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/recurrentgemma/poetry.lock b/security_scanning/examples/models/core/recurrentgemma/poetry.lock index 11ce65c43963..32685dc86b4f 100644 --- a/security_scanning/examples/models/core/recurrentgemma/poetry.lock +++ b/security_scanning/examples/models/core/recurrentgemma/poetry.lock @@ -971,17 +971,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -989,7 +990,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/models/core/whisper/poetry.lock b/security_scanning/examples/models/core/whisper/poetry.lock index c72ba3deb7fb..60c2996cbdd4 100644 --- a/security_scanning/examples/models/core/whisper/poetry.lock +++ b/security_scanning/examples/models/core/whisper/poetry.lock @@ -558,14 +558,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -1005,17 +1005,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1023,7 +1024,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/ngram/poetry.lock b/security_scanning/examples/ngram/poetry.lock index 851a875f9982..e35d521f0dd4 100644 --- a/security_scanning/examples/ngram/poetry.lock +++ b/security_scanning/examples/ngram/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/quantization/poetry.lock b/security_scanning/examples/quantization/poetry.lock index 12bb9bf11f05..75f9f276fff5 100644 --- a/security_scanning/examples/quantization/poetry.lock +++ b/security_scanning/examples/quantization/poetry.lock @@ -792,17 +792,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -810,7 +811,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/redrafter/poetry.lock b/security_scanning/examples/redrafter/poetry.lock index 2bbb5304abb8..181c780b1fff 100644 --- a/security_scanning/examples/redrafter/poetry.lock +++ b/security_scanning/examples/redrafter/poetry.lock @@ -828,17 +828,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -846,7 +847,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index 6e1bcb9b90a6..fc859e5055c6 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -1664,17 +1664,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1682,7 +1683,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index ba779efbfc2f..2be449206b0b 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -500,14 +500,14 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -992,17 +992,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1010,7 +1011,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 6f9083cbc70d..a818f2514e2c 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "68d3db1423ad175a4a5492acacfa592470d16702", - "timestamp": "2026-05-26T02:47:45Z" + "commit_hash": "5dd96d6c5f0030b09edecfc6f2f60b6243d1d224", + "timestamp": "2026-05-27T02:46:59Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index 9c9e7c2dbd8c..ede1188680e5 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -1213,13 +1213,13 @@ cu13 = ["cuda-bindings[all] (==13.*)"] [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -1598,7 +1598,7 @@ nvep = ["cuda-python (>=13.0)"] type = "git" url = "https://github.com/flashinfer-ai/flashinfer.git" reference = "v0.6.12rc1" -resolved_reference = "a94541edac53dd5b3e3c944d30ed290e3f7accf0" +resolved_reference = "dd5d33bc54df939c59d6d9d963a61dd0de6c47c1" [[package]] name = "fonttools" @@ -2116,17 +2116,18 @@ files = [ [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -2134,7 +2135,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index e05007023499..691b697532f7 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -512,13 +512,13 @@ all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolki [[package]] name = "cuda-pathfinder" -version = "1.5.4" +version = "1.5.5" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, + {file = "cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689"}, ] [[package]] @@ -1172,17 +1172,18 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.1" +version = "1.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, - {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, + {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, + {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, ] [package.dependencies] +click = ">=8.4.0" filelock = ">=3.10.0" fsspec = ">=2023.5.0" hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} @@ -1190,7 +1191,7 @@ httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" tqdm = ">=4.42.1" -typer = ">=0.20.0" +typer = ">=0.20.0,<0.26.0" typing-extensions = ">=4.1.0" [package.extras] @@ -2432,47 +2433,53 @@ test = ["zope.testrunner (>=6.4)"] [[package]] name = "zope-interface" -version = "8.4" +version = "8.5" description = "Interfaces for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "zope_interface-8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:415de524326ddd61a78f0816f65942fa1aa35dced19e72579ad30dd106ce523e"}, - {file = "zope_interface-8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa0a26d5767087170b3da9ff503221d535ea266bf61b522d0afa2590fd05db0a"}, - {file = "zope_interface-8.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:8544081e32b515bbaf1c6339eef06b23ed470cf4876ff2f575803f82a744cc43"}, - {file = "zope_interface-8.4-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:892b4b5350e58d6914858f58eb85d39fe9b992640ac6ece695f46c978046554d"}, - {file = "zope_interface-8.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8d683267a6243526869cb69677dcfc663416d5f87904c1576ddec6e420687d51"}, - {file = "zope_interface-8.4-cp310-cp310-win_amd64.whl", hash = "sha256:f00fd65343d2a241a2b17688a12f5e815aa704ed64f9ca375de5f9e0ae9c9bda"}, - {file = "zope_interface-8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8b733af6e89a2b0b8edf5ff7a37988fe4e1788806e84e72127b88c47858f0da6"}, - {file = "zope_interface-8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:265bad2df2ec070f23ff863249a89b408b11908fd4207662781fd18e3c6fc912"}, - {file = "zope_interface-8.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e195e76767847afb5379ffd67690c17d3c6efdab58dc0e477cf81ac94d5a5a15"}, - {file = "zope_interface-8.4-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5ec1a56b6cf9a757cbbce9da38284a01473b92b96c1517eabd99150f51f1bb69"}, - {file = "zope_interface-8.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:04c2c9b58e9c177628715d85e94834efa807c1f9f0a2f57ae0f7b553e8266ac4"}, - {file = "zope_interface-8.4-cp311-cp311-win_amd64.whl", hash = "sha256:376d0ef005a131b349e2088e302aa094fa23c826d2ec8a7db4b00fb33c71e0d9"}, - {file = "zope_interface-8.4-cp311-cp311-win_arm64.whl", hash = "sha256:caffd033b27e311b45e15f01923cc9e73c6bfd8e843b4532e29b59ee432bf893"}, - {file = "zope_interface-8.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:84064876ed96ddd0744e3ad5d37134c758d77885e54113567792671405a02bac"}, - {file = "zope_interface-8.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:81ed23698bfb588c48b1756129814b890febac971ff6c8a414f82601773145bb"}, - {file = "zope_interface-8.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e0b9d7e958657fad414f8272afcdf0b8a873fbbb2bb6a6287232d2f11a232bf8"}, - {file = "zope_interface-8.4-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eef0a49e041f4dc4d2a6ab894b4fd0c5354e0e8037e731fb953531e59b0d3d33"}, - {file = "zope_interface-8.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b302f955c36e924e1f4fe70dd9105ff06235857861c6ae72c3b10b016aeee99"}, - {file = "zope_interface-8.4-cp312-cp312-win_amd64.whl", hash = "sha256:4ae6a1e111642dbf724f635424dcaf5a5c8abbde49eac3f452f5323ffaa10232"}, - {file = "zope_interface-8.4-cp312-cp312-win_arm64.whl", hash = "sha256:2e9e4aa33b76877af903d5532545e64d24ade0f6f80d9d1a31e6efcea76a60bc"}, - {file = "zope_interface-8.4-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:cd55965d715413038774aead54851bc3dbdd74a69f3ce30252182a94407b9905"}, - {file = "zope_interface-8.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0d88c1f106a4f06e074a3ada2d20f4a602e3f2871c4f55726ed5d91e94ec19b1"}, - {file = "zope_interface-8.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:36c575356732d59ffd3279ad67e302a6fe517e67db5b061b36b377ee0fa016c4"}, - {file = "zope_interface-8.4-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:29f09ec8bda65f7b30294328070070a2590b90f252f834ee0817cdb0e2c35f6a"}, - {file = "zope_interface-8.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2bc388cebcb753d21eaf2a0481fd6f0ce6840a47300a40dcec0b56bac27d0f97"}, - {file = "zope_interface-8.4-cp313-cp313-win_amd64.whl", hash = "sha256:3e5866917ccb57d929e515a1136d729bd3fa4f367965fb16e38a4bc72cb05521"}, - {file = "zope_interface-8.4-cp313-cp313-win_arm64.whl", hash = "sha256:f1f854bef8bc137519e4413bcc1322d55faad28b20b3ca39f7bec49d2f1b26df"}, - {file = "zope_interface-8.4-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:7cbb887fdbfaacb4c362dbb487033551646e28013ad5ffe72e96eb260003a1a1"}, - {file = "zope_interface-8.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a5638c6be715116d3453e6d099c299c6844d54810de7445ce116424e905ede06"}, - {file = "zope_interface-8.4-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:b8147b40bfcd53803870a9519e0879ff066aeecc2fcff8295663c1b17fc38dc2"}, - {file = "zope_interface-8.4-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:049ba3c7b38cc400ae08e011617635706e0f442e1d075db1b015246fcbf6091e"}, - {file = "zope_interface-8.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9c4ac009c2c8e43283842f80387c4d4b41bcbc293391c3b9ab71532ae1ccc301"}, - {file = "zope_interface-8.4-cp314-cp314-win_amd64.whl", hash = "sha256:4713bf651ec36e7eea49d2ace4f0e89bec2b33a339674874b1121f2537edc62a"}, - {file = "zope_interface-8.4-cp314-cp314-win_arm64.whl", hash = "sha256:d934497c4b72d5f528d2b5ebe9b8b5a7004b5877948ebd4ea00c2432fb27178f"}, - {file = "zope_interface-8.4.tar.gz", hash = "sha256:9dbee7925a23aa6349738892c911019d4095a96cff487b743482073ecbc174a8"}, + {file = "zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe"}, + {file = "zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c"}, + {file = "zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37"}, + {file = "zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a"}, + {file = "zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58"}, + {file = "zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170"}, + {file = "zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057"}, + {file = "zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f"}, + {file = "zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376"}, + {file = "zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f"}, + {file = "zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2"}, + {file = "zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955"}, + {file = "zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646"}, + {file = "zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9"}, + {file = "zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c"}, + {file = "zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e"}, + {file = "zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560"}, + {file = "zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5"}, + {file = "zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e"}, + {file = "zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b"}, + {file = "zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52"}, + {file = "zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace"}, + {file = "zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb"}, + {file = "zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b"}, + {file = "zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4"}, + {file = "zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3"}, + {file = "zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e"}, + {file = "zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784"}, + {file = "zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f"}, + {file = "zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21"}, + {file = "zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8"}, + {file = "zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d"}, + {file = "zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140"}, + {file = "zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c"}, + {file = "zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714"}, + {file = "zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304"}, + {file = "zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08"}, + {file = "zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5"}, + {file = "zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f"}, + {file = "zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42"}, + {file = "zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d"}, ] [package.extras] From 37079f69e5bea2aea25301f8c17f14bdb4d4f29b Mon Sep 17 00:00:00 2001 From: Bo Li <22713281+bobboli@users.noreply.github.com> Date: Wed, 27 May 2026 11:46:51 +0800 Subject: [PATCH 045/308] [https://nvbugs/6215736][infra] Unwaive test_fp8_blockscale[throughput_mtp] (#14541) Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 923063a16176..c1ce08ae3578 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -19,7 +19,6 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp] SKIP (https://nvbugs/6215736) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6018046) From c61705ec6b71a45a2d592ede0fad06dc6eb0001c Mon Sep 17 00:00:00 2001 From: ruodil <200874449+ruodil@users.noreply.github.com> Date: Wed, 27 May 2026 13:21:26 +0800 Subject: [PATCH 046/308] [https://nvbugs/6175923][test] Revert gpt_oss_20b perf MoE-backend pin (#14612) Signed-off-by: Ruodi Lu Co-authored-by: Ruodi Lu --- .../defs/perf/pytorch_model_config.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 9ca366e03b3c..7fc7d9a67c53 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -314,29 +314,6 @@ def get_model_yaml_config(model_label: str, } } }, - # GPT-OSS 20B (NVBug 5720470: MMHA vs XQA kernel regression) - { - 'patterns': [ - 'gpt_oss_20b_fp4-bench-pytorch-float4', - ], - 'config': { - 'cuda_graph_config': { - 'max_batch_size': 512, - 'enable_padding': True, - }, - 'enable_chunked_prefill': False, - 'enable_attention_dp': False, - 'disable_overlap_scheduler': False, - 'kv_cache_config': { - 'enable_block_reuse': False, - 'free_gpu_memory_fraction': 0.9, - }, - 'moe_config': { - 'backend': 'TRITON' - }, - 'print_iter_log': True, - } - }, # GPT-OSS 120B max throughput test { 'patterns': [ From da0359b27b7c455e70e5be1bb8fa882857498b4a Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Wed, 27 May 2026 13:45:00 +0800 Subject: [PATCH 047/308] [https://nvbugs/6221621][test] Update trust_remote to nemotron and phi4 models (#14570) Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tests/integration/defs/perf/test_perf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index df1b01128efb..9550f1f502a0 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -64,6 +64,10 @@ "glm_5_fp8", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", + "nemotron_nano_12b_v2", + "phi_4_multimodal_instruct", + "phi_4_multimodal_instruct_fp4", + "phi_4_multimodal_instruct_fp8", } # Models that use random_image dataset in serve mode benchmarks. From 4db70515a6e6fa29ccb6dc07e49801d888245e01 Mon Sep 17 00:00:00 2001 From: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> Date: Wed, 27 May 2026 13:48:47 +0800 Subject: [PATCH 048/308] [None][chore] update VisualGen codeowner settings (#14530) Signed-off-by: Zhenhua Wang Signed-off-by: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> --- .github/CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6cd41f76b04e..92c937606bd2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -43,11 +43,13 @@ # TensorRT-LLM Pytorch backend /tensorrt_llm/_torch @NVIDIA/trt-llm-torch-devs -## TensorRT-LLM Pytorch - Visual Gen +## TensorRT-LLM Pytorch - VisualGen /tensorrt_llm/_torch/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs -/tests/unittest/_torch/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs +/tensorrt_llm/visual_gen @NVIDIA/trt-llm-llmapi-devs /tests/integration/defs/examples/test_visual_gen.py @NVIDIA/trt-llm-torch-visual-gen-devs -/tests/scripts/perf-sanity/visual_gen/ @NVIDIA/trt-llm-torch-visual-gen-devs +/tests/scripts/perf-sanity/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs +/tests/unittest/_torch/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs +/tests/integration/defs/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs ## TensorRT-LLM Pytorch - Modules /tensorrt_llm/_torch/modules @NVIDIA/trt-llm-torch-modules @@ -196,11 +198,9 @@ docs/source/performance/perf-benchmarking.md @NVIDIA/trtllm-bench-reviewers /tensorrt_llm/executor @NVIDIA/trt-llm-llmapi-devs /tensorrt_llm/serve @NVIDIA/trt-llm-llmapi-devs /tensorrt_llm/commands @NVIDIA/trt-llm-llmapi-devs -/tensorrt_llm/visual_gen @NVIDIA/trt-llm-llmapi-devs ## TensorRT-LLM Multimodal - LLM API & Serving /tensorrt_llm/llmapi/mm_encoder.py @NVIDIA/trt-llm-multimodal-devs @NVIDIA/trt-llm-llmapi-devs -/tensorrt_llm/serve/media_storage.py @NVIDIA/trt-llm-multimodal-devs @NVIDIA/trt-llm-llmapi-devs ## TensorRT-LLM Multimodal - Integration Tests /tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @NVIDIA/trt-llm-multimodal-devs @NVIDIA/trt-llm-qa-function From a0cdb8ce6a76ba9cb11a5c13eb68d8ec2c26895b Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Wed, 27 May 2026 13:50:22 +0800 Subject: [PATCH 049/308] [None][infra] Waive 8 failed cases for main in post-merge 2738 (#14615) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c1ce08ae3578..058a3d431b51 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -17,6 +17,8 @@ accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (h accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16] SKIP (https://nvbugs/6162114) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) +accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] SKIP (https://nvbugs/6221483) +accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4] SKIP (https://nvbugs/6221483) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) @@ -308,6 +310,9 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_12 perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k] SKIP (https://nvbugs/6167060) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k] SKIP (https://nvbugs/6190071) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6189928) +perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k] SKIP (https://nvbugs/6227472) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k] SKIP (https://nvbugs/6227472) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6192201) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) @@ -316,9 +321,11 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6016528) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6221024) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6200257) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6221022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) @@ -345,6 +352,7 @@ test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] SKIP (https://nvbugs/6190759) test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] SKIP (https://nvbugs/6115562) +test_e2e.py::test_openai_kv_cache_contamination SKIP (https://nvbugs/6227203) test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] SKIP (https://nvbugs/5448523) test_e2e.py::test_trtllm_multimodal_benchmark_serving SKIP (https://nvbugs/5864769) From 6481d4caa9afc0ab5fe6c99ad7b32e0723ccaa3c Mon Sep 17 00:00:00 2001 From: Guoming Zhang <137257613+nv-guomingz@users.noreply.github.com> Date: Wed, 27 May 2026 13:54:25 +0800 Subject: [PATCH 050/308] [None][perf] Fuse FlashInfer GDN prefill state I/O into Triton kernels (#14548) Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- .../_torch/modules/fla/flashinfer_chunk.py | 37 ++- .../_torch/modules/fla/fused_state_io.py | 276 ++++++++++++++++++ .../_torch/modules/mamba/gdn_mixer.py | 6 +- .../modules/mamba/test_fused_state_io.py | 207 +++++++++++++ 4 files changed, 512 insertions(+), 14 deletions(-) create mode 100644 tensorrt_llm/_torch/modules/fla/fused_state_io.py create mode 100644 tests/unittest/_torch/modules/mamba/test_fused_state_io.py diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py index 20bc49119037..0cef96bd3ad9 100644 --- a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -34,6 +34,10 @@ import torch +from tensorrt_llm._torch.modules.fla.fused_state_io import ( + gather_cast_transpose_kv_to_fp32_vk, + transpose_cast_scatter_fp32_vk_to_kv, +) from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd @@ -105,13 +109,10 @@ def chunk_gated_delta_rule( # --- Step 4: gather initial state and convert layout ----------------- # TRT-LLM stores SSM state with last two dims as (K, V); FlashInfer expects - # (V, K). Transpose last two dims and make contiguous so the kernel reads - # the buffer in the layout it expects. - if initial_state_indices is not None: - gathered_init = initial_state[initial_state_indices].to(torch.float32) - else: - gathered_init = initial_state.to(torch.float32) - gathered_init = gathered_init.transpose(-1, -2).contiguous() + # (V, K). Fuse gather + cast-to-fp32 + transpose + contiguous into a single + # Triton kernel so we get one HBM pass and one launch instead of three + # (``[indices]`` gather, ``.to(fp32)`` cast, ``.transpose.contiguous()`` copy). + gathered_init = gather_cast_transpose_kv_to_fp32_vk(initial_state, initial_state_indices) # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its @@ -160,15 +161,25 @@ def chunk_gated_delta_rule( out_state = None # --- Step 7: convert state layout back, scatter / return ----------- - # Convert FlashInfer's (V, K) state layout back to TRT-LLM's (K, V) before - # scattering to the SSM pool / returning to the caller. + # Fuse transpose + cast (fp32 → initial_state.dtype) + optional indexed + # scatter into a single Triton pass, mirroring Step 4. The non-inplace + # branch allocates a fresh (K, V) buffer and writes every row; the inplace + # branch writes only the slots named by ``initial_state_indices`` and + # leaves the rest of ``initial_state`` untouched. if inplace_indexed_state_update: - out_state_kv = out_state.transpose(-1, -2).contiguous() - initial_state[initial_state_indices] = out_state_kv.to(initial_state.dtype, copy=False) + transpose_cast_scatter_fp32_vk_to_kv(out_state, initial_state, initial_state_indices) final_to_return: Optional[torch.Tensor] = None elif output_final_state: - out_state_kv = out_state.transpose(-1, -2).contiguous() - final_to_return = out_state_kv.to(initial_state.dtype, copy=False) + num_seqs_out, num_h_out, v_out, k_out = out_state.shape + final_to_return = torch.empty( + num_seqs_out, + num_h_out, + k_out, + v_out, + dtype=initial_state.dtype, + device=out_state.device, + ) + transpose_cast_scatter_fp32_vk_to_kv(out_state, final_to_return, None) else: final_to_return = None diff --git a/tensorrt_llm/_torch/modules/fla/fused_state_io.py b/tensorrt_llm/_torch/modules/fla/fused_state_io.py new file mode 100644 index 000000000000..05aa06f4a607 --- /dev/null +++ b/tensorrt_llm/_torch/modules/fla/fused_state_io.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fused state I/O kernels for the FlashInfer GDN prefill adapter. + +The FlashInfer prefill kernel consumes the SSM state in fp32 with the last +two dims transposed (V, K) relative to TRT-LLM's native (K, V) layout, and +optionally requires gathering a subset of the SSM pool by index. The naive +PyTorch chain is + + gathered_init = initial_state[indices].to(torch.float32) # gather + cast + gathered_init = gathered_init.transpose(-1, -2).contiguous() # transpose copy + +which produces three separate kernel launches and two large intermediate +fp32 buffers (~``num_seqs * H * K * V * 4`` bytes each) per FlashInfer +call. ``gather_cast_transpose_kv_to_fp32_vk`` fuses all three steps into a +single Triton kernel. +""" + +from typing import Optional + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _gather_cast_transpose_kv_to_fp32_vk_kernel( + src_ptr, + dst_ptr, + indices_ptr, + HAS_INDICES: tl.constexpr, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + src_stride_n, + src_stride_h, + src_stride_k, + src_stride_v, + dst_stride_n, + dst_stride_h, + dst_stride_v, + dst_stride_k, + BLOCK_K: tl.constexpr, + BLOCK_V: tl.constexpr, +): + """Read src[indices[seq], h, k, v] (any dtype) → write dst[seq, h, v, k] (fp32). + + Each program instance handles one (seq, head, K-block, V-block) tile. + """ + pid_seq = tl.program_id(0) + pid_h = tl.program_id(1) + pid_kv = tl.program_id(2) + + num_v_blocks: tl.constexpr = (V + BLOCK_V - 1) // BLOCK_V + pid_kb = pid_kv // num_v_blocks + pid_vb = pid_kv % num_v_blocks + + if HAS_INDICES: + src_seq = tl.load(indices_ptr + pid_seq).to(tl.int64) + else: + src_seq = pid_seq.to(tl.int64) + + k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) + v_offs = pid_vb * BLOCK_V + tl.arange(0, BLOCK_V) + k_mask = k_offs < K + v_mask = v_offs < V + mask = k_mask[:, None] & v_mask[None, :] + + src_addrs = ( + src_ptr + + src_seq * src_stride_n + + pid_h * src_stride_h + + k_offs[:, None] * src_stride_k + + v_offs[None, :] * src_stride_v + ) + src_data = tl.load(src_addrs, mask=mask, other=0.0) + src_data_fp32 = src_data.to(tl.float32) + + dst_addrs = ( + dst_ptr + + pid_seq * dst_stride_n + + pid_h * dst_stride_h + + v_offs[None, :] * dst_stride_v + + k_offs[:, None] * dst_stride_k + ) + tl.store(dst_addrs, src_data_fp32, mask=mask) + + +def gather_cast_transpose_kv_to_fp32_vk( + initial_state: torch.Tensor, + initial_state_indices: Optional[torch.Tensor], +) -> torch.Tensor: + """Fused ``initial_state[indices].to(fp32).transpose(-1, -2).contiguous()``. + + Args: + initial_state: ``[N_pool, H, K, V]`` of any float dtype. + initial_state_indices: optional ``[num_seqs]`` int tensor. ``None`` means + "use the whole pool" (equivalent to identity gather). + + Returns: + ``[num_seqs, H, V, K]`` fp32 contiguous, where ``num_seqs == len(indices)`` + when ``indices`` is provided, otherwise ``num_seqs == N_pool``. + """ + assert initial_state.dim() == 4, f"initial_state must be 4D, got {initial_state.shape}" + N_pool, H, K, V = initial_state.shape + + if initial_state_indices is not None: + num_seqs = initial_state_indices.shape[0] + has_indices = True + else: + num_seqs = N_pool + has_indices = False + + output = torch.empty(num_seqs, H, V, K, dtype=torch.float32, device=initial_state.device) + + # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the + # entire (K, V) plane per (seq, head). Larger tiles save grid overhead; + # smaller tiles improve occupancy at small num_seqs * H. + BLOCK_K = min(K, 128) + BLOCK_V = min(V, 128) + num_k_blocks = (K + BLOCK_K - 1) // BLOCK_K + num_v_blocks = (V + BLOCK_V - 1) // BLOCK_V + + grid = (num_seqs, H, num_k_blocks * num_v_blocks) + + _gather_cast_transpose_kv_to_fp32_vk_kernel[grid]( + initial_state, + output, + initial_state_indices, + HAS_INDICES=has_indices, + H=H, + K=K, + V=V, + src_stride_n=initial_state.stride(0), + src_stride_h=initial_state.stride(1), + src_stride_k=initial_state.stride(2), + src_stride_v=initial_state.stride(3), + dst_stride_n=output.stride(0), + dst_stride_h=output.stride(1), + dst_stride_v=output.stride(2), + dst_stride_k=output.stride(3), + BLOCK_K=BLOCK_K, + BLOCK_V=BLOCK_V, + ) + return output + + +@triton.jit +def _transpose_cast_scatter_fp32_vk_to_kv_kernel( + src_ptr, + dst_ptr, + indices_ptr, + HAS_INDICES: tl.constexpr, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + src_stride_n, + src_stride_h, + src_stride_v, + src_stride_k, + dst_stride_n, + dst_stride_h, + dst_stride_k, + dst_stride_v, + BLOCK_K: tl.constexpr, + BLOCK_V: tl.constexpr, +): + """Read src[seq, h, v, k] (fp32 V,K layout) → write dst[dst_seq, h, k, v] (dst dtype, K,V layout). + + ``dst_seq = indices[seq] if HAS_INDICES else seq``. Casts to ``dst_ptr``'s element dtype. + """ + pid_seq = tl.program_id(0) + pid_h = tl.program_id(1) + pid_kv = tl.program_id(2) + + num_v_blocks: tl.constexpr = (V + BLOCK_V - 1) // BLOCK_V + pid_kb = pid_kv // num_v_blocks + pid_vb = pid_kv % num_v_blocks + + if HAS_INDICES: + dst_seq = tl.load(indices_ptr + pid_seq).to(tl.int64) + else: + dst_seq = pid_seq.to(tl.int64) + + k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) + v_offs = pid_vb * BLOCK_V + tl.arange(0, BLOCK_V) + k_mask = k_offs < K + v_mask = v_offs < V + mask = k_mask[:, None] & v_mask[None, :] + + src_addrs = ( + src_ptr + + pid_seq * src_stride_n + + pid_h * src_stride_h + + v_offs[None, :] * src_stride_v + + k_offs[:, None] * src_stride_k + ) + src_data = tl.load(src_addrs, mask=mask, other=0.0) + src_data_cast = src_data.to(dst_ptr.dtype.element_ty) + + dst_addrs = ( + dst_ptr + + dst_seq * dst_stride_n + + pid_h * dst_stride_h + + k_offs[:, None] * dst_stride_k + + v_offs[None, :] * dst_stride_v + ) + tl.store(dst_addrs, src_data_cast, mask=mask) + + +def transpose_cast_scatter_fp32_vk_to_kv( + src_vk: torch.Tensor, + dst: torch.Tensor, + scatter_indices: Optional[torch.Tensor] = None, +) -> None: + """Fused ``src.transpose(-1, -2).contiguous().to(dst.dtype)`` + optional indexed scatter. + + Reverse of :func:`gather_cast_transpose_kv_to_fp32_vk`. Writes ``src_vk`` into + ``dst`` in TRT-LLM's (K, V) last-two-dims layout, casting to ``dst.dtype``, + in a single Triton pass. + + Args: + src_vk: ``[num_seqs, H, V, K]`` fp32 (FlashInfer output_state layout). + dst: target tensor with last two dims ``(K, V)``. Two cases: + + * ``scatter_indices is None``: ``dst.shape == [num_seqs, H, K, V]`` and + every row is written. + * ``scatter_indices is not None``: ``dst.shape == [N_pool, H, K, V]`` + and ``dst[scatter_indices[i]]`` gets row ``i`` of ``src_vk``; + other rows of ``dst`` are untouched. + + scatter_indices: optional ``[num_seqs]`` int tensor. + + Returns: + ``None``; writes happen in-place into ``dst``. + """ + assert src_vk.dim() == 4, f"src_vk must be 4D, got {src_vk.shape}" + assert dst.dim() == 4, f"dst must be 4D, got {dst.shape}" + num_seqs, H, V, K = src_vk.shape + assert dst.shape[1:] == ( + H, + K, + V, + ), f"dst shape {tuple(dst.shape)} incompatible with src {tuple(src_vk.shape)}" + if scatter_indices is not None: + assert scatter_indices.shape == (num_seqs,), ( + f"scatter_indices shape {tuple(scatter_indices.shape)} must match num_seqs={num_seqs}" + ) + + has_indices = scatter_indices is not None + BLOCK_K = min(K, 128) + BLOCK_V = min(V, 128) + num_k_blocks = (K + BLOCK_K - 1) // BLOCK_K + num_v_blocks = (V + BLOCK_V - 1) // BLOCK_V + + grid = (num_seqs, H, num_k_blocks * num_v_blocks) + + _transpose_cast_scatter_fp32_vk_to_kv_kernel[grid]( + src_vk, + dst, + scatter_indices, + HAS_INDICES=has_indices, + H=H, + K=K, + V=V, + src_stride_n=src_vk.stride(0), + src_stride_h=src_vk.stride(1), + src_stride_v=src_vk.stride(2), + src_stride_k=src_vk.stride(3), + dst_stride_n=dst.stride(0), + dst_stride_h=dst.stride(1), + dst_stride_k=dst.stride(2), + dst_stride_v=dst.stride(3), + BLOCK_K=BLOCK_K, + BLOCK_V=BLOCK_V, + ) diff --git a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py index d2550a1505db..a6882d11fbae 100644 --- a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py @@ -256,7 +256,11 @@ def fused_gdn_gating_with_sigmoid( seq_len = 1 grid = (batch, seq_len, triton.cdiv(num_heads, 8)) g = torch.empty_like(a, dtype=torch.float32) - beta_out = torch.empty_like(b) + # Allocate beta in fp32 since (1) the kernel already computes sigmoid in fp32 + # and was previously casting back to b.dtype only to be re-cast to fp32 by the + # FlashInfer GDN prefill wrapper, and (2) the Triton chunk_gated_delta_rule + # path also accepts fp32 beta. Eliminates a redundant cast in the FI hot path. + beta_out = torch.empty_like(b, dtype=torch.float32) fused_gdn_gating_with_sigmoid_kernel[grid]( g, beta_out, diff --git a/tests/unittest/_torch/modules/mamba/test_fused_state_io.py b/tests/unittest/_torch/modules/mamba/test_fused_state_io.py new file mode 100644 index 000000000000..1de2d8e91ef0 --- /dev/null +++ b/tests/unittest/_torch/modules/mamba/test_fused_state_io.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Operator-level tests for the fused state I/O Triton kernels. + +Covers ``gather_cast_transpose_kv_to_fp32_vk`` and +``transpose_cast_scatter_fp32_vk_to_kv`` from +``tensorrt_llm._torch.modules.fla.fused_state_io``, comparing each against +the equivalent PyTorch chain (``[indices].to(fp32).transpose(-1,-2).contiguous()`` +and its inverse). Both kernels are pure layout-transform + dtype cast, so +the expected tolerance is **bit-exact** when input precision is preserved +through the chain. +""" + +import pytest +import torch + + +def _supported_arch() -> bool: + if not torch.cuda.is_available(): + return False + major, _ = torch.cuda.get_device_capability(0) + # SM90 (Hopper) or SM100 (Blackwell) + return major in (9, 10) + + +skip_unsupported = pytest.mark.skipif( + not _supported_arch(), + reason="Fused state I/O kernels target SM90 (Hopper) or SM100 (Blackwell)", +) + + +# Reference (un-fused) implementations ------------------------------------- + + +def _ref_gather_cast_transpose(initial_state, indices): + """PyTorch reference: ``init[idx].to(fp32).transpose(-1, -2).contiguous()``.""" + if indices is not None: + gathered = initial_state[indices].to(torch.float32) + else: + gathered = initial_state.to(torch.float32) + return gathered.transpose(-1, -2).contiguous() + + +def _ref_transpose_cast_scatter(src_vk, dst, scatter_indices): + """PyTorch reference: ``src.transpose(-1, -2).contiguous().to(dst.dtype)`` + scatter. + + Mutates ``dst`` in place. Returns nothing (matches the fused kernel API). + """ + out_kv = src_vk.transpose(-1, -2).contiguous().to(dst.dtype, copy=False) + if scatter_indices is not None: + dst[scatter_indices] = out_kv + else: + dst.copy_(out_kv) + + +# Forward kernel: gather + cast + transpose -------------------------------- + + +@skip_unsupported +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) +@pytest.mark.parametrize( + "shape,num_seqs", + [ + ((16, 4, 128, 128), 2), + ((1, 4, 128, 128), 1), + ((32, 16, 128, 128), 8), + ((8, 4, 64, 128), 3), # K != V + ((8, 4, 128, 64), 3), # K != V + ], +) +def test_gather_cast_transpose_matches_torch_ref(dtype, shape, num_seqs): + """Indexed gather path — output must be bit-exact vs the PyTorch chain.""" + from tensorrt_llm._torch.modules.fla.fused_state_io import gather_cast_transpose_kv_to_fp32_vk + + N_pool, H, K, V = shape + torch.manual_seed(0) + src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) + indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] + + ref = _ref_gather_cast_transpose(src, indices) + out = gather_cast_transpose_kv_to_fp32_vk(src, indices) + + assert out.shape == (num_seqs, H, V, K) + assert out.dtype == torch.float32 + assert out.is_contiguous() + torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0) + + +@skip_unsupported +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +def test_gather_cast_transpose_no_indices_full_pool(dtype): + """``indices=None`` must process the entire pool, output shape == input shape (with last two transposed).""" + from tensorrt_llm._torch.modules.fla.fused_state_io import gather_cast_transpose_kv_to_fp32_vk + + N_pool, H, K, V = 12, 4, 128, 128 + torch.manual_seed(1) + src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) + + ref = _ref_gather_cast_transpose(src, None) + out = gather_cast_transpose_kv_to_fp32_vk(src, None) + + assert out.shape == (N_pool, H, V, K) + torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0) + + +# Reverse kernel: transpose + cast + scatter ------------------------------- + + +@skip_unsupported +@pytest.mark.parametrize("dst_dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize( + "N_pool,num_seqs,H,K,V", + [ + (16, 2, 4, 128, 128), + (32, 8, 16, 128, 128), + (8, 3, 4, 64, 128), # K != V + ], +) +def test_transpose_cast_scatter_inplace_preserves_untouched(dst_dtype, N_pool, num_seqs, H, K, V): + """Scattered slots match torch ref; untouched slots are bit-exact unchanged.""" + from tensorrt_llm._torch.modules.fla.fused_state_io import transpose_cast_scatter_fp32_vk_to_kv + + torch.manual_seed(2) + src_vk = torch.randn(num_seqs, H, V, K, device="cuda", dtype=torch.float32) * 0.1 + pool_init = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dst_dtype) + indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] + + pool_ref = pool_init.clone() + _ref_transpose_cast_scatter(src_vk, pool_ref, indices) + + pool_out = pool_init.clone() + transpose_cast_scatter_fp32_vk_to_kv(src_vk, pool_out, indices) + + # The touched slots must match the torch ref bit-exact (RNE cast is deterministic). + torch.testing.assert_close(pool_out[indices], pool_ref[indices], atol=0.0, rtol=0.0) + # Untouched slots must remain bit-exact to the initial pool. + untouched = [i for i in range(N_pool) if i not in indices.tolist()] + if untouched: + torch.testing.assert_close(pool_out[untouched], pool_init[untouched], atol=0.0, rtol=0.0) + + +@skip_unsupported +@pytest.mark.parametrize("dst_dtype", [torch.bfloat16, torch.float16, torch.float32]) +def test_transpose_cast_scatter_full_write(dst_dtype): + """``scatter_indices=None`` must write every row; output equals torch ref.""" + from tensorrt_llm._torch.modules.fla.fused_state_io import transpose_cast_scatter_fp32_vk_to_kv + + num_seqs, H, K, V = 5, 4, 128, 128 + torch.manual_seed(3) + src_vk = torch.randn(num_seqs, H, V, K, device="cuda", dtype=torch.float32) * 0.1 + dst_init = torch.randn(num_seqs, H, K, V, device="cuda").to(dst_dtype) + + dst_ref = dst_init.clone() + _ref_transpose_cast_scatter(src_vk, dst_ref, None) + + dst_out = dst_init.clone() + transpose_cast_scatter_fp32_vk_to_kv(src_vk, dst_out, None) + + torch.testing.assert_close(dst_out, dst_ref, atol=0.0, rtol=0.0) + + +# Roundtrip ---------------------------------------------------------------- + + +@skip_unsupported +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_roundtrip_gather_then_scatter_bitexact(dtype): + """``gather(src, idx) → scatter back to same idx`` must restore src bit-exact. + + Justification: bf16/fp16 → fp32 is lossless (precision strictly extends); + fp32 → bf16/fp16 via RNE is deterministic and recovers the original value + when the fp32 representation came from an unmodified bf16/fp16 source. + """ + from tensorrt_llm._torch.modules.fla.fused_state_io import ( + gather_cast_transpose_kv_to_fp32_vk, + transpose_cast_scatter_fp32_vk_to_kv, + ) + + N_pool, H, K, V = 16, 4, 128, 128 + num_seqs = 5 + torch.manual_seed(4) + src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) + indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] + + # gather → vk fp32 + vk = gather_cast_transpose_kv_to_fp32_vk(src, indices) + # scatter back to a fresh pool, same indices + restored = src.clone() + transpose_cast_scatter_fp32_vk_to_kv(vk, restored, indices) + + # The selected slots should be identical to the original (RNE roundtrip is lossless + # when no math was performed in fp32). + torch.testing.assert_close(restored[indices], src[indices], atol=0.0, rtol=0.0) + # Other slots must be unchanged (we passed in restored = src.clone()). + untouched = [i for i in range(N_pool) if i not in indices.tolist()] + if untouched: + torch.testing.assert_close(restored[untouched], src[untouched], atol=0.0, rtol=0.0) + + +# Import smoke (no GPU required) ------------------------------------------ + + +def test_module_importable(): + from tensorrt_llm._torch.modules.fla.fused_state_io import ( # noqa: F401 + gather_cast_transpose_kv_to_fp32_vk, + transpose_cast_scatter_fp32_vk_to_kv, + ) From 272c10148e519917ade886440cb2ce3e4df03df6 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Wed, 27 May 2026 14:52:54 +0800 Subject: [PATCH 051/308] [https://nvbugs/6164924][fix] Lower free_gpu_memory_fraction for Exaone tests (#14486) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 9 +++++++-- tests/integration/test_lists/waives.txt | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 9e24f1a8328e..9feb0d48d42b 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -6084,10 +6084,15 @@ def test_w4_4gpus_online_eplb(self, kv_cache_dtype, mocker): @skip_pre_hopper class TestEXAONE4(LlmapiAccuracyTestHarness): MODEL_NAME = "LGAI-EXAONE/EXAONE-4.0-32B" + # 32B BF16 leaves little headroom on 80 GiB H100; reduce KV cache and + # use expandable_segments to avoid autotuner warmup OOM (nvbugs/6164924). kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False) + enable_partial_reuse=False, + free_gpu_memory_fraction=0.8) - def test_auto_dtype(self): + def test_auto_dtype(self, mocker): + patch_mpi_pool_session_for_env( + mocker, {"PYTORCH_ALLOC_CONF": "expandable_segments:True"}) model_path = f"{llm_models_root()}/EXAONE-4.0-32B" with LLM(model_path, kv_cache_config=self.kv_cache_config) as llm: task = MMLU(self.MODEL_NAME) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 058a3d431b51..0dab7b11186f 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -72,7 +72,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRT accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5945081) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) -accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype SKIP (https://nvbugs/6164924) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/5981293) From 46bf87cfec1d494f446c47b4887f1bfc8ad62165 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Wed, 27 May 2026 15:09:37 +0800 Subject: [PATCH 052/308] [https://nvbugs/6163033][fix] Guard `q_a_proj.weight` dict access behind `nvfp4_fused_a`; update test to `chec (#14033) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: bhsueh_NV <11360707+byshiue@users.noreply.github.com> --- .../_torch/models/modeling_deepseekv3.py | 4 ++-- .../_torch/models/modeling_mistral.py | 20 ++++++++++++------- .../defs/accuracy/test_llm_api_pytorch.py | 2 +- tests/integration/test_lists/waives.txt | 1 - 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index e64f1b4b90fb..e981dc233010 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -440,8 +440,8 @@ def detect_shared_mtp_weights() -> bool: # Non-lite models (V3, R1, V3.2) fuse q_a_proj into # kv_a_proj_with_mqa, so both must be NVFP4 for the fused # path. Lite models (V3-Lite) have no q_a_proj. - if not is_lite: - nvfp4_fused_a &= weights[ + if not is_lite and nvfp4_fused_a: + nvfp4_fused_a = weights[ f"{'.'.join(names[:-1])}.q_a_proj.weight"].dtype == fp4_utils.float4_e2m1x2 if nvfp4_fused_a: ########### input_scale diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 9a7c6337f1d7..77dbcf1d0f31 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -310,13 +310,19 @@ def get_num_tokens_per_image(self, image_size): Image.new("RGB", (w, h))) return ncols * nrows + nrows - def __call__(self, text, images, **kwargs): - mm_items = [] - if images: - mm_items = [{ - "type": "image", - "base64": encode_base64_image(image) - } for image in images] + def __call__(self, text, images=None, **kwargs): + if not images: + # Plain-text inputs (e.g. text-only evaluation like MMLU/GSM8K): tokenize + # directly without wrapping in a multi-modal chat conversation, which would + # otherwise inject chat-template tokens and corrupt continuation prompts. + encoded = self.tokenizer.transformers_tokenizer(text, + return_tensors='pt') + return {"input_ids": encoded["input_ids"]} + + mm_items = [{ + "type": "image", + "base64": encode_base64_image(image) + } for image in images] conversation = [{ "role": "user", diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 9feb0d48d42b..24539016f67a 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -6956,7 +6956,7 @@ def test_fp8(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, eagle3_model_arch="mistral_large3") with LLM( f"{llm_models_root()}/Mistral-Large-3-675B/Mistral-Large-3-675B-Instruct-2512/", - checkpoint_format="mistral", + checkpoint_format="mistral_large_3", tensor_parallel_size=tp_size, pipeline_parallel_size=pp_size, moe_expert_parallel_size=ep_size, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 0dab7b11186f..dba3559fb9c6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -133,7 +133,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6163033) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6162121) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) From bb168829398dc2b5e6d61e8417ab33cf478b89f2 Mon Sep 17 00:00:00 2001 From: Joyjit Daw <1127155+tijyojwad@users.noreply.github.com> Date: Wed, 27 May 2026 04:22:41 -0400 Subject: [PATCH 053/308] [None][fix] Bypass FlashInfer SSD prefill to fix state dtype precision (#14600) Signed-off-by: tijyojwad <1127155+tijyojwad@users.noreply.github.com> --- tensorrt_llm/_torch/modules/mamba/ssd_combined.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/modules/mamba/ssd_combined.py b/tensorrt_llm/_torch/modules/mamba/ssd_combined.py index 35a836c872ff..efb02364d291 100644 --- a/tensorrt_llm/_torch/modules/mamba/ssd_combined.py +++ b/tensorrt_llm/_torch/modules/mamba/ssd_combined.py @@ -437,9 +437,16 @@ def mamba_chunk_scan_combined( # FlashInfer fused CUTLASS kernel on Blackwell (SM100+); both varlen and # non-varlen route here based on cu_seqlens. Falls back to Triton when the # MMA tile constraints on (chunk_size, dstate, headdim) aren't met. + # + # DISABLED: The FlashInfer SSD kernel hardcodes state_dtype=bf16 while the + # Triton path accumulates in fp32 (states_in_fp32=True), causing silent + # precision loss during prefill. + # TODO: Re-enable once _get_flashinfer_ssd_kernel uses state_dtype=float32. + _USE_FLASHINFER_SSD_PREFILL = False dstate = B.shape[-1] headdim = x.shape[-1] - flashinfer_eligible = (z is None and is_sm_100f()) + flashinfer_eligible = (_USE_FLASHINFER_SSD_PREFILL and z is None + and is_sm_100f()) if flashinfer_eligible and _flashinfer_ssd_supported( chunk_size, dstate, headdim): return _mamba_chunk_scan_flashinfer_fwd( From 5a5d093131d57b7e6c67a840b05dfa2178fad2fb Mon Sep 17 00:00:00 2001 From: William Zhang <133824995+2ez4bz@users.noreply.github.com> Date: Wed, 27 May 2026 01:24:38 -0700 Subject: [PATCH 054/308] [None][fix] Exclude Qwen3 VL vision model from quantization (#12851) Signed-off-by: William Zhang <133824995+2ez4bz@users.noreply.github.com> --- tensorrt_llm/_torch/models/modeling_qwen3vl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index ecdbc5fde4b3..4e8ec6e2db37 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -875,10 +875,10 @@ def __init__( self.model_config = model_config self.model_dtype = self.model_config.pretrained_config.text_config.dtype - # NOTE: Re-setting QuantConfig to exclude vision encoder weights from quantization load. - self.model_config.quant_config = QuantConfig( - kv_cache_quant_algo=self.model_config.quant_config.kv_cache_quant_algo - ) + # NOTE: Re-setting QuantConfig to exclude vision encoder from quantization, + # including KV cache quantization (vision encoder head dims may not be + # supported by FP8 FMHA kernels). + self.model_config.quant_config = QuantConfig() self.visual = model_class(self.model_config).to(self.model_dtype) From 8cd28ed0226fa7bc3e7b2a93aaef5f9bc507a54a Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Wed, 27 May 2026 16:39:01 +0800 Subject: [PATCH 055/308] [https://nvbugs/6162860][fix] Set free_gpu_memory_fraction=0.6 only when torch_compile=True for test_bfloat16_ (#14109) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 4 +++- tests/integration/test_lists/waives.txt | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 24539016f67a..db94d7943bdc 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1881,7 +1881,9 @@ def test_bfloat16_4gpus(self, tp_size, pp_size, ep_size, mtp_nextn, pp_partition[-1] = num_hidden_layers - sum(pp_partition[:-1]) else: pp_partition = None - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75) + free_gpu_memory_fraction = 0.6 if torch_compile else 0.75 + kv_cache_config = KvCacheConfig( + free_gpu_memory_fraction=free_gpu_memory_fraction) torch_compile_config = _get_default_torch_compile_config(torch_compile) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index dba3559fb9c6..3c5978a37277 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -39,7 +39,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewi accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[mtp3_fp8kv_chunked] SKIP (https://nvbugs/5989920) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6095851) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6162860) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) From 11dbd9deb4cd3c28810a55108886e55e0cc89d37 Mon Sep 17 00:00:00 2001 From: Bo Li <22713281+bobboli@users.noreply.github.com> Date: Wed, 27 May 2026 20:13:23 +0800 Subject: [PATCH 056/308] [None][chore] Remove one-warp-per-token policy from MoE A2A kernels (#14550) Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> --- cpp/tensorrt_llm/common/envUtils.cpp | 33 +++-- cpp/tensorrt_llm/common/envUtils.h | 5 +- .../moeAlltoAllKernels.cu | 138 ++++-------------- .../communicationKernels/moeAlltoAllKernels.h | 7 +- cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp | 6 +- 5 files changed, 49 insertions(+), 140 deletions(-) diff --git a/cpp/tensorrt_llm/common/envUtils.cpp b/cpp/tensorrt_llm/common/envUtils.cpp index bf3142a160dd..1a1b75e2f10e 100644 --- a/cpp/tensorrt_llm/common/envUtils.cpp +++ b/cpp/tensorrt_llm/common/envUtils.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -530,17 +530,6 @@ bool getEnvDisableChunkedAttentionInGenPhase() return getBoolEnv("TRTLLM_DISABLE_CHUNKED_ATTENTION_IN_GEN_PHASE"); } -bool getEnvMoeA2AOneBlockPerToken() -{ - // Default true; return false only if env set to "0" - static std::optional const val = getIntEnv("TLLM_MOE_A2A_ONE_BLOCK_PER_TOKEN"); - if (!val.has_value()) - { - return true; - } - return val.value() != 0; -} - static int sanitizeBlockSize(std::optional const& val) { // Default 256 when not set or invalid @@ -557,15 +546,31 @@ static int sanitizeBlockSize(std::optional const& val) return block; } +// Read an integer env var and sanitize it as a CUDA block size. Treats malformed +// values (e.g. non-numeric strings that would throw inside std::stoi) as unset and +// falls back to the default, so this debug knob never becomes a hard failure. +static int getSanitizedBlockSizeFromEnv(char const* name) +{ + try + { + return sanitizeBlockSize(getIntEnv(name)); + } + catch (std::exception const&) + { + TLLM_LOG_WARNING("Invalid value for %s. Falling back to default block size.", name); + return sanitizeBlockSize(std::nullopt); + } +} + int getEnvMoeA2ADispatchBlockSize() { - static int const kBlock = sanitizeBlockSize(getIntEnv("TLLM_MOE_A2A_DISPATCH_BLOCK_SIZE")); + static int const kBlock = getSanitizedBlockSizeFromEnv("TLLM_MOE_A2A_DISPATCH_BLOCK_SIZE"); return kBlock; } int getEnvMoeA2ACombineBlockSize() { - static int const kBlock = sanitizeBlockSize(getIntEnv("TLLM_MOE_A2A_COMBINE_BLOCK_SIZE")); + static int const kBlock = getSanitizedBlockSizeFromEnv("TLLM_MOE_A2A_COMBINE_BLOCK_SIZE"); return kBlock; } diff --git a/cpp/tensorrt_llm/common/envUtils.h b/cpp/tensorrt_llm/common/envUtils.h index 82c56f300362..832421ec60ef 100644 --- a/cpp/tensorrt_llm/common/envUtils.h +++ b/cpp/tensorrt_llm/common/envUtils.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -153,9 +153,6 @@ bool getEnvDisaggBenchmarkGenOnly(); // Whether to disable the chunked-attention in the generation phase. bool getEnvDisableChunkedAttentionInGenPhase(); -// Whether to use one block per token for MoE A2A kernels (default true). -bool getEnvMoeA2AOneBlockPerToken(); - // TODO: For DEV purpose temporarily. // Block size (threads per block) for MoE A2A Dispatch kernels (default 256 if unset or invalid) int getEnvMoeA2ADispatchBlockSize(); diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu index e1fd9bc7f08a..7b8c1ecd1723 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -159,18 +159,6 @@ using tensorrt_llm::common::launchWithPdlWhenEnabled; } \ } -#define SWITCH_POLICY(one_block_per_token, POLICY, ...) \ - if (one_block_per_token) \ - { \ - using POLICY = BlockPolicy; \ - __VA_ARGS__ \ - } \ - else \ - { \ - using POLICY = WarpPolicy; \ - __VA_ARGS__ \ - } - #if DISABLE_TIMEOUT #define check_timeout(s) false #else @@ -201,29 +189,6 @@ __device__ int compute_target_rank_id(int expert_id, int num_experts_per_rank) // Helper Functions for Vectorized Memory Operations // ============================================================================ -struct WarpPolicy -{ - __device__ static int stride() - { - return warpSize; - } - - __device__ static int offset() - { - return (threadIdx.x % warpSize); - } - - __device__ static int token_idx() - { - return (blockIdx.x * blockDim.x + threadIdx.x) / warpSize; - } - - __device__ static void sync() - { - __syncwarp(); - } -}; - struct BlockPolicy { __device__ static int stride() @@ -421,22 +386,10 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ if (local_token_idx >= local_num_tokens) return; - // Prepare per-policy shared-memory tiles for this token + // One block per token: a single shared-memory tile is reused by the entire CTA. extern __shared__ int smem[]; - int* smem_topk_target_ranks; - int* smem_topk_send_indices; - int warps_per_block = blockDim.x / warpSize; - if constexpr (std::is_same::value) - { - int lane_id = threadIdx.x / warpSize; - smem_topk_target_ranks = smem + lane_id * TOP_K; - smem_topk_send_indices = smem + warps_per_block * TOP_K + lane_id * TOP_K; - } - else - { - smem_topk_target_ranks = smem; - smem_topk_send_indices = smem + TOP_K; - } + int* smem_topk_target_ranks = smem; + int* smem_topk_send_indices = smem + TOP_K; uint64_t already_copied = 0; int num_experts_per_rank = num_experts / ep_size; @@ -660,44 +613,21 @@ void moe_a2a_dispatch_launch(MoeA2ADispatchParams const& params) kernel_ptrs.eplb_local_stats = params.eplb_local_stats; int const kBlockSize = tensorrt_llm::common::getEnvMoeA2ADispatchBlockSize(); - constexpr int kWarpSize = 32; - int const kWarpsPerBlock = kBlockSize / kWarpSize; - // Configure kernel launch - if (params.one_block_per_token) + // One block per token: grid_size == local_num_tokens. If 0, launch a single block to + // keep the synchronization path alive. + int grid_size = params.local_num_tokens; + if (grid_size == 0) { - int grid_size = params.local_num_tokens; - // If local_num_tokens is 0, we still need to launch a minimal kernel to participate in the synchronization. - if (grid_size == 0) - { - grid_size = 1; - } - int shared_bytes = 2 * params.top_k * (int) sizeof(int); - SWITCH_BOOL(params.enable_eplb, EPLB_STATS, SWITCH_TOP_K(params.top_k, TOP_K, { - auto kernel_fn = moeA2ADispatchKernel; - launchWithPdlWhenEnabled("moeA2ADispatchKernel", kernel_fn, grid_size, kBlockSize, shared_bytes, - params.stream, params.token_selected_experts, kernel_ptrs, params.num_payloads, - params.max_tokens_per_rank, params.local_num_tokens, params.ep_rank, params.ep_size, params.num_experts, - params.eplb_stats_num_experts); - })) - } - else - { - int grid_size = ceilDiv(params.local_num_tokens, kWarpsPerBlock); - // If local_num_tokens is 0, we still need to launch a minimal kernel to participate in the synchronization. - if (grid_size == 0) - { - grid_size = 1; - } - int shared_bytes = 2 * kWarpsPerBlock * params.top_k * (int) sizeof(int); - SWITCH_BOOL(params.enable_eplb, EPLB_STATS, SWITCH_TOP_K(params.top_k, TOP_K, { - auto kernel_fn = moeA2ADispatchKernel; - launchWithPdlWhenEnabled("moeA2ADispatchKernel", kernel_fn, grid_size, kBlockSize, shared_bytes, - params.stream, params.token_selected_experts, kernel_ptrs, params.num_payloads, - params.max_tokens_per_rank, params.local_num_tokens, params.ep_rank, params.ep_size, params.num_experts, - params.eplb_stats_num_experts); - })) + grid_size = 1; } + int shared_bytes = 2 * params.top_k * (int) sizeof(int); + SWITCH_BOOL(params.enable_eplb, EPLB_STATS, SWITCH_TOP_K(params.top_k, TOP_K, { + auto kernel_fn = moeA2ADispatchKernel; + launchWithPdlWhenEnabled("moeA2ADispatchKernel", kernel_fn, grid_size, kBlockSize, shared_bytes, params.stream, + params.token_selected_experts, kernel_ptrs, params.num_payloads, params.max_tokens_per_rank, + params.local_num_tokens, params.ep_rank, params.ep_size, params.num_experts, params.eplb_stats_num_experts); + })) } // ============================================================================ @@ -1272,7 +1202,6 @@ __global__ void moeA2ACombineKernel( void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params) { constexpr int kBlockSize = 256; - constexpr int kWarpsPerBlock = kBlockSize / 32; // 8 warps per block // FP8 in-place (payload_in_workspace=true, prepare_payload==nullptr): each CTA writes // FP8 at the BF16-stride position, so CTAs never race — all tokens must be processed. @@ -1280,9 +1209,7 @@ void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params) int global_token_num = (params.use_low_precision || params.prepare_payload != nullptr) ? params.ep_size * params.max_tokens_per_rank : 1; - int grid_size_warp = ceilDiv(global_token_num, kWarpsPerBlock); - int grid_size_block = global_token_num; // one block per token - int grid = params.one_block_per_token ? grid_size_block : grid_size_warp; + int grid = global_token_num; // one block per token uint8_t* recv_buffer_bytes = static_cast(const_cast(params.recv_buffers[params.ep_rank])); void const* payload = params.prepare_payload; @@ -1297,8 +1224,7 @@ void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params) int const stride_per_token = low_precision_staged ? params.elements_per_token : params.elements_per_token * static_cast(sizeof(SrcT)); - auto kernel_fn = params.one_block_per_token ? moeA2APrepareCombineKernel - : moeA2APrepareCombineKernel; + auto kernel_fn = moeA2APrepareCombineKernel; launchWithPdlWhenEnabled("moeA2APrepareCombineKernel", kernel_fn, grid, kBlockSize, 0, params.stream, recv_buffer_bytes, payload, params.elements_per_token, params.ep_size, params.max_tokens_per_rank, params.flag_val, params.recv_counters, stride_per_token); @@ -1318,19 +1244,13 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) TLLM_CHECK(params.local_num_tokens >= 0); TLLM_CHECK(params.elements_per_token > 0); - // Configure kernel launch + // Configure kernel launch (one block per token). int const kBlockSize = tensorrt_llm::common::getEnvMoeA2ACombineBlockSize(); - int const kWarpsPerBlock = kBlockSize / 32; // warpSize - int grid_size_warp = ceilDiv(params.local_num_tokens, kWarpsPerBlock); - int grid_size_block = params.local_num_tokens; + int grid = params.local_num_tokens; // If local_num_tokens is 0, we still need to launch a minimal kernel to participate in the synchronization. - if (grid_size_warp == 0) + if (grid == 0) { - grid_size_warp = 1; - } - if (grid_size_block == 0) - { - grid_size_block = 1; + grid = 1; } // Prepare kernel pointers struct for combine @@ -1356,8 +1276,6 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) kernel_ptrs.topk_target_ranks = params.topk_target_ranks; kernel_ptrs.topk_send_indices = params.topk_send_indices; - int grid = params.one_block_per_token ? grid_size_block : grid_size_warp; - // stride_per_token: byte distance between tokens in the recv buffer. // FP8 external payload: EPT × 1 (compact FP8 layout) // FP8 in-place / non-FP8: EPT × sizeof(PayloadT) (payload-dtype stride) @@ -1374,13 +1292,11 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) // Launch appropriate kernel with compact macros SWITCH_DTYPE(effective_dtype, TKernelType, { - SWITCH_POLICY(params.one_block_per_token, Policy, { - SWITCH_TOP_K(params.top_k, TOP_K, { - auto kernel_fn = moeA2ACombineKernel; - launchWithPdlWhenEnabled("moeA2ACombineKernel", kernel_fn, grid, kBlockSize, 0, params.stream, - kernel_ptrs, params.max_tokens_per_rank, params.elements_per_token, params.local_num_tokens, - params.ep_rank, params.ep_size, stride_per_token); - }); + SWITCH_TOP_K(params.top_k, TOP_K, { + auto kernel_fn = moeA2ACombineKernel; + launchWithPdlWhenEnabled("moeA2ACombineKernel", kernel_fn, grid, kBlockSize, 0, params.stream, kernel_ptrs, + params.max_tokens_per_rank, params.elements_per_token, params.local_num_tokens, params.ep_rank, + params.ep_size, stride_per_token); }); }); } diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h index aa2ae1fbd4dd..317ff4d2240c 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,9 +87,6 @@ struct CombineKernelPointers // Dispatch phase parameters struct MoeA2ADispatchParams { - // Threading policy - bool one_block_per_token; // True: one block per token, False: one warp per token - // EP configuration int ep_size; // Number of EP ranks int ep_rank; // Current EP rank @@ -140,8 +137,6 @@ void moe_a2a_prepare_dispatch_launch(MoeA2ADispatchParams const& params); // Combine phase parameters struct MoeA2ACombineParams { - bool one_block_per_token; // True: one block per token, False: one warp per token - // EP configuration int ep_size; // Number of EP ranks int ep_rank; // Current EP rank diff --git a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp index 4d3b396e23c9..eb74c27ffc89 100644 --- a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -304,8 +304,6 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( // Setup dispatch parameters MoeA2ADispatchParams params{}; - params.one_block_per_token - = tensorrt_llm::common::getEnvMoeA2AOneBlockPerToken(); // TODO: Decide this based on the workload params.ep_size = static_cast(epSize); params.ep_rank = static_cast(epRank); params.num_experts = static_cast(numExperts); @@ -492,8 +490,6 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke // Setup combine parameters MoeA2ACombineParams params{}; - params.one_block_per_token - = tensorrt_llm::common::getEnvMoeA2AOneBlockPerToken(); // TODO: Decide this based on the workload params.ep_size = static_cast(epSize); params.ep_rank = static_cast(epRank); params.local_num_tokens = static_cast(localNumTokens); From 09f59589f217fb7c2e956229fbdf5de30b5ad818 Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 27 May 2026 21:24:26 +0800 Subject: [PATCH 057/308] [None][test] Waive 7 failed cases for main in QA CI (#14498) Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Signed-off-by: Jie Li <76780849+jieli-matrix@users.noreply.github.com> Co-authored-by: Jie Li <76780849+jieli-matrix@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3c5978a37277..90a30f5ea7f6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -21,6 +21,7 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80g accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4] SKIP (https://nvbugs/6221483) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) +accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6018046) From 021e4d8039998e1ccd70815279d90002106ad5e4 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 27 May 2026 22:47:57 +0800 Subject: [PATCH 058/308] [None][doc] Add CUTLASS DSL uninstall step to installation guide (#14621) Signed-off-by: yihwang-nv --- docs/source/installation/installation-guide.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/installation/installation-guide.md b/docs/source/installation/installation-guide.md index 6281cf24e94b..76f65da3871b 100644 --- a/docs/source/installation/installation-guide.md +++ b/docs/source/installation/installation-guide.md @@ -64,6 +64,13 @@ image hosted on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-l Once all prerequisites are in place, TensorRT LLM can be installed as follows: +Before installing the latest version, uninstall any previous CUTLASS DSL installation as described in the +[CUTLASS DSL installation guide](https://docs.nvidia.com/cutlass/latest/media/docs/pythonDSL/quick_start.html#installation): + +```bash +pip3 uninstall nvidia-cutlass-dsl nvidia-cutlass-dsl-libs-base nvidia-cutlass-dsl-libs-cu13 +``` + ```bash pip3 install --ignore-installed pip setuptools wheel && pip3 install tensorrt_llm ``` From ec067d755dce09781b59b15bebad5ddcbec1ee7b Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Wed, 27 May 2026 23:07:48 +0800 Subject: [PATCH 059/308] [https://nvbugs/6099723][fix] Gate supports_mnnvl() False on SM120/121 in _mnnvl_utils.py and add the same Mnn (#14424) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: ziyixiong-nv <219238287+ziyixiong-nv@users.noreply.github.com> Co-authored-by: ziyixiong-nv <219238287+ziyixiong-nv@users.noreply.github.com> --- tensorrt_llm/_mnnvl_utils.py | 7 ++++++- .../fused_moe/communication/deep_ep_low_latency.py | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_mnnvl_utils.py b/tensorrt_llm/_mnnvl_utils.py index 06215292d6ca..af4157786c06 100644 --- a/tensorrt_llm/_mnnvl_utils.py +++ b/tensorrt_llm/_mnnvl_utils.py @@ -29,7 +29,7 @@ from cuda import cuda from ._dlpack_utils import pack_strided_memory -from ._utils import mpi_comm +from ._utils import get_sm_version, mpi_comm from .logger import logger from .mapping import Mapping @@ -382,6 +382,11 @@ def supports_mnnvl() -> bool: # We check if it has all NVLink up now. # But it is not equivalent to MNNVL support. # May need better support check. + # SM120/121 (RTX PRO 6000 Blackwell) lack NVSwitch fabric; MNNVL-class + # all-to-all kernels deadlock there even when local NVLink bridges + # report up. + if get_sm_version() in (120, 121): + return False dev_id = torch.cuda.current_device() support_nvlink_and_all_up = MnnvlMemory.support_nvlink(dev_id, True) return support_nvlink_and_all_up diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep_low_latency.py b/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep_low_latency.py index b34e5d6682d2..a37a3e6e1548 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep_low_latency.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/deep_ep_low_latency.py @@ -26,6 +26,7 @@ import torch from tensorrt_llm._torch.modules.fused_moe.deep_ep_utils import buffer_pool, deep_ep_installed +from tensorrt_llm._utils import get_sm_version from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -108,11 +109,12 @@ def destroy(self): @staticmethod def is_platform_supported() -> bool: - """ - Check if DeepEP Low Latency is supported on the current platform - """ + """Check if DeepEP Low Latency is supported on the current platform.""" if not deep_ep_installed: return False + # SM120/121 (RTX PRO 6000 Blackwell): no NVSwitch -> NVSHMEM-LL deadlocks. + if get_sm_version() in (120, 121): + return False return True def supports_post_quant_dispatch(self) -> bool: From af74e004c99fcef85cef4679646cc6561b6b47e4 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Thu, 28 May 2026 00:38:59 +0800 Subject: [PATCH 060/308] [https://nvbugs/6114464][fix] Add kv_cache_config to TestQwen3VL_MOE::test_auto_dtype (#13668) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../defs/accuracy/test_llm_api_pytorch_multimodal.py | 3 +++ tests/integration/test_lists/waives.txt | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 4638f01ecdda..0e36f38faa57 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -389,11 +389,14 @@ class TestQwen3VL_MOE(LlmapiAccuracyTestHarness): max_tokens=MAX_NUM_TOKENS, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, stop="<|endoftext|>" ) + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.4) + @pytest.mark.skip_less_device_memory(140000) def test_auto_dtype(self): with LLM( self.MODEL_PATH, max_num_tokens=self.MAX_NUM_TOKENS, + kv_cache_config=self.kv_cache_config, ) as llm: task = MMMU(self.MODEL_NAME) task.evaluate(llm, sampling_params=self.sampling_params) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 90a30f5ea7f6..92b375f8ba63 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -152,7 +152,6 @@ accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[ accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6189416) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) -accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype SKIP (https://nvbugs/6114464) accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6094070) cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689) cpp/test_e2e.py::test_benchmarks[gpt-80] SKIP (https://nvbugs/5550689) From 0b16ef9d8ba18f85af01834bf79088bf516a6b93 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Thu, 28 May 2026 01:33:36 +0800 Subject: [PATCH 061/308] [None][chore] Update flashinfer-python from 0.6.12rc1 to 0.6.12rc2 (#14607) Signed-off-by: yihwang-nv --- ATTRIBUTIONS-Python.md | 2 +- requirements.txt | 2 +- security_scanning/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index 39bfa93bf760..4ae2f5070ff1 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -5261,7 +5261,7 @@ For more information, please refer to - `Tracker`: https://github.com/tox-dev/py-filelock/issues -## flashinfer-python (0.6.12rc1) +## flashinfer-python (0.6.12rc2) ### Licenses License: `Apache-2.0` diff --git a/requirements.txt b/requirements.txt index 9b76d52b0c86..1373acedf491 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ ordered-set peft>=0.18.1,<0.19.0 patchelf einops -flashinfer-python @ git+https://github.com/flashinfer-ai/flashinfer.git@v0.6.12rc1#egg=flashinfer-python +flashinfer-python==0.6.12rc2 opencv-python-headless xgrammar==0.1.32 llguidance==0.7.29 diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 5a96a617f64c..97fe085fd20d 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -54,7 +54,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python @ git+https://github.com/flashinfer-ai/flashinfer.git@v0.6.12rc1", + "flashinfer-python (==0.6.12rc2)", "opencv-python-headless (>=4.13.0.92,<5.0.0.0)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", From 80a22f733b79d50ee71489a81a3a7f1a9190c95a Mon Sep 17 00:00:00 2001 From: dongfengy <99041270+dongfengy@users.noreply.github.com> Date: Wed, 27 May 2026 11:09:29 -0700 Subject: [PATCH 062/308] [https://nvbugs/6109750][test] Unwaive passing GPTOSS tests (#14596) Signed-off-by: Dongfeng Yu --- tests/integration/test_lists/waives.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 92b375f8ba63..43c49f7fabeb 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -79,9 +79,7 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughpu accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5772993) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-no_overlap_scheduler] SKIP (https://nvbugs/6220815) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] SKIP (https://nvbugs/6215702) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[two_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) @@ -90,7 +88,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-fp8] SKIP (https://nvbugs/5651865) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto] SKIP (https://nvbugs/5596343) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8] SKIP (https://nvbugs/6109750) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) From 86c1b0ccaf0ba884f991d2ab19f6035c20da8912 Mon Sep 17 00:00:00 2001 From: Gal Hubara-Agam <96368689+galagam@users.noreply.github.com> Date: Wed, 27 May 2026 21:45:55 +0300 Subject: [PATCH 063/308] [https://nvbugs/6215690][fix] AutoDeploy: FlashInfer 128-byte alignment for Mamba inputs (also addresses nvbugs/6162114) (#14535) Signed-off-by: Gal Hubara Agam <96368689+galagam@users.noreply.github.com> Signed-off-by: Gal Hubara-Agam <96368689+galagam@users.noreply.github.com> --- .../custom_ops/mamba/flashinfer_backend_mamba.py | 15 +++++++++++++++ tests/integration/test_lists/waives.txt | 5 +---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/flashinfer_backend_mamba.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/flashinfer_backend_mamba.py index 67ad432a2231..563cfe34c9b9 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/flashinfer_backend_mamba.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/flashinfer_backend_mamba.py @@ -36,6 +36,17 @@ ) +def _fi_align(t: torch.Tensor) -> torch.Tensor: + """Ensure 128-byte alignment required by FlashInfer kernels. + + - Contiguous + aligned: contiguous() is a no-op, returns t unchanged. + - Non-contiguous: contiguous() allocates fresh aligned storage, returns it. + - Contiguous + misaligned: contiguous() is a no-op, clone() forces a new aligned allocation. + """ + t = t.contiguous() + return t if t.data_ptr() % 128 == 0 else t.clone() + + @torch.library.custom_op( "auto_deploy::flashinfer_cached_ssm", mutates_args=("ssm_state_cache", "intermediate_ssm_state_cache"), @@ -205,6 +216,10 @@ def _flashinfer_cached_ssm( D_full, ) = decode_inputs + x_decode = _fi_align(x_decode) + B_decode = _fi_align(B_decode) + C_decode = _fi_align(C_decode) + slot_idx_decode_i32 = slot_idx_decode.to(torch.int32) y_decode = _flashinfer_ssm_update( ssm_state_cache, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 43c49f7fabeb..f53f0443a92a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -15,11 +15,8 @@ accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbu accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) -accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16] SKIP (https://nvbugs/6162114) -accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8] SKIP (https://nvbugs/6162114) +accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[fp8_ws4_80gb-trtllm] SKIP (https://nvbugs/6221483) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] SKIP (https://nvbugs/6221483) -accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4] SKIP (https://nvbugs/6221483) -accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6215690) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) From b1eb703635647db24f35b0a8239fd4e2b775d5fc Mon Sep 17 00:00:00 2001 From: Holegots Date: Thu, 28 May 2026 05:05:05 +0800 Subject: [PATCH 064/308] [None][docs] Add deprecation notice to legacy support-matrix.md (#14495) Signed-off-by: holegots --- docs/source/legacy/reference/support-matrix.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/legacy/reference/support-matrix.md b/docs/source/legacy/reference/support-matrix.md index 46fd97cef2bf..4c7629843299 100644 --- a/docs/source/legacy/reference/support-matrix.md +++ b/docs/source/legacy/reference/support-matrix.md @@ -1,5 +1,9 @@ # Support Matrix +```{deprecated} +This page is outdated and no longer maintained. Please refer to the up-to-date [Supported Models](../../models/supported-models.md) page instead. +``` + TensorRT-LLM optimizes the performance of a range of well-known models on NVIDIA GPUs. The following sections provide a list of supported GPU architectures as well as important features implemented in TensorRT-LLM. ## Models (PyTorch Backend) From 9fa13581ab0c6dd1eb84ecaab3a6bfc1008ebe7e Mon Sep 17 00:00:00 2001 From: o-stoner <245287810+o-stoner@users.noreply.github.com> Date: Wed, 27 May 2026 18:27:06 -0400 Subject: [PATCH 065/308] [None][fix] make FA4 proper pip dependency (#13788) Signed-off-by: Olivia Stoner <245287810+o-stoner@users.noreply.github.com> --- requirements.txt | 1 + .../attention_backend/flash_attn4.py | 8 +- .../visual_gen/attention_backend/parallel.py | 4 +- .../_torch/visual_gen/jit_kernels/__init__.py | 0 .../jit_kernels/flash_attention/__init__.py | 0 .../jit_kernels/flash_attention/cute/.flake8 | 4 - .../jit_kernels/flash_attention/cute/AUTHORS | 5 - .../jit_kernels/flash_attention/cute/LICENSE | 29 - .../flash_attention/cute/README.md | 0 .../flash_attention/cute/__init__.py | 21 - .../flash_attention/cute/ampere_helpers.py | 103 - .../flash_attention/cute/barrier.py | 71 - .../flash_attention/cute/benchmark.py | 268 -- .../flash_attention/cute/blackwell_helpers.py | 753 ----- .../flash_attention/cute/block_info.py | 108 - .../cute/block_sparse_utils.py | 1451 -------- .../flash_attention/cute/block_sparsity.py | 218 -- .../cute/compute_block_sparsity.py | 377 --- .../flash_attention/cute/copy_utils.py | 340 -- .../flash_attention/cute/cute_dsl_utils.py | 150 - .../flash_attention/cute/fast_math.py | 21 - .../flash_attention/cute/flash_bwd.py | 1583 --------- .../cute/flash_bwd_postprocess.py | 463 --- .../cute/flash_bwd_preprocess.py | 365 -- .../flash_attention/cute/flash_bwd_sm100.py | 2942 ----------------- .../flash_attention/cute/flash_bwd_sm90.py | 1701 ---------- .../flash_attention/cute/flash_fwd.py | 2468 -------------- .../flash_attention/cute/flash_fwd_combine.py | 704 ---- .../flash_attention/cute/flash_fwd_sm100.py | 2815 ---------------- .../flash_attention/cute/hopper_helpers.py | 101 - .../flash_attention/cute/interface.py | 1831 ---------- .../jit_kernels/flash_attention/cute/mask.py | 609 ---- .../flash_attention/cute/mma_sm100_desc.py | 291 -- .../flash_attention/cute/named_barrier.py | 31 - .../flash_attention/cute/pack_gqa.py | 164 - .../flash_attention/cute/paged_kv.py | 188 -- .../flash_attention/cute/pipeline.py | 272 -- .../flash_attention/cute/pyproject.toml | 56 - .../flash_attention/cute/seqlen_info.py | 138 - .../flash_attention/cute/softmax.py | 580 ---- .../flash_attention/cute/testing.py | 423 --- .../flash_attention/cute/tile_scheduler.py | 719 ---- .../jit_kernels/flash_attention/cute/utils.py | 852 ----- 43 files changed, 3 insertions(+), 23225 deletions(-) delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/__init__.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/__init__.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/.flake8 delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/AUTHORS delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/LICENSE delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/README.md delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/__init__.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/ampere_helpers.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/barrier.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/benchmark.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/blackwell_helpers.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_info.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparse_utils.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparsity.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/compute_block_sparsity.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/copy_utils.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/cute_dsl_utils.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/fast_math.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_postprocess.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_preprocess.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm100.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm90.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_combine.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_sm100.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/hopper_helpers.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/interface.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mask.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mma_sm100_desc.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/named_barrier.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pack_gqa.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/paged_kv.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pipeline.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pyproject.toml delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/seqlen_info.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/softmax.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/testing.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/tile_scheduler.py delete mode 100644 tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/utils.py diff --git a/requirements.txt b/requirements.txt index 1373acedf491..3a1de400c452 100644 --- a/requirements.txt +++ b/requirements.txt @@ -78,6 +78,7 @@ partial_json_parser mcp apache-tvm-ffi==0.1.6 # used for reduce nvidia-cutlass-dsl host overhead torch-c-dlpack-ext==0.1.3 # used for reduce nvidia-cutlass-dsl host overhead, optional package for improved torch tensor calling perf +flash-attn-4==4.0.0b11 mistral-common>=1.10.0 torchao>=0.14.1,<0.16.0 cuda-core diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py b/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py index 8981faa0de11..0090e43e3005 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py @@ -17,10 +17,6 @@ Uses Flash Attention 4 with the CUTE JIT kernel. Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16. - -Cute kernel source: tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/ -(https://github.com/Dao-AILab/flash-attention/tree/main/flash_attn/cute -at commit ea8f73506369d7cdd498396474107a978858138c) """ import math @@ -33,9 +29,7 @@ _flash_attn_fwd_import_error = None try: - from tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.interface import ( - _flash_attn_fwd, - ) + from flash_attn.cute.interface import _flash_attn_fwd except (ImportError, OSError) as e: _flash_attn_fwd = None _flash_attn_fwd_import_error = e diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py index efa282350808..0cd02787cd8b 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py @@ -31,9 +31,7 @@ _flash_attn_combine_import_error = None try: - from tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.interface import ( - flash_attn_combine as _flash_attn_combine, - ) + from flash_attn.cute.interface import flash_attn_combine as _flash_attn_combine except (ImportError, OSError) as e: _flash_attn_combine = None _flash_attn_combine_import_error = e diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/__init__.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/__init__.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/.flake8 b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/.flake8 deleted file mode 100644 index bae5b85c002e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 100 -# W503: line break before binary operator -ignore = E731, E741, F841, W503 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/AUTHORS b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/AUTHORS deleted file mode 100644 index 150a51e61251..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/AUTHORS +++ /dev/null @@ -1,5 +0,0 @@ -Tri Dao, tri@tridao.me -Jay Shah -Ted Zadouri -Markus Hoehnerbach -Vijay Thakkar diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/LICENSE b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/LICENSE deleted file mode 100644 index 5860e4b33f3d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/README.md b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/__init__.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/__init__.py deleted file mode 100644 index 7706f746aee9..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Flash Attention CUTE (CUDA Template Engine) implementation.""" - -__version__ = "0.1.0" - -import cutlass.cute as cute - -from .interface import ( - flash_attn_func, - flash_attn_varlen_func, -) - -from .cute_dsl_utils import cute_compile_patched - -# Patch cute.compile to optionally dump SASS -cute.compile = cute_compile_patched - - -__all__ = [ - "flash_attn_func", - "flash_attn_varlen_func", -] diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/ampere_helpers.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/ampere_helpers.py deleted file mode 100644 index e3072d8ce85c..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/ampere_helpers.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2025, Tri Dao. -from typing import Type, Callable, Optional - -import cutlass -import cutlass.cute as cute - - -def get_smem_layout_atom(dtype: Type[cutlass.Numeric], k_dim: int) -> cute.ComposedLayout: - dtype_byte = cutlass.const_expr(dtype.width // 8) - bytes_per_row = cutlass.const_expr(k_dim * dtype_byte) - smem_k_block_size = ( - cutlass.const_expr( - 128 - if bytes_per_row % 128 == 0 - else (64 if bytes_per_row % 64 == 0 else (32 if bytes_per_row % 32 == 0 else 16)) - ) - // dtype_byte - ) - swizzle_bits = ( - 4 - if smem_k_block_size == 128 - else (3 if smem_k_block_size == 64 else (2 if smem_k_block_size == 32 else 1)) - ) - swizzle_base = 2 if dtype_byte == 4 else (3 if dtype_byte == 2 else 4) - return cute.make_composed_layout( - cute.make_swizzle(swizzle_bits, swizzle_base, swizzle_base), - 0, - cute.make_ordered_layout( - (8 if cutlass.const_expr(k_dim % 32 == 0) else 16, smem_k_block_size), order=(1, 0) - ), - ) - - -@cute.jit -def gemm( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - tCsA: cute.Tensor, - tCsB: cute.Tensor, - smem_thr_copy_A: cute.TiledCopy, - smem_thr_copy_B: cute.TiledCopy, - hook_fn: Optional[Callable] = None, - A_in_regs: cutlass.Constexpr[bool] = False, - B_in_regs: cutlass.Constexpr[bool] = False, - swap_AB: cutlass.Constexpr[bool] = False, -) -> None: - if cutlass.const_expr(swap_AB): - gemm( - tiled_mma, - acc, - tCrB, - tCrA, - tCsB, - tCsA, - smem_thr_copy_B, - smem_thr_copy_A, - hook_fn, - A_in_regs=B_in_regs, - B_in_regs=A_in_regs, - swap_AB=False, - ) - else: - tCrA_copy_view = smem_thr_copy_A.retile(tCrA) - tCrB_copy_view = smem_thr_copy_B.retile(tCrB) - if cutlass.const_expr(not A_in_regs): - cute.copy(smem_thr_copy_A, tCsA[None, None, 0], tCrA_copy_view[None, None, 0]) - if cutlass.const_expr(not B_in_regs): - cute.copy(smem_thr_copy_B, tCsB[None, None, 0], tCrB_copy_view[None, None, 0]) - for k in cutlass.range_constexpr(cute.size(tCsA.shape[2])): - if k < cute.size(tCsA.shape[2]) - 1: - if cutlass.const_expr(not A_in_regs): - cute.copy( - smem_thr_copy_A, tCsA[None, None, k + 1], tCrA_copy_view[None, None, k + 1] - ) - if cutlass.const_expr(not B_in_regs): - cute.copy( - smem_thr_copy_B, tCsB[None, None, k + 1], tCrB_copy_view[None, None, k + 1] - ) - cute.gemm(tiled_mma, acc, tCrA[None, None, k], tCrB[None, None, k], acc) - if cutlass.const_expr(k == 0 and hook_fn is not None): - hook_fn() - - -@cute.jit -def gemm_rs( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - tCsB: cute.Tensor, - smem_thr_copy_B: cute.TiledCopy, - hook_fn: Optional[Callable] = None, -) -> None: - tCrB_copy_view = smem_thr_copy_B.retile(tCrB) - cute.copy(smem_thr_copy_B, tCsB[None, None, 0], tCrB_copy_view[None, None, 0]) - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])): - if cutlass.const_expr(k < cute.size(tCrA.shape[2]) - 1): - cute.copy(smem_thr_copy_B, tCsB[None, None, k + 1], tCrB_copy_view[None, None, k + 1]) - cute.gemm(tiled_mma, acc, tCrA[None, None, k], tCrB[None, None, k], acc) - if cutlass.const_expr(k == 0 and hook_fn is not None): - hook_fn() diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/barrier.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/barrier.py deleted file mode 100644 index c999b180167d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/barrier.py +++ /dev/null @@ -1,71 +0,0 @@ -import cutlass -import cutlass.cute as cute -from cutlass import Int32 -from cutlass.cutlass_dsl import T, dsl_user_op -from cutlass._mlir.dialects import llvm - - -@dsl_user_op -def ld_acquire(lock_ptr: cute.Pointer, *, loc=None, ip=None) -> cutlass.Int32: - lock_ptr_i64 = lock_ptr.toint(loc=loc, ip=ip).ir_value() - state = llvm.inline_asm( - T.i32(), - [lock_ptr_i64], - "ld.global.acquire.gpu.b32 $0, [$1];", - "=r,l", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - return cutlass.Int32(state) - - -@dsl_user_op -def red_relaxed( - lock_ptr: cute.Pointer, val: cutlass.Constexpr[Int32], *, loc=None, ip=None -) -> None: - lock_ptr_i64 = lock_ptr.toint(loc=loc, ip=ip).ir_value() - llvm.inline_asm( - None, - [lock_ptr_i64, Int32(val).ir_value(loc=loc, ip=ip)], - "red.relaxed.gpu.global.add.s32 [$0], $1;", - "l,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@dsl_user_op -def red_release( - lock_ptr: cute.Pointer, val: cutlass.Constexpr[Int32], *, loc=None, ip=None -) -> None: - lock_ptr_i64 = lock_ptr.toint(loc=loc, ip=ip).ir_value() - llvm.inline_asm( - None, - [lock_ptr_i64, Int32(val).ir_value(loc=loc, ip=ip)], - "red.release.gpu.global.add.s32 [$0], $1;", - "l,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@cute.jit -def wait_eq(lock_ptr: cute.Pointer, thread_idx: int | Int32, flag_offset: int, val: Int32) -> None: - flag_ptr = lock_ptr + flag_offset - if thread_idx == 0: - read_val = Int32(0) - while read_val != val: - read_val = ld_acquire(flag_ptr) - - -@cute.jit -def arrive_inc( - lock_ptr: cute.Pointer, thread_idx: int | Int32, flag_offset: int, val: cutlass.Constexpr[Int32] -) -> None: - flag_ptr = lock_ptr + flag_offset - if thread_idx == 0: - red_release(flag_ptr, val) - # red_relaxed(flag_ptr, val) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/benchmark.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/benchmark.py deleted file mode 100644 index 9a7820e7b0c7..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/benchmark.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright (c) 2023, Tri Dao. -"""Useful functions for writing test code.""" - -import torch -import torch.utils.benchmark as benchmark - - -def benchmark_forward( - fn, *inputs, repeats=10, desc="", verbose=True, amp=False, amp_dtype=torch.float16, **kwinputs -): - """Use Pytorch Benchmark on the forward pass of an arbitrary function.""" - if verbose: - print(desc, "- Forward pass") - - def amp_wrapper(*inputs, **kwinputs): - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - fn(*inputs, **kwinputs) - - t = benchmark.Timer( - stmt="fn_amp(*inputs, **kwinputs)", - globals={"fn_amp": amp_wrapper, "inputs": inputs, "kwinputs": kwinputs}, - num_threads=torch.get_num_threads(), - ) - m = t.timeit(repeats) - if verbose: - print(m) - return t, m - - -def benchmark_backward( - fn, - *inputs, - grad=None, - repeats=10, - desc="", - verbose=True, - amp=False, - amp_dtype=torch.float16, - **kwinputs, -): - """Use Pytorch Benchmark on the backward pass of an arbitrary function.""" - if verbose: - print(desc, "- Backward pass") - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - y = fn(*inputs, **kwinputs) - if type(y) is tuple: - y = y[0] - if grad is None: - grad = torch.randn_like(y) - else: - if grad.shape != y.shape: - raise RuntimeError("Grad shape does not match output shape") - - def f(*inputs, y, grad): - # Set .grad to None to avoid extra operation of gradient accumulation - for x in inputs: - if isinstance(x, torch.Tensor): - x.grad = None - y.backward(grad, retain_graph=True) - - t = benchmark.Timer( - stmt="f(*inputs, y=y, grad=grad)", - globals={"f": f, "inputs": inputs, "y": y, "grad": grad}, - num_threads=torch.get_num_threads(), - ) - m = t.timeit(repeats) - if verbose: - print(m) - return t, m - - -def benchmark_combined( - fn, - *inputs, - grad=None, - repeats=10, - desc="", - verbose=True, - amp=False, - amp_dtype=torch.float16, - **kwinputs, -): - """Use Pytorch Benchmark on the forward+backward pass of an arbitrary function.""" - if verbose: - print(desc, "- Forward + Backward pass") - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - y = fn(*inputs, **kwinputs) - if type(y) is tuple: - y = y[0] - if grad is None: - grad = torch.randn_like(y) - else: - if grad.shape != y.shape: - raise RuntimeError("Grad shape does not match output shape") - - def f(grad, *inputs, **kwinputs): - for x in inputs: - if isinstance(x, torch.Tensor): - x.grad = None - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - y = fn(*inputs, **kwinputs) - if type(y) is tuple: - y = y[0] - y.backward(grad, retain_graph=True) - - t = benchmark.Timer( - stmt="f(grad, *inputs, **kwinputs)", - globals={"f": f, "fn": fn, "inputs": inputs, "grad": grad, "kwinputs": kwinputs}, - num_threads=torch.get_num_threads(), - ) - m = t.timeit(repeats) - if verbose: - print(m) - return t, m - - -def benchmark_fwd_bwd( - fn, - *inputs, - grad=None, - repeats=10, - desc="", - verbose=True, - amp=False, - amp_dtype=torch.float16, - **kwinputs, -): - """Use Pytorch Benchmark on the forward+backward pass of an arbitrary function.""" - return ( - benchmark_forward( - fn, - *inputs, - repeats=repeats, - desc=desc, - verbose=verbose, - amp=amp, - amp_dtype=amp_dtype, - **kwinputs, - ), - benchmark_backward( - fn, - *inputs, - grad=grad, - repeats=repeats, - desc=desc, - verbose=verbose, - amp=amp, - amp_dtype=amp_dtype, - **kwinputs, - ), - ) - - -def benchmark_all( - fn, - *inputs, - grad=None, - repeats=10, - desc="", - verbose=True, - amp=False, - amp_dtype=torch.float16, - **kwinputs, -): - """Use Pytorch Benchmark on the forward+backward pass of an arbitrary function.""" - return ( - benchmark_forward( - fn, - *inputs, - repeats=repeats, - desc=desc, - verbose=verbose, - amp=amp, - amp_dtype=amp_dtype, - **kwinputs, - ), - benchmark_backward( - fn, - *inputs, - grad=grad, - repeats=repeats, - desc=desc, - verbose=verbose, - amp=amp, - amp_dtype=amp_dtype, - **kwinputs, - ), - benchmark_combined( - fn, - *inputs, - grad=grad, - repeats=repeats, - desc=desc, - verbose=verbose, - amp=amp, - amp_dtype=amp_dtype, - **kwinputs, - ), - ) - - -def pytorch_profiler( - fn, - *inputs, - trace_filename=None, - backward=False, - amp=False, - amp_dtype=torch.float16, - cpu=False, - verbose=True, - **kwinputs, -): - """Wrap benchmark functions in Pytorch profiler to see CUDA information.""" - if backward: - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - out = fn(*inputs, **kwinputs) - if type(out) is tuple: - out = out[0] - g = torch.randn_like(out) - for _ in range(30): # Warm up - if backward: - for x in inputs: - if isinstance(x, torch.Tensor): - x.grad = None - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - out = fn(*inputs, **kwinputs) - if type(out) is tuple: - out = out[0] - # Backward should be done outside autocast - if backward: - out.backward(g, retain_graph=True) - activities = ([torch.profiler.ProfilerActivity.CPU] if cpu else []) + [ - torch.profiler.ProfilerActivity.CUDA - ] - with torch.profiler.profile( - activities=activities, - record_shapes=True, - # profile_memory=True, - with_stack=True, - ) as prof: - if backward: - for x in inputs: - if isinstance(x, torch.Tensor): - x.grad = None - with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): - out = fn(*inputs, **kwinputs) - if type(out) is tuple: - out = out[0] - if backward: - out.backward(g, retain_graph=True) - if verbose: - # print(prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=50)) - print(prof.key_averages().table(row_limit=50)) - if trace_filename is not None: - prof.export_chrome_trace(trace_filename) - - -def benchmark_memory(fn, *inputs, desc="", verbose=True, **kwinputs): - torch.cuda.empty_cache() - torch.cuda.reset_peak_memory_stats() - torch.cuda.synchronize() - fn(*inputs, **kwinputs) - torch.cuda.synchronize() - mem = torch.cuda.max_memory_allocated() / ((2**20) * 1000) - if verbose: - print(f"{desc} max memory: {mem}GB") - torch.cuda.empty_cache() - return mem diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/blackwell_helpers.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/blackwell_helpers.py deleted file mode 100644 index 861e12512d99..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/blackwell_helpers.py +++ /dev/null @@ -1,753 +0,0 @@ -# Copyright (c) 2025, Tri Dao. -from typing import Optional, Tuple - -import cutlass -import cutlass.cute as cute -from cutlass import Int32, Boolean, const_expr -from cutlass.cute.nvgpu import tcgen05 -from cutlass._mlir.dialects import llvm - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.mma_sm100_desc as sm100_desc -from .utils import parse_swizzle_from_pointer - - -@cute.jit -def gemm_w_idx( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - A_idx: Optional[Int32] = None, - B_idx: Optional[Int32] = None, - zero_init: bool | Boolean = False, - swap_AB: bool = False, -) -> None: - if const_expr(swap_AB): - return gemm_w_idx( - tiled_mma, acc, tCrB, tCrA, B_idx, A_idx, zero_init=zero_init, swap_AB=False - ) - else: - rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx] - rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx] - mma_atom = cute.make_mma_atom(tiled_mma.op) - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])): - mma_atom.set(tcgen05.Field.ACCUMULATE, not zero_init or k != 0) - cute.gemm(mma_atom, acc, rA[None, None, k], rB[None, None, k], acc) - - -@cute.jit -def gemm_ptx_w_idx( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - sA: Optional[cute.Tensor], - sB: cute.Tensor, - A_idx: Optional[Int32] = None, - B_idx: Optional[Int32] = None, - zero_init: bool | Boolean = False, - **kwargs, -) -> None: - rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx] - rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx] - sA_cur = None - if const_expr(sA is not None): - sA_cur = sA if const_expr(A_idx is None) else sA[None, None, None, A_idx] - sB_cur = sB if const_expr(B_idx is None) else sB[None, None, None, B_idx] - mma_atom = cute.make_mma_atom(tiled_mma.op) - acc_tmem_addr = acc.iterator.toint() - gemm_ptx_partial( - mma_atom.op, acc_tmem_addr, rA, rB, sA_cur, sB_cur, zero_init=zero_init, **kwargs - ) - - -@cute.jit -def gemm( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - zero_init: bool | Boolean = False, -) -> cute.TiledMma: - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])): - tiled_mma.set(tcgen05.Field.ACCUMULATE, not zero_init or k != 0) - cute.gemm(tiled_mma, acc, tCrA[None, None, k], tCrB[None, None, k], acc) - return tiled_mma - - -def i64_to_i32x2(i: int) -> Tuple[int, int]: - """Convert a 64-bit integer to a tuple of two 32-bit integers.""" - return i & 0xFFFF_FFFF, (i >> 32) & 0xFFFF_FFFF - - -@cute.jit -def gemm_ptx( - op: cute.nvgpu.tcgen05.mma.MmaOp, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - sA: Optional[cute.Tensor], - sB: cute.Tensor, - zero_init: bool | Boolean = False, -) -> None: - is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM - if const_expr(not is_ts): - assert sA is not None, "sA must be provided when a_src is not TMEM" - sA_layout = sA.layout if sA is not None else None - sB_layout = sB.layout - idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op)) - if const_expr(not is_ts): - sA_swizzle = parse_swizzle_from_pointer(sA.iterator) - smem_desc_base_a: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.a_dtype.width, sA_layout[0]), - sA_swizzle, - sm100_desc.Major.K - if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a) - smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo) - smem_desc_a_hi = const_expr(smem_desc_a_hi) - else: - smem_desc_base_a = None - smem_desc_base_a_lo, smem_desc_a_hi = None, None - sB_swizzle = parse_swizzle_from_pointer(sB.iterator) - smem_desc_base_b: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.b_dtype.width, sB_layout[0]), - sB_swizzle, - sm100_desc.Major.K - if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b) - smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo) - smem_desc_b_hi = const_expr(smem_desc_b_hi) - - if const_expr(not is_ts): - smem_desc_start_a_lo = Int32(smem_desc_base_a_lo) | sm100_desc.make_smem_desc_start_addr( - sA[None, None, 0].iterator - ) - else: - smem_desc_start_a_lo = None - smem_desc_start_b_lo = Int32(smem_desc_base_b_lo) | sm100_desc.make_smem_desc_start_addr( - sB[None, None, 0].iterator - ) - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])): - if const_expr(not is_ts): - smem_desc_a_lo = smem_desc_start_a_lo + ( - (cute.crd2idx((0, 0, k), sA_layout) * sA.element_type.width // 8) >> 4 - ) - smem_desc_b_lo = smem_desc_start_b_lo + ( - (cute.crd2idx((0, 0, k), sB_layout) * sB.element_type.width // 8) >> 4 - ) - # with cute.arch.elect_one(): - # cute.printf("smem_desc_a_lo = {}, smem_desc_b_lo = {}", smem_desc_a_lo, smem_desc_b_lo) - # cute.printf("smem_desc_a_lo_correct = {}, smem_desc_b_lo_correct = {}", smem_desc_a_lo_correct, smem_desc_b_lo_correct) - with cute.arch.elect_one(): - if const_expr(not is_ts): - llvm.inline_asm( - None, - [ - acc.iterator.toint().ir_value(), - smem_desc_a_lo.ir_value(), - smem_desc_b_lo.ir_value(), - Int32(not zero_init or k != 0).ir_value(), - ], - "{\n\t" - ".reg .pred p;\n\t" - ".reg .b64 smem_desc_a, smem_desc_b;\n\t" - ".reg .b32 idesc;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - f"mov.b64 smem_desc_a, {{$1, {hex(smem_desc_a_hi)}}};\n\t" - f"mov.b64 smem_desc_b, {{$2, {hex(smem_desc_b_hi)}}};\n\t" - "setp.ne.b32 p, $3, 0;\n\t" - f"tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, p;\n\t" - "}\n", - "r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - else: - llvm.inline_asm( - None, - [ - acc.iterator.toint().ir_value(), - tCrA[None, None, k].iterator.toint().ir_value(), - smem_desc_b_lo.ir_value(), - Int32(not zero_init or k != 0).ir_value(), - ], - "{\n\t" - ".reg .pred p;\n\t" - ".reg .b64 smem_desc_b;\n\t" - f"mov.b64 smem_desc_b, {{$2, {hex(smem_desc_b_hi)}}};\n\t" - "setp.ne.b32 p, $3, 0;\n\t" - f"tcgen05.mma.cta_group::1.kind::f16 [$0], [$1], smem_desc_b, {hex(idesc)}, p;\n\t" - "}\n", - "r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@cute.jit -def gemm_ptx_loop( - op: cute.nvgpu.tcgen05.mma.MmaOp, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - sA: Optional[cute.Tensor], - sB: cute.Tensor, - zero_init: bool | Boolean = False, -) -> None: - is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM - if const_expr(not is_ts): - assert sA is not None, "sA must be provided when a_src is not TMEM" - sA_layout = sA.layout if sA is not None else tCrA.layout - sB_layout = sB.layout - idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op)) - if const_expr(not is_ts): - sA_swizzle = parse_swizzle_from_pointer(sA.iterator) - smem_desc_base_a: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.a_dtype.width, sA_layout[0]), - sA_swizzle, - sm100_desc.Major.K - if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a) - smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo) - smem_desc_a_hi = const_expr(smem_desc_a_hi) - else: - smem_desc_base_a = None - smem_desc_base_a_lo, smem_desc_a_hi = None, None - sB_swizzle = parse_swizzle_from_pointer(sB.iterator) - smem_desc_base_b: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.b_dtype.width, sB_layout[0]), - sB_swizzle, - sm100_desc.Major.K - if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b) - smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo) - smem_desc_b_hi = const_expr(smem_desc_b_hi) - - if const_expr(not is_ts): - offset_a = [ - (cute.crd2idx((0, 0, k), sA_layout) * sA.element_type.width // 8) >> 4 - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])) - ] - else: - offset_a = [ - cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 32 - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])) - ] - offset_a_diff = [ - offset_a[k] - offset_a[k - 1] for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2])) - ] - offset_b = [ - (cute.crd2idx((0, 0, k), sB_layout) * sB.element_type.width // 8) >> 4 - for k in cutlass.range_constexpr(cute.size(tCrB.shape[2])) - ] - offset_b_diff = [ - offset_b[k] - offset_b[k - 1] for k in cutlass.range_constexpr(1, cute.size(tCrB.shape[2])) - ] - - if const_expr(not is_ts): - smem_desc_start_a_lo = Int32( - smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator) - ) - else: - smem_desc_start_a_lo = None - smem_desc_start_b_lo = Int32( - smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator) - ) - pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1" - if const_expr(not is_ts): - llvm.inline_asm( - None, - [ - acc.iterator.toint().ir_value(), - Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(), - Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(), - Int32(not zero_init).ir_value(), - ], - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_a, smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - "mov.b32 smem_desc_a_lo, $1;\n\t" - "mov.b32 smem_desc_b_lo, $2;\n\t" - f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $3, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, {pred_str};\n\t" - + "".join( - ( - f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, 1;\n\t" - ) - for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2])) - ) - + "}\n", - "r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - else: - llvm.inline_asm( - None, - [ - acc.iterator.toint().ir_value(), - Int32(tCrA[None, None, 0].iterator.toint()).ir_value(), - Int32(smem_desc_start_b_lo).ir_value(), - Int32(not zero_init).ir_value(), - ], - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 tmem_a;\n\t" - ".reg .b32 smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - "mov.b32 tmem_a, $1;\n\t" - "mov.b32 smem_desc_b_lo, $2;\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $3, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {pred_str};\n\t" - + "".join( - ( - # f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - # f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, 1;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t" - ) - for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2])) - ) - + "}\n", - "r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@cute.jit -def gemm_ptx_partial( - op: cute.nvgpu.tcgen05.mma.MmaOp, - acc_tmem_addr: Int32, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - sA: Optional[cute.Tensor], - sB: cute.Tensor, - mbar_ptr: Optional[cutlass.Pointer] = None, - mbar_phase: Optional[Int32] = None, - zero_init: bool | Boolean = False, - # sA_offset: Int32 = 0, - # acc_offset: Int32 = 0, - tA_addr: Optional[Int32] = None, -) -> None: - # acc_tmem_addr += acc_offset - is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM - if const_expr(not is_ts): - assert sA is not None, "sA must be provided when a_src is not TMEM" - sA_layout = sA.layout if sA is not None else tCrA.layout - sB_layout = sB.layout - idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op)) - if const_expr(not is_ts): - sA_swizzle = parse_swizzle_from_pointer(sA.iterator) - smem_desc_base_a: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.a_dtype.width, sA_layout[0]), - sA_swizzle, - sm100_desc.Major.K - if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a) - smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo) - smem_desc_a_hi = const_expr(smem_desc_a_hi) - else: - smem_desc_base_a = None - smem_desc_base_a_lo, smem_desc_a_hi = None, None - sB_swizzle = parse_swizzle_from_pointer(sB.iterator) - smem_desc_base_b: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.b_dtype.width, sB_layout[0]), - sB_swizzle, - sm100_desc.Major.K - if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b) - smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo) - smem_desc_b_hi = const_expr(smem_desc_b_hi) - - tCrA_layout = ( - tCrA.layout - if const_expr(not is_ts) - else cute.recast_layout(32, tCrA.element_type.width, tCrA.layout) - ) - offset_a = [cute.crd2idx((0, 0, k), tCrA_layout) for k in range(cute.size(tCrA.shape[2]))] - offset_a_diff = [offset_a[k] - offset_a[k - 1] for k in range(1, cute.size(tCrA.shape[2]))] - offset_b = [cute.crd2idx((0, 0, k), tCrB.layout) for k in range(cute.size(tCrB.shape[2]))] - offset_b_diff = [offset_b[k] - offset_b[k - 1] for k in range(1, cute.size(tCrB.shape[2]))] - - if const_expr(not is_ts): - smem_desc_start_a_lo = Int32( - smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator) - ) - # ) + sA_offset - else: - smem_desc_start_a_lo = None - smem_desc_start_b_lo = Int32( - smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator) - ) - pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1" - if const_expr(not is_ts): - assert mbar_ptr is None, "mbar_ptr must be None when a_src is not TMEM" - llvm.inline_asm( - None, - [ - # acc.iterator.toint().ir_value(), - Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(), - Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(), - Int32(not zero_init).ir_value(), - Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(), - ], - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 tmem_acc;\n\t" - ".reg .b32 smem_desc_a_lo_start, smem_desc_b_lo_start;\n\t" - ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_a, smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t" - f"mov.b32 tmem_acc, $3;\n\t" - "mov.b32 smem_desc_a_lo_start, $0;\n\t" - "mov.b32 smem_desc_b_lo_start, $1;\n\t" - f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo_start, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $2, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {pred_str};\n\t" - + "".join( - ( - # f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t" - # f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"add.u32 smem_desc_a_lo, smem_desc_a_lo_start, {hex(offset_a[k])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, 1;\n\t" - ) - for k in range(1, cute.size(tCrA.shape[2])) - ) - + "}\n", - # "r,r,r", - "r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - else: - # For TS gemm, somehow tCrA.iterator.toint() returns 0 no matter what, so we need to - # explicitly pass in the tA_addr for correctness. - tA_addr = tCrA[None, None, 0].iterator.toint() if tA_addr is None else tA_addr - input_args = [ - # Int32(cute.arch.make_warp_uniform(tCrA[None, None, 0].iterator.toint())).ir_value(), - Int32(cute.arch.make_warp_uniform(tA_addr)).ir_value(), - Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(), - Int32(not zero_init).ir_value(), - Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(), - ] - if const_expr(mbar_ptr is not None): - assert mbar_phase is not None, "mbar_phase must be provided when mbar_ptr is not None" - input_args.append(mbar_ptr.toint().ir_value()) - input_args.append(Int32(mbar_phase).ir_value()) - mbar_wait_str = ( - ".reg .pred P1; \n\t" - "LAB_WAIT: \n\t" - "mbarrier.try_wait.parity.shared::cta.b64 P1, [$4], $5, 10000000; \n\t" - "@P1 bra DONE; \n\t" - "bra LAB_WAIT; \n\t" - "DONE: \n\t" - ) - else: - mbar_wait_str = "" - llvm.inline_asm( - None, - # [ - # # acc.iterator.toint().ir_value(), - # Int32(tCrA[None, None, 0].iterator.toint()).ir_value(), - # Int32(smem_desc_start_b_lo).ir_value(), - # Int32(not zero_init).ir_value(), - # ], - input_args, - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 tmem_acc;\n\t" - ".reg .b32 tmem_a;\n\t" - ".reg .b32 smem_desc_b_lo_start;\n\t" - ".reg .b32 smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t" - f"mov.b32 tmem_acc, $3;\n\t" - f"mov.b32 tmem_a, $0;\n\t" - f"mov.b32 smem_desc_b_lo_start, $1;\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $2, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a], smem_desc_b, idesc, {pred_str};\n\t" - + "".join( - ( - # f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t" - # f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - # f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a], smem_desc_b, idesc, 1;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t" - ) - for k in range( - 1, - cute.size(tCrA.shape[2]) - if const_expr(mbar_ptr is None) - else cute.size(tCrA.shape[2]) // 4 * 3, - ) - ) - + mbar_wait_str - + ( - "".join( - ( - f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t" - ) - for k in range(cute.size(tCrA.shape[2]) // 4 * 3, cute.size(tCrA.shape[2])) - ) - if const_expr(mbar_ptr is not None) - else "" - ) - + "}\n", - "r,r,r,r" if const_expr(mbar_ptr is None) else "r,r,r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@cute.jit -def gemm_ptx_partial1( - op: cute.nvgpu.tcgen05.mma.MmaOp, - acc_tmem_addr: cutlass.Constexpr[int], - tCrA: cute.Tensor, - tCrB: cute.Tensor, - sA_base_addr_for_desc: Int32, - sA_addr_offset_for_desc: cutlass.Constexpr[int], - sA_stage: Int32, - sB_base_addr_for_desc: Int32, - sB_addr_offset_for_desc: cutlass.Constexpr[int], - sB_stage: Int32, - sA_layout: Optional[cute.Layout], - sB_layout: Optional[cute.Layout], - sA_swizzle: Optional[cute.Swizzle], - sB_swizzle: cute.Swizzle, - zero_init: bool | Boolean = False, -) -> None: - is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM - if const_expr(not is_ts): - assert sA_layout is not None, "sA_layout must be provided when a_src is not TMEM" - assert sA_swizzle is not None, "sA_swizzle must be provided when a_src is not TMEM" - idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op)) - if const_expr(not is_ts): - smem_desc_base_a: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.a_dtype.width, sA_layout[0]), - sA_swizzle, - sm100_desc.Major.K - if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a) - smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo) - smem_desc_a_hi = const_expr(smem_desc_a_hi) - else: - smem_desc_base_a = None - smem_desc_base_a_lo, smem_desc_a_hi = None, None - smem_desc_base_b: int = const_expr( - sm100_desc.make_smem_desc_base( - cute.recast_layout(128, op.b_dtype.width, sB_layout[0]), - sB_swizzle, - sm100_desc.Major.K - if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K) - else sm100_desc.Major.MN, - ) - ) - smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b) - smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo) - smem_desc_b_hi = const_expr(smem_desc_b_hi) - mask = [Int32(0)] * 4 - - if const_expr(not is_ts): - offset_a = [ - (cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 8) >> 4 - for k in range(cute.size(tCrA.shape[2])) - ] - else: - offset_a = [ - cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 32 - for k in range(cute.size(tCrA.shape[2])) - ] - offset_a_diff = [offset_a[k] - offset_a[k - 1] for k in range(1, cute.size(tCrA.shape[2]))] - offset_b = [ - (cute.crd2idx((0, 0, k), sB_layout) * op.b_dtype.width // 8) >> 4 - for k in range(cute.size(tCrB.shape[2])) - ] - offset_b_diff = [offset_b[k] - offset_b[k - 1] for k in range(1, cute.size(tCrB.shape[2]))] - - if const_expr(not is_ts): - # smem_desc_start_a_lo = Int32(smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator)) - smem_desc_start_a_lo = const_expr(smem_desc_base_a_lo) - else: - smem_desc_start_a_lo = None - # smem_desc_start_b_lo = Int32(smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator)) - smem_desc_start_b_lo = const_expr(smem_desc_base_b_lo) - pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1" - if const_expr(not is_ts): - llvm.inline_asm( - None, - [ - # acc.iterator.toint().ir_value(), - # Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(), - Int32(sA_base_addr_for_desc).ir_value(), - Int32(sA_stage).ir_value(), - # Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(), - Int32(sB_base_addr_for_desc).ir_value(), - Int32(sB_stage).ir_value(), - Int32(not zero_init).ir_value(), - mask[0].ir_value(), - mask[1].ir_value(), - mask[2].ir_value(), - mask[3].ir_value(), - ], - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 tmem_acc;\n\t" - ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_a, smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t" - # "mov.b32 smem_desc_a_lo, $0;\n\t" - # f"add.u32 smem_desc_a_lo, $0, {hex(smem_desc_start_a_lo)};\n\t" - f"mad.lo.u32 smem_desc_a_lo, $1, {hex(sA_addr_offset_for_desc)}, $0;\n\t" - # "mov.b32 smem_desc_b_lo, $2;\n\t" - f"mad.lo.u32 smem_desc_b_lo, $3, {hex(sB_addr_offset_for_desc)}, $2;\n\t" - f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $4, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {{$5, $6, $7, $8}}, {pred_str};\n\t" - + "".join( - ( - f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {{$5, $6, $7, $8}}, 1;\n\t" - ) - for k in range(1, cute.size(tCrA.shape[2])) - ) - + "}\n", - "r,r,r,r,r,r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - else: - llvm.inline_asm( - None, - [ - # acc.iterator.toint().ir_value(), - Int32(tCrA[None, None, 0].iterator.toint()).ir_value(), - Int32(smem_desc_start_b_lo).ir_value(), - Int32(not zero_init).ir_value(), - mask[0].ir_value(), - mask[1].ir_value(), - mask[2].ir_value(), - mask[3].ir_value(), - ], - "{\n\t" - ".reg .pred leader_thread;\n\t" - ".reg .pred p;\n\t" - ".reg .b32 idesc;\n\t" - ".reg .b32 tmem_a;\n\t" - ".reg .b32 smem_desc_b_lo;\n\t" - ".reg .b32 smem_desc_b_hi;\n\t" - ".reg .b64 smem_desc_b;\n\t" - "elect.sync _|leader_thread, -1;\n\t" - f"mov.b32 idesc, {hex(idesc)};\n\t" - f"mov.b32 tmem_a, $1;\n\t" - f"mov.b32 smem_desc_b_lo, $2;\n\t" - f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - "setp.ne.b32 p, $3, 0;\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {{$4, $5, $6, $7}}, {pred_str};\n\t" - + "".join( - ( - f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t" - f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t" - f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t" - f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {{$4, $5, $6, $7}}, 1;\n\t" - ) - for k in range(1, cute.size(tCrA.shape[2])) - ) - + "}\n", - "r,r,r,r,r,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_info.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_info.py deleted file mode 100644 index a5a2544a7883..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_info.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -from typing import Tuple, Optional -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass import Int32, const_expr - -from .seqlen_info import SeqlenInfoQK - - -@dataclass(frozen=True) -class BlockInfo: - tile_m: cutlass.Constexpr[int] - tile_n: cutlass.Constexpr[int] - is_causal: cutlass.Constexpr[bool] - is_local: cutlass.Constexpr[bool] = False - is_split_kv: cutlass.Constexpr[bool] = False - window_size_left: Optional[Int32] = None - window_size_right: Optional[Int32] = None - qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1 - - @cute.jit - def get_n_block_min_max( - self, - seqlen_info: SeqlenInfoQK, - m_block: Int32, - split_idx: cutlass.Int32 = 0, - num_splits: cutlass.Int32 = 1, - ) -> Tuple[Int32, Int32]: - n_block_max = cute.ceil_div(seqlen_info.seqlen_k, self.tile_n) - if const_expr(self.is_causal or (self.is_local and self.window_size_right is not None)): - m_idx_max = (m_block + 1) * self.tile_m - if const_expr(self.qhead_per_kvhead_packgqa > 1): - m_idx_max = cute.ceil_div(m_idx_max, self.qhead_per_kvhead_packgqa) - n_idx = m_idx_max + seqlen_info.seqlen_k - seqlen_info.seqlen_q - n_idx_right = n_idx if const_expr(self.is_causal) else n_idx + self.window_size_right - n_block_max = min(n_block_max, cute.ceil_div(n_idx_right, self.tile_n)) - n_block_min = 0 - if const_expr(self.is_local and self.window_size_left is not None): - m_idx_min = m_block * self.tile_m - if const_expr(self.qhead_per_kvhead_packgqa > 1): - m_idx_min = m_idx_min // self.qhead_per_kvhead_packgqa - n_idx = m_idx_min + seqlen_info.seqlen_k - seqlen_info.seqlen_q - n_idx_left = n_idx - self.window_size_left - n_block_min = cutlass.max(n_idx_left // self.tile_n, 0) - if cutlass.const_expr(self.is_split_kv): - num_n_blocks_per_split = ( - cutlass.Int32(0) - if n_block_max <= n_block_min - else (n_block_max - n_block_min + num_splits - 1) // num_splits - ) - n_block_min = n_block_min + split_idx * num_n_blocks_per_split - n_block_max = cutlass.min(n_block_min + num_n_blocks_per_split, n_block_max) - return n_block_min, n_block_max - - @cute.jit - def get_m_block_min_max(self, seqlen_info: SeqlenInfoQK, n_block: Int32) -> Tuple[Int32, Int32]: - m_block_max = cute.ceil_div(seqlen_info.seqlen_q, self.tile_m) - m_block_min = 0 - if const_expr(self.is_causal or (self.is_local and self.window_size_right is not None)): - n_idx_min = n_block * self.tile_n - m_idx = n_idx_min + seqlen_info.seqlen_q - seqlen_info.seqlen_k - m_idx_right = m_idx if const_expr(self.is_causal) else m_idx - self.window_size_right - m_block_min = max(m_block_min, m_idx_right // self.tile_m) - if const_expr(self.is_local and self.window_size_left is not None): - n_idx_max = (n_block + 1) * self.tile_n - m_idx = n_idx_max + seqlen_info.seqlen_q - seqlen_info.seqlen_k - m_idx_left = m_idx + self.window_size_left - m_block_max = min(m_block_max, cute.ceil_div(m_idx_left, self.tile_m)) - return m_block_min, m_block_max - - @cute.jit - def get_n_block_min_causal_local_mask( - self, - seqlen_info: SeqlenInfoQK, - m_block: Int32, - n_block_min: Int32, - ) -> Int32: - """If we have separate iterations with causal or local masking at the start, where do we stop""" - m_idx_min = m_block * self.tile_m - if const_expr(self.qhead_per_kvhead_packgqa > 1): - m_idx_min = m_idx_min // self.qhead_per_kvhead_packgqa - n_idx = m_idx_min + seqlen_info.seqlen_k - seqlen_info.seqlen_q - n_idx_right = ( - n_idx - if const_expr(not self.is_local or self.window_size_right is None) - else n_idx + self.window_size_right - ) - return cutlass.max(n_block_min, n_idx_right // self.tile_n) - - @cute.jit - def get_n_block_min_before_local_mask( - self, - seqlen_info: SeqlenInfoQK, - m_block: Int32, - n_block_min: Int32, - ) -> Int32: - """If we have separate iterations with local masking at the end, where do we stop the non-masked iterations""" - if const_expr(not self.is_local or self.window_size_left is None): - return n_block_min - else: - m_idx_max = (m_block + 1) * self.tile_m - if const_expr(self.qhead_per_kvhead_packgqa > 1): - m_idx_max = cute.ceil_div(m_idx_max, self.qhead_per_kvhead_packgqa) - n_idx = m_idx_max + seqlen_info.seqlen_k - seqlen_info.seqlen_q - n_idx_left = n_idx - self.window_size_left - return cutlass.max(n_block_min, cute.ceil_div(n_idx_left, self.tile_n)) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparse_utils.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparse_utils.py deleted file mode 100644 index 780fc536042b..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparse_utils.py +++ /dev/null @@ -1,1451 +0,0 @@ -""" -Block-sparse runtime utilities for CUTE DSL kernels. - -This module contains runtime execution functions for block-sparse attention kernels. -These utilities are used by CUTE DSL kernels to produce and consume block-sparse loads. -""" - -from typing import Callable, Optional -from functools import partial -import math -import cutlass -import cutlass.cute as cute -from cutlass import Float32, Int32, const_expr - -# Import data structures from block_sparsity -from .block_sparsity import BlockSparseTensors -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -from .named_barrier import NamedBarrierBwd - - -@cute.jit -def load_block_list( - block_indices: cute.Tensor, - block_count, - load_q_with_first: cutlass.Constexpr, - first_block_preloaded: cutlass.Constexpr, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_k, - pipeline_v, - use_tma_q: cutlass.Constexpr, - tma_q_bytes: cutlass.Constexpr, - intra_wg_overlap: cutlass.Constexpr, -): - """Iterate over the sparse blocks and load K, V (and Q) into the pipeline. - for the intra_wg_overlap case, we overlap the loads of K and V. And this - means we need to pipeline the last V load from the partial block case, - with the loads for the full blocks. Set first_block_preloaded when the - caller has already issued the first K load for the list. - - Note: - we iterate along the block_n indices in reverse. - - Returns: - Updated kv_producer_state after processing the block list. - - """ - if block_count > 0: - if const_expr(not intra_wg_overlap): - # Peel first iteration: the first block may need to load Q alongside K, - # Parameters are already Constexpr, so no need to wrap in const_expr() - n_block_first = block_indices[block_count - 1] - extra_tx = tma_q_bytes if const_expr(load_q_with_first) and const_expr(use_tma_q) else 0 - pipeline_k.producer_acquire(kv_producer_state, extra_tx_count=extra_tx) - - if const_expr(load_q_with_first and use_tma_q): - load_Q(tma_bar_ptr=pipeline_k.producer_get_barrier(kv_producer_state)) - - load_K(src_idx=n_block_first, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block_first, producer_state=kv_producer_state) - kv_producer_state.advance() - - for offset in cutlass.range(1, block_count): - n_block = block_indices[block_count - 1 - offset] - pipeline_k.producer_acquire(kv_producer_state) - load_K(src_idx=n_block, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block, producer_state=kv_producer_state) - kv_producer_state.advance() - else: - n_block_first = block_indices[block_count - 1] - if const_expr(not first_block_preloaded): - extra_tx = ( - tma_q_bytes if const_expr(load_q_with_first) and const_expr(use_tma_q) else 0 - ) - pipeline_k.producer_acquire(kv_producer_state, extra_tx_count=extra_tx) - - if const_expr(load_q_with_first and use_tma_q): - load_Q(tma_bar_ptr=pipeline_k.producer_get_barrier(kv_producer_state)) - - load_K(src_idx=n_block_first, producer_state=kv_producer_state) - - for idx in cutlass.range(block_count - 1, unroll=1): - n_block_prev = block_indices[block_count - 1 - idx] - n_block = block_indices[block_count - 2 - idx] - kv_producer_state_prev = kv_producer_state.clone() - kv_producer_state.advance() - pipeline_k.producer_acquire(kv_producer_state) - load_K(src_idx=n_block, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state_prev) - load_V(src_idx=n_block_prev, producer_state=kv_producer_state_prev) - - return kv_producer_state - - -@cute.jit -def finish_overlap_v_load( - block_indices: cute.Tensor, - block_count, - load_V, - pipeline_v, - kv_producer_state, -): - """Load the final V block after overlapped K/V loads.""" - if block_count > 0: - n_block_last = block_indices[0] - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block_last, producer_state=kv_producer_state) - kv_producer_state.advance() - - return kv_producer_state - - -@cute.jit -def sparse_tensor_m_block( - m_block, - qhead_per_kvhead: cutlass.Constexpr[int], -): - """Map packed m_block indices to block-sparse tensor indices.""" - if const_expr(qhead_per_kvhead != 1): - return m_block // qhead_per_kvhead - return m_block - - -@cute.jit -def produce_block_sparse_loads( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - m_block, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_k, - pipeline_v, - use_tma_q: cutlass.Constexpr, - tma_q_bytes: cutlass.Constexpr, - intra_wg_overlap: cutlass.Constexpr, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, -): - """Iterate over the mask and full block lists for a single tile. - - The masked (partial) list may leave the last V load pending when intra-warp-group - overlap is enabled. The first full block must consume that pending V while - issuing its own K load on the next pipeline stage. - - In the intra-wg-overlap path, the last masked block leaves its V copy in flight - while we advance the producer state to start the next full K. Either the full list - overlaps that pending V load, or, if no full blocks exist, we explicitly drain it. - - Args: - qhead_per_kvhead: Pack-GQA factor. When > 1, m_block is in packed space and - must be converted to unpacked for sparse tensor indexing. - """ - - mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors - - m_block_sparse = sparse_tensor_m_block(m_block, qhead_per_kvhead) - - curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block_sparse, None] - - if const_expr(full_block_cnt is not None): - curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block_sparse, None] - else: - curr_full_block_cnt = Int32(0) - curr_full_block_idx = None - - mask_empty = curr_mask_block_cnt == 0 - full_empty = curr_full_block_cnt == 0 - - if mask_empty: - # No masked blocks: the full list owns the initial Q+K load. - kv_producer_state = load_block_list( - curr_full_block_idx, - curr_full_block_cnt, - load_q_with_first=True, - first_block_preloaded=False, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_k=pipeline_k, - pipeline_v=pipeline_v, - use_tma_q=use_tma_q, - tma_q_bytes=tma_q_bytes, - intra_wg_overlap=intra_wg_overlap, - ) - - if const_expr(intra_wg_overlap) and curr_full_block_cnt > 0: - kv_producer_state = finish_overlap_v_load( - curr_full_block_idx, - curr_full_block_cnt, - load_V, - pipeline_v, - kv_producer_state, - ) - else: - # Masked blocks present: load Q together with the first masked K so consumers can - # start immediately. When overlap is disabled this fully drains the list. - kv_producer_state = load_block_list( - curr_mask_block_idx, - curr_mask_block_cnt, - load_q_with_first=True, - first_block_preloaded=False, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_k=pipeline_k, - pipeline_v=pipeline_v, - use_tma_q=use_tma_q, - tma_q_bytes=tma_q_bytes, - intra_wg_overlap=intra_wg_overlap, - ) - - if full_empty: - if const_expr(intra_wg_overlap): - kv_producer_state = finish_overlap_v_load( - curr_mask_block_idx, - curr_mask_block_cnt, - load_V, - pipeline_v, - kv_producer_state, - ) - else: - if const_expr(intra_wg_overlap): - # Bridge the masked list to the full list by overlapping the pending masked V - # with the first full K load. - n_block_mask_last = curr_mask_block_idx[0] - n_block_full_first = curr_full_block_idx[curr_full_block_cnt - 1] - kv_producer_state_prev = kv_producer_state.clone() - kv_producer_state.advance() - pipeline_k.producer_acquire(kv_producer_state) - load_K(src_idx=n_block_full_first, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state_prev) - load_V(src_idx=n_block_mask_last, producer_state=kv_producer_state_prev) - - kv_producer_state = load_block_list( - curr_full_block_idx, - curr_full_block_cnt, - load_q_with_first=False, - first_block_preloaded=True, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_k=pipeline_k, - pipeline_v=pipeline_v, - use_tma_q=use_tma_q, - tma_q_bytes=tma_q_bytes, - intra_wg_overlap=intra_wg_overlap, - ) - - kv_producer_state = finish_overlap_v_load( - curr_full_block_idx, - curr_full_block_cnt, - load_V, - pipeline_v, - kv_producer_state, - ) - else: - # Non-overlap path with both lists: run the full list normally (skipping the Q - # reload because the masked list already issued it). - kv_producer_state = load_block_list( - curr_full_block_idx, - curr_full_block_cnt, - load_q_with_first=False, - first_block_preloaded=False, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_k=pipeline_k, - pipeline_v=pipeline_v, - use_tma_q=use_tma_q, - tma_q_bytes=tma_q_bytes, - intra_wg_overlap=intra_wg_overlap, - ) - - return kv_producer_state - - -@cute.jit -def consume_block_sparse_loads( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - m_block, - seqlen, - kv_consumer_state, - mma_pv_fn, - mma_one_n_block, - process_first_half_block, - process_last_half_block, - mask_fn, - score_mod_fn, - O_should_accumulate, - mask_mod, - fastdiv_mods, - intra_wg_overlap: cutlass.Constexpr, - warp_scheduler_barrier_sync: Callable, - warp_scheduler_barrier_arrive: Callable, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, -): - """Consume the mask and full block lists for a single tile on the consumer side. - - Mirrors `produce_block_sparse_loads` so that the consumer pipeline uses - the same sparse tensor indexing. - - Args: - qhead_per_kvhead: Pack-GQA factor. When > 1, m_block is in packed space and - must be converted to unpacked for sparse tensor indexing. - """ - - mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors - - m_block_sparse = sparse_tensor_m_block(m_block, qhead_per_kvhead) - - curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block_sparse, None] - curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block_sparse, None] - - processed_any = curr_mask_block_cnt + curr_full_block_cnt > 0 - - if const_expr(not intra_wg_overlap): - if curr_mask_block_cnt > 0: - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1] - warp_scheduler_barrier_sync() - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=mask_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial( - mask_fn, - mask_mod=mask_mod, - mask_seqlen=True, - fastdiv_mods=fastdiv_mods if cutlass.const_expr(mask_mod is not None) else None, - ), - is_first_n_block=True, - ) - O_should_accumulate = True - for i in cutlass.range(1, curr_mask_block_cnt): - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i] - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=mask_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=mask_mod, mask_seqlen=False), - is_first_n_block=False, - ) - O_should_accumulate = True - if curr_full_block_cnt == 0: - warp_scheduler_barrier_arrive() - - if curr_full_block_cnt > 0: - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1] - if curr_mask_block_cnt == 0: - warp_scheduler_barrier_sync() - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_seqlen=True), - is_first_n_block=True, - ) - O_should_accumulate = True - for i in cutlass.range(1, curr_full_block_cnt): - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i] - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_seqlen=False), - is_first_n_block=False, - ) - O_should_accumulate = True - else: - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=True), - is_first_n_block=False, - ) - O_should_accumulate = True - for i in cutlass.range(1, curr_full_block_cnt): - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i] - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=False), - is_first_n_block=False, - ) - O_should_accumulate = True - warp_scheduler_barrier_arrive() - else: - if curr_mask_block_cnt > 0: - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1] - kv_consumer_state = process_first_half_block( - n_block=mask_n_block, - seqlen=seqlen, - kv_consumer_state=kv_consumer_state, - mask_fn=partial( - mask_fn, - mask_mod=mask_mod, - mask_seqlen=True, - fastdiv_mods=fastdiv_mods if cutlass.const_expr(mask_mod is not None) else None, - ), - score_mod_fn=score_mod_fn, - is_first_block=True, - ) - for i in cutlass.range(1, curr_mask_block_cnt): - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i] - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=mask_n_block, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=mask_mod, mask_seqlen=False), - ) - O_should_accumulate = True - - if curr_full_block_cnt > 0: - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1] - if curr_mask_block_cnt == 0: - kv_consumer_state = process_first_half_block( - n_block=full_n_block, - seqlen=seqlen, - kv_consumer_state=kv_consumer_state, - mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=True), - score_mod_fn=score_mod_fn, - is_first_block=True, - ) - else: - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=True), - ) - O_should_accumulate = True - for i in cutlass.range(1, curr_full_block_cnt): - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i] - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=full_n_block, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=False), - ) - O_should_accumulate = True - - if curr_mask_block_cnt + curr_full_block_cnt > 0: - kv_consumer_state = process_last_half_block( - kv_consumer_state=kv_consumer_state, - zero_init=not O_should_accumulate, - ) - O_should_accumulate = True - - return kv_consumer_state, O_should_accumulate, processed_any - - -@cute.jit -def load_block_list_sm100( - block_indices: cute.Tensor, - block_count, - load_q_with_first: cutlass.Constexpr, - m_block, - q_stage: cutlass.Constexpr, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_kv, -): - """SM100 version of load_block_list (no intra_wg_overlap, no extra_tx_count).""" - if block_count > 0: - # First iteration: load Q alongside K if requested - n_block_first = block_indices[block_count - 1] - - if const_expr(load_q_with_first): - # SM100 loads Q0 and optionally Q1 - load_Q(block=q_stage * m_block + 0, stage=0) - if const_expr(q_stage == 2): - load_Q(block=q_stage * m_block + 1, stage=1) - - # SM100 doesn't use producer_acquire for pipeline_kv in load path - # The pipeline barriers are handled inside load_KV - load_K(block=n_block_first, producer_state=kv_producer_state, page_idx=None) - kv_producer_state.advance() - load_V(block=n_block_first, producer_state=kv_producer_state, page_idx=None) - kv_producer_state.advance() - - # Remaining blocks - for offset in cutlass.range(1, block_count): - n_block = block_indices[block_count - 1 - offset] - load_K(block=n_block, producer_state=kv_producer_state, page_idx=None) - kv_producer_state.advance() - load_V(block=n_block, producer_state=kv_producer_state, page_idx=None) - kv_producer_state.advance() - - return kv_producer_state - - -# SM100-specific tile processor using SM100 helpers -@cute.jit -def produce_block_sparse_loads_sm100( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - m_block, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_kv, - q_stage: cutlass.Constexpr, - q_producer_phase: Int32, - qhead_per_kvhead: cutlass.Constexpr, -): - """SM100 entry point for sparse block iteration. - - SM100 uses PipelineTmaUmma which doesn't support extra_tx_count, so we use - simplified block processing that just calls producer_acquire without extras. - - Args: - m_block: which tile of m we are processing - qhead_per_kvhead: Constexpr pack factor - """ - # NB: Compute unpacked index for sparse tensor access - if const_expr(qhead_per_kvhead != 1): - m_block_sparse = m_block // qhead_per_kvhead - else: - m_block_sparse = m_block - - mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors - - curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block_sparse, None] - - if const_expr(full_block_cnt is not None): - curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block_sparse, None] - else: - curr_full_block_cnt = Int32(0) - curr_full_block_idx = None - - mask_empty = curr_mask_block_cnt == 0 - full_empty = curr_full_block_cnt == 0 - - q_phase_flipped = False - - if mask_empty: - # No masked blocks: process full list with Q loading - kv_producer_state = load_block_list_sm100( - curr_full_block_idx, - curr_full_block_cnt, - load_q_with_first=True, - m_block=m_block, - q_stage=q_stage, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_kv=pipeline_kv, - ) - q_phase_flipped = not full_empty - else: - # Process masked blocks with Q loading - kv_producer_state = load_block_list_sm100( - curr_mask_block_idx, - curr_mask_block_cnt, - load_q_with_first=True, - m_block=m_block, - q_stage=q_stage, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_kv=pipeline_kv, - ) - q_phase_flipped = True - - if not full_empty: - # Process full blocks without Q loading - kv_producer_state = load_block_list_sm100( - curr_full_block_idx, - curr_full_block_cnt, - load_q_with_first=False, - m_block=m_block, - q_stage=q_stage, - kv_producer_state=kv_producer_state, - load_Q=load_Q, - load_K=load_K, - load_V=load_V, - pipeline_kv=pipeline_kv, - ) - - if q_phase_flipped: - q_producer_phase ^= 1 - - return kv_producer_state, q_producer_phase - - -@cute.jit -def get_total_block_count( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - m_block, - qhead_per_kvhead: cutlass.Constexpr, -): - # NB: Convert packed m_block to unpacked for sparse tensor indexing - if const_expr(qhead_per_kvhead != 1): - m_block_sparse = m_block // qhead_per_kvhead - else: - m_block_sparse = m_block - - mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors - if const_expr(full_block_cnt is not None): - return ( - mask_block_cnt[batch_idx, head_idx, m_block_sparse] - + full_block_cnt[batch_idx, head_idx, m_block_sparse] - ) - else: - return mask_block_cnt[batch_idx, head_idx, m_block_sparse] - - -@cute.jit -def handle_block_sparse_empty_tile_correction_sm100( - tidx: Int32, - q_stage: cutlass.Constexpr, - m_block_size: cutlass.Constexpr, - qhead_per_kvhead, - pack_gqa: cutlass.Constexpr, - is_split_kv: cutlass.Constexpr, - learnable_sink, - mLSE, - seqlen, - m_block: Int32, - head_idx: Int32, - batch_idx: Int32, - split_idx: Int32, - sScale: cute.Tensor, - stats: list, - correction_epilogue: Callable, - thr_mma_pv: cute.core.ThrMma, - tOtOs: tuple[cute.Tensor], - sO: cute.Tensor, - mbar_ptr, - mbar_softmax_corr_full_offset: Int32, - mbar_softmax_corr_empty_offset: Int32, - mbar_P_full_O_rescaled_offset: Int32, - mbar_P_full_2_offset: Int32, - mbar_corr_epi_full_offset: Int32, - mbar_corr_epi_empty_offset: Int32, - softmax_corr_consumer_phase: Int32, - o_corr_consumer_phase: Int32, - corr_epi_producer_phase: Int32, - softmax_scale_log2: Float32, - mO_cur: Optional[cute.Tensor] = None, - gO: Optional[cute.Tensor] = None, - gmem_tiled_copy_O: Optional[cute.TiledCopy] = None, -): - """Handle the block-sparse case where a tile is fully masked: - * zero staged results - * seed stats - * satisfy the usual barrier protocol so downstream warps continue to make progress. - """ - LOG2_E = Float32(math.log2(math.e)) - - for stage in cutlass.range_constexpr(q_stage): - row_sum_value = Float32(1.0) - row_max_value = ( - -Float32.inf if const_expr(mLSE is not None or learnable_sink is not None) else None - ) - if const_expr(learnable_sink is not None): - sink_val = -Float32.inf - if const_expr(not pack_gqa): - sink_val = Float32(learnable_sink[head_idx]) - elif tidx < m_block_size: - q_head_idx = ( - (q_stage * m_block + stage) * m_block_size + tidx - ) % qhead_per_kvhead + head_idx * qhead_per_kvhead - sink_val = Float32(learnable_sink[q_head_idx]) - if sink_val != -Float32.inf and (const_expr(not is_split_kv) or split_idx == 0): - if row_max_value == -Float32.inf: - row_max_value = sink_val * (LOG2_E / softmax_scale_log2) - row_sum_value = Float32(1.0) - else: - row_sum_value = row_sum_value + utils.exp2f( - sink_val * LOG2_E - row_max_value * softmax_scale_log2 - ) - if tidx < m_block_size: - scale_row_idx = tidx + stage * m_block_size - sScale[scale_row_idx] = row_sum_value - if const_expr(mLSE is not None or learnable_sink is not None): - sScale[scale_row_idx + m_block_size * 2] = row_max_value - acc_flag = row_sum_value == Float32(0.0) or row_sum_value != row_sum_value - stats[stage] = (row_sum_value, row_max_value, acc_flag) - - cute.arch.mbarrier_wait( - mbar_ptr + mbar_softmax_corr_full_offset + stage, - softmax_corr_consumer_phase, - ) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_empty_offset + stage) - - if const_expr(gmem_tiled_copy_O is None): - cute.arch.mbarrier_wait( - mbar_ptr + mbar_corr_epi_empty_offset + stage, - corr_epi_producer_phase, - ) - correction_epilogue( - thr_mma_pv, - tOtOs[stage], - tidx, - stage, - m_block, - seqlen.seqlen_q, - Float32(0.0), # zero scale ensures empty tile writes zeros into staged outputs - sO[None, None, stage], - mO_cur, - gO, - gmem_tiled_copy_O, - ) - if const_expr(gmem_tiled_copy_O is None): - cute.arch.mbarrier_arrive(mbar_ptr + mbar_corr_epi_full_offset + stage) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_O_rescaled_offset + stage) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_2_offset + stage) - - softmax_corr_consumer_phase ^= 1 - o_corr_consumer_phase ^= 1 - corr_epi_producer_phase ^= 1 - - return ( - softmax_corr_consumer_phase, - o_corr_consumer_phase, - corr_epi_producer_phase, - ) - - -@cute.jit -def softmax_block_sparse_sm100( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - m_block, - softmax_step: Callable, - mask_fn: Callable, - mask_fn_none: Callable, - mma_si_consumer_phase: Int32, - si_corr_producer_phase: Int32, - s0_s1_sequence_phase: Int32, - mbar_ptr, - mbar_softmax_corr_full_offset: Int32, - mbar_softmax_corr_empty_offset: Int32, - mbar_P_full_O_rescaled_offset: Int32, - mbar_P_full_2_offset: Int32, - q_stage: cutlass.Constexpr, - stage_idx: Int32, - check_m_boundary: bool, - qhead_per_kvhead: cutlass.Constexpr, -): - # Convert packed m_block to unpacked for sparse tensor indexing - if const_expr(qhead_per_kvhead != 1): - m_block_sparse = m_block // qhead_per_kvhead - else: - m_block_sparse = m_block - - mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors - - curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block_sparse, None] - - if const_expr(full_block_cnt is not None): - curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block_sparse] - curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block_sparse, None] - else: - curr_full_block_cnt = Int32(0) - curr_full_block_idx = None - - total_block_cnt = curr_mask_block_cnt + curr_full_block_cnt - - if total_block_cnt == 0: - cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_full_offset + stage_idx) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_O_rescaled_offset + stage_idx) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_2_offset + stage_idx) - cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_empty_offset + stage_idx) - else: - if curr_mask_block_cnt > 0: - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1] - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - ) = softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - mask_n_block, - is_first=True, - mask_fn=partial(mask_fn, mask_seqlen=True, check_q_boundary=check_m_boundary), - ) - for i in cutlass.range(1, curr_mask_block_cnt): - mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i] - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - ) = softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - mask_n_block, - mask_fn=partial(mask_fn, mask_seqlen=False, check_q_boundary=check_m_boundary), - ) - - if curr_full_block_cnt > 0: - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1] - if curr_mask_block_cnt == 0: - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - ) = softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - full_n_block, - is_first=True, - mask_fn=partial( - mask_fn_none, mask_seqlen=True, check_q_boundary=check_m_boundary - ), - ) - else: - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - ) = softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - full_n_block, - is_first=False, - mask_fn=partial( - mask_fn_none, mask_seqlen=False, check_q_boundary=check_m_boundary - ), - ) - for i in cutlass.range(1, curr_full_block_cnt): - full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i] - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - ) = softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - full_n_block, - mask_fn=partial( - mask_fn_none, mask_seqlen=False, check_q_boundary=check_m_boundary - ), - ) - - return ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - total_block_cnt == 0, - ) - - -# ============================================================================= -# Backward-specific block-sparse helpers (SM100) -# ============================================================================= -# -# In backward, iteration is transposed compared to forward: -# - Forward: outer loop over m_blocks (Q tiles), inner loop over n_blocks (KV tiles) -# - Backward: outer loop over n_blocks (KV tiles), inner loop over m_blocks (Q tiles) -# -# The backward block-sparse tensors use "Q direction" indexing: -# - q_block_cnt[batch, head, n_block] → count of m_blocks to process for this KV tile -# - q_block_idx[batch, head, n_block, :] → indices of m_blocks to process -# - - -@cute.jit -def get_total_q_block_count_bwd( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - subtile_factor: cutlass.Constexpr = 1, - m_block_max: int = 0, -): - """Count total tile iterations for given n_block (KV tile) in backward.""" - q_block_cnt, _, full_block_cnt, _ = blocksparse_tensors - total = q_block_cnt[batch_idx, head_idx, n_block] - if const_expr(full_block_cnt is not None): - total = total + full_block_cnt[batch_idx, head_idx, n_block] - return total * subtile_factor - - -@cute.jit -def produce_block_sparse_q_loads_bwd_sm100( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - # Pipeline states (will be returned after advancing) - producer_state_Q_LSE, - producer_state_dO_dPsum, - # Pipelines - pipeline_Q, - pipeline_LSE, - pipeline_dO, - pipeline_dPsum, - # Load functions - load_K, - load_V, - load_Q, - load_dO, - copy_stats, - # Global tensors for LSE/dPsum - gLSE, - sLSE, - gdPsum, - sdPsum, - # TMA copy bytes for extra_tx_count - tma_copy_bytes_K, - tma_copy_bytes_V, - # Flags for which loads to perform - should_load_Q: cutlass.Constexpr, - should_load_dO: cutlass.Constexpr, - # Subtiling factor and bounds - subtile_factor: cutlass.Constexpr = 1, - m_block_max: int = 0, -): - """SM100 backward block sparse loading with subtiling. - - Returns updated (producer_state_Q_LSE, producer_state_dO_dPsum). - First iteration loads K/V alongside Q/dO; subsequent iterations load only Q/dO. - """ - ( - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - loop_count, - ) = get_block_sparse_iteration_info_bwd( - blocksparse_tensors, batch_idx, head_idx, n_block, subtile_factor, m_block_max - ) - - for iter_idx in cutlass.range(loop_count, unroll=1): - m_block, _ = get_m_block_from_iter_bwd( - iter_idx, - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - subtile_factor, - m_block_max, - ) - m_block_safe = m_block - if m_block_max > 0: - m_block_safe = cutlass.min(m_block, m_block_max - 1) - - if iter_idx == 0: - # First block: load K/V alongside Q/dO - if const_expr(should_load_Q): - pipeline_Q.producer_acquire(producer_state_Q_LSE, extra_tx_count=tma_copy_bytes_K) - load_K(tma_bar_ptr=pipeline_Q.producer_get_barrier(producer_state_Q_LSE)) - load_Q(m_block_safe, producer_state=producer_state_Q_LSE) - pipeline_Q.producer_commit(producer_state_Q_LSE) - pipeline_LSE.producer_acquire(producer_state_Q_LSE) - with cute.arch.elect_one(): - copy_stats( - gLSE[None, m_block_safe], - sLSE[None, producer_state_Q_LSE.index], - mbar_ptr=pipeline_LSE.producer_get_barrier(producer_state_Q_LSE), - ) - producer_state_Q_LSE.advance() - if const_expr(should_load_dO): - pipeline_dO.producer_acquire( - producer_state_dO_dPsum, extra_tx_count=tma_copy_bytes_V - ) - load_V(tma_bar_ptr=pipeline_dO.producer_get_barrier(producer_state_dO_dPsum)) - load_dO(m_block_safe, producer_state=producer_state_dO_dPsum) - pipeline_dO.producer_commit(producer_state_dO_dPsum) - pipeline_dPsum.producer_acquire(producer_state_dO_dPsum) - with cute.arch.elect_one(): - copy_stats( - gdPsum[None, m_block_safe], - sdPsum[None, producer_state_dO_dPsum.index], - mbar_ptr=pipeline_dPsum.producer_get_barrier(producer_state_dO_dPsum), - ) - producer_state_dO_dPsum.advance() - else: - # Subsequent blocks: just load Q/dO (K/V already loaded) - if const_expr(should_load_Q): - pipeline_Q.producer_acquire(producer_state_Q_LSE) - load_Q(m_block_safe, producer_state=producer_state_Q_LSE) - pipeline_Q.producer_commit(producer_state_Q_LSE) - pipeline_LSE.producer_acquire(producer_state_Q_LSE) - with cute.arch.elect_one(): - copy_stats( - gLSE[None, m_block_safe], - sLSE[None, producer_state_Q_LSE.index], - mbar_ptr=pipeline_LSE.producer_get_barrier(producer_state_Q_LSE), - ) - producer_state_Q_LSE.advance() - if const_expr(should_load_dO): - pipeline_dO.producer_acquire(producer_state_dO_dPsum) - load_dO(m_block_safe, producer_state=producer_state_dO_dPsum) - pipeline_dO.producer_commit(producer_state_dO_dPsum) - pipeline_dPsum.producer_acquire(producer_state_dO_dPsum) - with cute.arch.elect_one(): - copy_stats( - gdPsum[None, m_block_safe], - sdPsum[None, producer_state_dO_dPsum.index], - mbar_ptr=pipeline_dPsum.producer_get_barrier(producer_state_dO_dPsum), - ) - producer_state_dO_dPsum.advance() - - return producer_state_Q_LSE, producer_state_dO_dPsum - - -@cute.jit -def get_block_sparse_iteration_info_bwd( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - subtile_factor: cutlass.Constexpr = 1, - m_block_max: int = 0, -): - """Extract block-sparse iteration info for backward pass. - - Returns (curr_q_cnt, curr_q_idx, curr_full_cnt, curr_full_idx, total_count). - """ - q_cnt, q_idx, full_cnt, full_idx = blocksparse_tensors - curr_q_cnt = q_cnt[batch_idx, head_idx, n_block] - curr_q_idx = q_idx[batch_idx, head_idx, n_block, None] - - if const_expr(full_cnt is not None): - curr_full_cnt = full_cnt[batch_idx, head_idx, n_block] - curr_full_idx = full_idx[batch_idx, head_idx, n_block, None] - else: - curr_full_cnt = Int32(0) - curr_full_idx = None - - sparse_block_count = curr_q_cnt - if const_expr(full_cnt is not None): - sparse_block_count = sparse_block_count + curr_full_cnt - total_count = sparse_block_count * subtile_factor - - return curr_q_cnt, curr_q_idx, curr_full_cnt, curr_full_idx, total_count - - -@cute.jit -def get_m_block_from_iter_bwd( - iter_idx, - curr_q_cnt, - curr_q_idx: cute.Tensor, - curr_full_cnt, - curr_full_idx: Optional[cute.Tensor], - subtile_factor: cutlass.Constexpr = 1, - m_block_max: int = 0, -): - """Derive m_block index and is_full_block flag from iteration index. - - Returns (m_block, is_full_block): - - m_block: The actual Q-tile block index - - is_full_block: True if this is a full block (no mask_mod needed) - """ - sparse_iter_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - - sparse_m_block = Int32(0) - is_full_block = False - if const_expr(curr_full_idx is not None): - if sparse_iter_idx < curr_q_cnt: - sparse_m_block = curr_q_idx[sparse_iter_idx] - else: - sparse_m_block = curr_full_idx[sparse_iter_idx - curr_q_cnt] - is_full_block = True - else: - sparse_m_block = curr_q_idx[sparse_iter_idx] - - return sparse_m_block * subtile_factor + subtile_offset, is_full_block - - -@cute.jit -def _load_q_do_block_sm90( - m_block, - producer_state_Q, - producer_state_dO, - pipeline_Q, - pipeline_dO, - load_K, - load_V, - load_Q, - load_dO, - load_LSE, - load_dPsum, - tma_copy_bytes_K, - tma_copy_bytes_V, - Q_stage_eq_dO_stage: cutlass.Constexpr, - load_kv: bool, -): - """Load one Q/dO block, optionally loading K/V on first iteration.""" - if load_kv: - pipeline_Q.producer_acquire(producer_state_Q, extra_tx_count=tma_copy_bytes_K) - load_K(tma_bar_ptr=pipeline_Q.producer_get_barrier(producer_state_Q)) - else: - pipeline_Q.producer_acquire(producer_state_Q) - load_Q(m_block, producer_state=producer_state_Q) - with cute.arch.elect_one(): - load_LSE(m_block, producer_state=producer_state_Q) - - producer_state_dO_cur = ( - producer_state_dO if const_expr(not Q_stage_eq_dO_stage) else producer_state_Q - ) - if load_kv: - pipeline_dO.producer_acquire(producer_state_dO_cur, extra_tx_count=tma_copy_bytes_V) - load_V(tma_bar_ptr=pipeline_dO.producer_get_barrier(producer_state_dO_cur)) - else: - pipeline_dO.producer_acquire(producer_state_dO_cur) - load_dO(m_block, producer_state=producer_state_dO_cur) - with cute.arch.elect_one(): - load_dPsum(m_block, producer_state=producer_state_dO_cur) - - producer_state_Q.advance() - producer_state_dO.advance() - return producer_state_Q, producer_state_dO - - -@cute.jit -def produce_block_sparse_q_loads_bwd_sm90( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - producer_state_Q, - producer_state_dO, - pipeline_Q, - pipeline_dO, - load_K, - load_V, - load_Q, - load_dO, - load_LSE, - load_dPsum, - tma_copy_bytes_K, - tma_copy_bytes_V, - Q_stage_eq_dO_stage: cutlass.Constexpr, - subtile_factor: cutlass.Constexpr, - m_block_max: int, -): - """SM90 backward block sparse loading with separate partial/full loops. - - K/V are loaded with the first valid block. Iterates partial blocks first, - then full blocks, matching consumer order. - - Returns updated (producer_state_Q, producer_state_dO). - """ - q_cnt, q_idx, full_cnt, full_idx = blocksparse_tensors - curr_q_cnt = q_cnt[batch_idx, head_idx, n_block] - curr_q_idx = q_idx[batch_idx, head_idx, n_block, None] - - if const_expr(full_cnt is not None): - curr_full_cnt = full_cnt[batch_idx, head_idx, n_block] - curr_full_idx = full_idx[batch_idx, head_idx, n_block, None] - else: - curr_full_cnt = Int32(0) - curr_full_idx = None - - kv_loaded = False - - for iter_idx in cutlass.range(curr_q_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_q_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - producer_state_Q, producer_state_dO = _load_q_do_block_sm90( - m_block, - producer_state_Q, - producer_state_dO, - pipeline_Q, - pipeline_dO, - load_K, - load_V, - load_Q, - load_dO, - load_LSE, - load_dPsum, - tma_copy_bytes_K, - tma_copy_bytes_V, - Q_stage_eq_dO_stage, - load_kv=not kv_loaded, - ) - kv_loaded = True - - if const_expr(full_cnt is not None): - for iter_idx in cutlass.range(curr_full_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_full_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - producer_state_Q, producer_state_dO = _load_q_do_block_sm90( - m_block, - producer_state_Q, - producer_state_dO, - pipeline_Q, - pipeline_dO, - load_K, - load_V, - load_Q, - load_dO, - load_LSE, - load_dPsum, - tma_copy_bytes_K, - tma_copy_bytes_V, - Q_stage_eq_dO_stage, - load_kv=not kv_loaded, - ) - kv_loaded = True - - return producer_state_Q, producer_state_dO - - -@cute.jit -def consume_block_sparse_mma_bwd_sm90( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - consumer_state_Q, - consumer_state_dO, - mma_one_m_block_fn, - mask, - mask_mod, - is_causal: cutlass.Constexpr, - is_local: cutlass.Constexpr, - thr_mma_SdP, - softmax_scale, - seqlen, - subtile_factor: cutlass.Constexpr, - m_block_max: int, - aux_tensors=None, - fastdiv_mods=(None, None), -): - """SM90 backward block sparse MMA consumption with separate partial/full loops. - - Partial blocks are processed first (with mask_mod applied), then full blocks - (without mask_mod). This ensures mask_mod is only applied where needed. - - Returns updated (consumer_state_Q, consumer_state_dO). - """ - q_cnt, q_idx, full_cnt, full_idx = blocksparse_tensors - curr_q_cnt = q_cnt[batch_idx, head_idx, n_block] - curr_q_idx = q_idx[batch_idx, head_idx, n_block, None] - - if const_expr(full_cnt is not None): - curr_full_cnt = full_cnt[batch_idx, head_idx, n_block] - curr_full_idx = full_idx[batch_idx, head_idx, n_block, None] - else: - curr_full_cnt = Int32(0) - curr_full_idx = None - - dKV_accumulate = False - - mask_fn_partial = partial( - mask.apply_mask, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - thr_mma=thr_mma_SdP, - mask_seqlen=True, - mask_causal=is_causal, - mask_local=is_local, - mask_mod=mask_mod, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - mask_fn_full = partial( - mask.apply_mask, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - thr_mma=thr_mma_SdP, - mask_seqlen=True, - mask_causal=is_causal, - mask_local=is_local, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - for iter_idx in cutlass.range(curr_q_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_q_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - consumer_state_Q, consumer_state_dO = mma_one_m_block_fn( - m_block, - consumer_state_Q, - consumer_state_dO, - mask_fn=mask_fn_partial, - dKV_accumulate=dKV_accumulate, - thr_mma_SdP=thr_mma_SdP, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - softmax_scale=softmax_scale, - seqlen=seqlen, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - dKV_accumulate = True - - if const_expr(full_cnt is not None): - for iter_idx in cutlass.range(curr_full_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_full_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - consumer_state_Q, consumer_state_dO = mma_one_m_block_fn( - m_block, - consumer_state_Q, - consumer_state_dO, - mask_fn=mask_fn_full, - dKV_accumulate=dKV_accumulate, - thr_mma_SdP=thr_mma_SdP, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - softmax_scale=softmax_scale, - seqlen=seqlen, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - dKV_accumulate = True - - return consumer_state_Q, consumer_state_dO - - -@cute.jit -def _store_one_dQaccum_sm90( - m_block, - sdQaccum: cute.Tensor, - gdQaccum: cute.Tensor, - num_mma_warp_groups: cutlass.Constexpr, - num_threads_per_warp_group: cutlass.Constexpr, - tma_copy_bytes_dQ, -): - """Store dQaccum for a single m_block.""" - for warp_group_idx in cutlass.range_constexpr(num_mma_warp_groups): - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.dQFullWG0) + warp_group_idx, - number_of_threads=num_threads_per_warp_group + cute.arch.WARP_SIZE, - ) - with cute.arch.elect_one(): - copy_utils.cpasync_reduce_bulk_add_f32( - sdQaccum[None, warp_group_idx].iterator, - gdQaccum[None, warp_group_idx, m_block].iterator, - tma_copy_bytes_dQ, - ) - cute.arch.cp_async_bulk_commit_group() - for warp_group_idx in cutlass.range_constexpr(num_mma_warp_groups): - cute.arch.cp_async_bulk_wait_group(num_mma_warp_groups - 1 - warp_group_idx, read=True) - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierBwd.dQEmptyWG0) + warp_group_idx, - number_of_threads=num_threads_per_warp_group + cute.arch.WARP_SIZE, - ) - - -@cute.jit -def dQaccum_store_block_sparse_bwd_sm90( - blocksparse_tensors: BlockSparseTensors, - batch_idx, - head_idx, - n_block, - sdQaccum: cute.Tensor, - gdQaccum: cute.Tensor, - subtile_factor: cutlass.Constexpr, - m_block_max: int, - num_mma_warp_groups: cutlass.Constexpr, - num_threads_per_warp_group: cutlass.Constexpr, - tma_copy_bytes_dQ, -): - """SM90 backward block sparse dQaccum store with separate partial/full loops. - - Iterates partial blocks first, then full blocks, matching producer/consumer order. - """ - q_cnt, q_idx, full_cnt, full_idx = blocksparse_tensors - curr_q_cnt = q_cnt[batch_idx, head_idx, n_block] - curr_q_idx = q_idx[batch_idx, head_idx, n_block, None] - - if const_expr(full_cnt is not None): - curr_full_cnt = full_cnt[batch_idx, head_idx, n_block] - curr_full_idx = full_idx[batch_idx, head_idx, n_block, None] - else: - curr_full_cnt = Int32(0) - curr_full_idx = None - - for iter_idx in cutlass.range(curr_q_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_q_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - _store_one_dQaccum_sm90( - m_block, - sdQaccum, - gdQaccum, - num_mma_warp_groups, - num_threads_per_warp_group, - tma_copy_bytes_dQ, - ) - - if const_expr(full_cnt is not None): - for iter_idx in cutlass.range(curr_full_cnt * subtile_factor, unroll=1): - sparse_idx = iter_idx // subtile_factor - subtile_offset = iter_idx % subtile_factor - m_block = curr_full_idx[sparse_idx] * subtile_factor + subtile_offset - - if m_block < m_block_max: - _store_one_dQaccum_sm90( - m_block, - sdQaccum, - gdQaccum, - num_mma_warp_groups, - num_threads_per_warp_group, - tma_copy_bytes_dQ, - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparsity.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparsity.py deleted file mode 100644 index 7e4afb7c215d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/block_sparsity.py +++ /dev/null @@ -1,218 +0,0 @@ -""" -Block-sparsity utilities for FlexAttention -""" - -from typing import Callable, NamedTuple, Tuple - -import cutlass.cute as cute -import torch - -from .cute_dsl_utils import to_cute_tensor - - -def ceildiv(a: int, b: int) -> int: - return (a + b - 1) // b - - -class BlockSparseTensors(NamedTuple): - mask_block_cnt: cute.Tensor - mask_block_idx: cute.Tensor - full_block_cnt: cute.Tensor | None - full_block_idx: cute.Tensor | None - - def __new_from_mlir_values__(self, values): - if len(values) == 2: - values = (*values, None, None) - return BlockSparseTensors(*values) - - -class BlockSparseTensorsTorch(NamedTuple): - mask_block_cnt: torch.Tensor - mask_block_idx: torch.Tensor - full_block_cnt: torch.Tensor | None = None - full_block_idx: torch.Tensor | None = None - - -def _expand_sparsity_tensor( - tensor: torch.Tensor, - expected_shape: Tuple[int, ...], - tensor_name: str, - context: str | None, - hint: str | Callable[[], str] | None, -) -> torch.Tensor: - """Check if we need to expand the tensor to expected shape, and do so if possible.""" - needs_expand = tensor.shape != expected_shape - if not needs_expand: - return tensor - can_expand = all(map(lambda cur, tgt: cur == tgt or cur == 1, tensor.shape, expected_shape)) - if not can_expand: - context_clause = f" ({context})" if context else "" - resolved_hint = hint() if callable(hint) else hint - hint_clause = f" Hint: {resolved_hint}" if resolved_hint else "" - raise ValueError( - f"{tensor_name}{context_clause} with shape {tensor.shape} cannot be expanded to expected shape {expected_shape}." - f"{hint_clause}" - ) - return tensor.expand(*expected_shape).contiguous() - - -def _check_and_expand_block( - name: str, - cnt: torch.Tensor | None, - idx: torch.Tensor | None, - expected_count_shape: Tuple[int, int, int], - expected_index_shape: Tuple[int, int, int, int], - context: str | None, - hint: str | Callable[[], str] | None, -) -> Tuple[torch.Tensor | None, torch.Tensor | None]: - if (cnt is None) != (idx is None): - raise ValueError( - f"{name}_block_cnt and {name}_block_idx must both be provided or both be None" - ) - if cnt is None or idx is None: - return None, None - if cnt.dtype != torch.int32 or idx.dtype != torch.int32: - raise ValueError(f"{name}_block tensors must have dtype torch.int32") - if cnt.device != idx.device: - raise ValueError(f"{name}_block_cnt and {name}_block_idx must be on the same device") - if not cnt.is_cuda or not idx.is_cuda: - raise ValueError(f"{name}_block tensors must live on CUDA") - expanded_cnt = _expand_sparsity_tensor( - cnt, expected_count_shape, f"{name}_block_cnt", context, hint - ) - expanded_idx = _expand_sparsity_tensor( - idx, expected_index_shape, f"{name}_block_idx", context, hint - ) - return expanded_cnt, expanded_idx - - -def get_block_sparse_expected_shapes( - batch_size: int, - num_head: int, - seqlen_q: int, - seqlen_k: int, - m_block_size: int, - n_block_size: int, - q_stage: int, -) -> Tuple[Tuple[int, int, int], Tuple[int, int, int, int]]: - """Return (expected_count_shape, expected_index_shape) for block sparse normalization.""" - m_block_size_effective = q_stage * m_block_size - expected_m_blocks = ceildiv(seqlen_q, m_block_size_effective) - expected_n_blocks = ceildiv(seqlen_k, n_block_size) - expected_count_shape = (batch_size, num_head, expected_m_blocks) - expected_index_shape = (batch_size, num_head, expected_m_blocks, expected_n_blocks) - return expected_count_shape, expected_index_shape - - -def get_block_sparse_expected_shapes_bwd( - batch_size: int, - num_head: int, - seqlen_q: int, - seqlen_k: int, - m_block_size: int, - n_block_size: int, - subtile_factor: int, -) -> Tuple[Tuple[int, int, int], Tuple[int, int, int, int]]: - """Return (expected_count_shape, expected_index_shape) for backward block sparse normalization. - - Backward uses Q-direction indexing (transposed from forward), where shapes are - indexed by N-blocks first, then M-blocks. The sparse_block_size_q is determined - by subtile_factor * m_block_size. - """ - sparse_block_size_q = subtile_factor * m_block_size - expected_m_blocks = ceildiv(seqlen_q, sparse_block_size_q) - expected_n_blocks = ceildiv(seqlen_k, n_block_size) - expected_count_shape = (batch_size, num_head, expected_n_blocks) - expected_index_shape = (batch_size, num_head, expected_n_blocks, expected_m_blocks) - return expected_count_shape, expected_index_shape - - -def normalize_block_sparse_tensors( - tensors: BlockSparseTensorsTorch, - *, - expected_count_shape: Tuple[int, int, int], - expected_index_shape: Tuple[int, int, int, int], - context: str | None = None, - hint: str | Callable[[], str] | None = None, -) -> BlockSparseTensorsTorch: - if tensors.mask_block_cnt is None or tensors.mask_block_idx is None: - raise ValueError("mask_block_cnt and mask_block_idx must be provided for block sparsity.") - - mask_cnt, mask_idx = _check_and_expand_block( - "mask", - tensors.mask_block_cnt, - tensors.mask_block_idx, - expected_count_shape, - expected_index_shape, - context, - hint, - ) - if mask_cnt is None or mask_idx is None: - raise ValueError("mask_block_cnt and mask_block_idx must be provided for block sparsity.") - - full_cnt, full_idx = _check_and_expand_block( - "full", - tensors.full_block_cnt, - tensors.full_block_idx, - expected_count_shape, - expected_index_shape, - context, - hint, - ) - if full_cnt is not None and mask_cnt.device != full_cnt.device: - raise ValueError("All block sparse tensors must be on the same device") - - return BlockSparseTensorsTorch( - mask_block_cnt=mask_cnt, - mask_block_idx=mask_idx, - full_block_cnt=full_cnt, - full_block_idx=full_idx, - ) - - -def is_block_sparsity_enabled(tensors: BlockSparseTensorsTorch) -> bool: - return any(t is not None for t in (tensors.full_block_cnt, tensors.mask_block_cnt)) - - -def to_cute_block_sparse_tensors( - tensors: BlockSparseTensorsTorch, enable_tvm_ffi: bool = True -) -> BlockSparseTensors | None: - """Convert torch block sparsity tensors to CuTe tensors, optionally for tvm ffi""" - if not is_block_sparsity_enabled(tensors): - return None - ( - mask_block_cnt, - mask_block_idx, - full_block_cnt, - full_block_idx, - ) = tensors - - ( - mask_block_cnt_tensor, - mask_block_idx_tensor, - ) = [ - to_cute_tensor(t, assumed_align=4, leading_dim=-1, enable_tvm_ffi=enable_tvm_ffi) - for t in (mask_block_cnt, mask_block_idx) - ] - ( - full_block_cnt_tensor, - full_block_idx_tensor, - ) = [ - to_cute_tensor(t, assumed_align=4, leading_dim=-1, enable_tvm_ffi=enable_tvm_ffi) - if t is not None - else None - for t in (full_block_cnt, full_block_idx) - ] - - return BlockSparseTensors( - mask_block_cnt_tensor, - mask_block_idx_tensor, - full_block_cnt_tensor, - full_block_idx_tensor, - ) - - -def fast_sampling(mask_mod): - """Convenience decorator to mark mask_mod as safe for 5-point fast sampling""" - mask_mod.use_fast_sampling = True - return mask_mod diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/compute_block_sparsity.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/compute_block_sparsity.py deleted file mode 100644 index 062ecb28878a..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/compute_block_sparsity.py +++ /dev/null @@ -1,377 +0,0 @@ -from functools import partial -from typing import Callable, Optional, Tuple - -import cutlass -import cutlass.cute as cute -import torch -from cutlass import Boolean, Int8, Int32, const_expr - -from .block_sparsity import ( - BlockSparseTensors, - BlockSparseTensorsTorch, - to_cute_block_sparse_tensors, -) -from .utils import hash_callable, scalar_to_ssa, ssa_to_scalar -from .seqlen_info import SeqlenInfoQK - - -class BlockSparsityKernel: - """Block sparsity kernel for FlexAttention. - - This kernel computes `mask_mod` for every token of each block - to determine if an n block is full, masked, or neither. - - Writes block counts and indices to a BlockSparseTensors object. - - When use_fast_sampling=True, uses 5-point sampling (4 corners + center) - which is much faster but only suitable for masks where this is sufficient. - - TODO: - - optimize mask_mod evaluation - - varlen support - - transposed tensors for bwd pass - """ - - def __init__( - self, - mask_mod: Callable, - tile_mn: Tuple[int, int], - compute_full_blocks: bool = True, - use_aux_tensors: bool = False, - use_fast_sampling: bool = False, - ): - self.mask_mod = mask_mod - self.tile_mn = tile_mn - self.compute_full_blocks = compute_full_blocks - self.use_aux_tensors = use_aux_tensors - self.use_fast_sampling = use_fast_sampling - - @cute.jit - def __call__( - self, - blocksparse_tensors: BlockSparseTensors, - seqlen_q: Int32, - seqlen_k: Int32, - aux_tensors: Optional[list] = None, - ): - self.mask_cnt, self.mask_idx, self.full_cnt, self.full_idx = blocksparse_tensors - - if const_expr(self.compute_full_blocks): - assert self.full_cnt is not None and self.full_idx is not None, ( - "full block tensors must be provided when computing full blocks" - ) - - batch_size, num_heads, num_m_blocks, num_n_blocks = self.mask_idx.shape - # launch 1 CTA per m block - grid = [num_m_blocks, num_heads, batch_size] - - if const_expr(self.use_fast_sampling): - num_threads = 5 - self.num_warps = 1 - else: - num_threads = self.tile_mn[0] - self.num_warps = (num_threads + 32 - 1) // 32 - - self.kernel( - self.mask_cnt, - self.mask_idx, - self.full_cnt, - self.full_idx, - num_n_blocks, - seqlen_q, - seqlen_k, - aux_tensors, - ).launch(grid=grid, block=[num_threads, 1, 1]) - - @cute.kernel - def kernel( - self, - mask_cnt: cute.Tensor, - mask_idx: cute.Tensor, - full_cnt: cute.Tensor, - full_idx: cute.Tensor, - num_n_blocks: Int32, - seqlen_q: Int32, - seqlen_k: Int32, - aux_tensors: Optional[list] = None, - ): - tidx, _, _ = cute.arch.thread_idx() - warp_idx = cute.arch.warp_idx() - lane_id = cute.arch.lane_idx() - m_block, head_idx, batch_idx = cute.arch.block_idx() - - ssa = partial(scalar_to_ssa, dtype=Int32) - - seqlen = SeqlenInfoQK.create( - batch_idx, - seqlen_q, - seqlen_k, - mCuSeqlensQ=None, - mCuSeqlensK=None, - mSeqUsedQ=None, - mSeqUsedK=None, - ) - - @cute.struct - class SharedStorage: - reduction_buffer_smem: cute.struct.Align[ - cute.struct.MemRange[cutlass.Int8, 2 * self.num_warps], 1024 - ] - - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage, 16) - - reduction_buffer = storage.reduction_buffer_smem.get_tensor( - cute.make_layout((self.num_warps, 2)) - ) - - num_mask_blocks = Int32(0) - num_full_blocks = Int32(0) - - for n_block in cutlass.range(num_n_blocks, unroll_full=True): - m_base = m_block * self.tile_mn[0] - n_base = n_block * self.tile_mn[1] - - if const_expr(self.use_fast_sampling): - # Fast path: 5-point sampling (4 corners + center) - # Clamps OOB indices to nearest in bounds. - thread_result = Boolean(False) - thread_is_valid = Boolean(False) - q_idx = Int32(0) - kv_idx = Int32(0) - - if tidx == 0: - # Top-left corner (0, 0); always in bounds - q_idx = m_base - kv_idx = n_base - elif tidx == 1: - # Top-right corner - q_idx = m_base - kv_idx = cutlass.min(n_base + self.tile_mn[1] - 1, seqlen_k - 1) - elif tidx == 2: - # Bottom-left corner - q_idx = cutlass.min(m_base + self.tile_mn[0] - 1, seqlen_q - 1) - kv_idx = n_base - elif tidx == 3: - # Bottom-right corner - q_idx = cutlass.min(m_base + self.tile_mn[0] - 1, seqlen_q - 1) - kv_idx = cutlass.min(n_base + self.tile_mn[1] - 1, seqlen_k - 1) - elif tidx == 4: - # Center point - q_idx = m_base + (cutlass.min(seqlen_q - m_base, self.tile_mn[0])) // 2 - kv_idx = n_base + (cutlass.min(seqlen_k - n_base, self.tile_mn[1])) // 2 - else: - thread_is_valid = Boolean(False) - - # Check bounds and determine if this thread has a valid index pair - if tidx < 5 and q_idx < seqlen_q and kv_idx < seqlen_k: - thread_is_valid = Boolean(True) - q_idx_ssa = ssa(q_idx) - kv_idx_ssa = ssa(kv_idx) - thread_result = ssa_to_scalar( - self.mask_mod( - ssa(batch_idx), - ssa(head_idx), - q_idx_ssa, - kv_idx_ssa, - seqlen, - aux_tensors, - ) - ) - else: - thread_is_valid = Boolean(False) - - # Use vote_any_sync to see if any valid thread found unmasked or masked - # Only count results from threads that checked valid indices - has_unmasked = cute.arch.vote_any_sync(thread_result & thread_is_valid) - has_masked = cute.arch.vote_any_sync((Boolean(not thread_result)) & thread_is_valid) - - else: - # Full path: check all elements in the block - # Track if this thread's row has any masked or unmasked elements - thread_has_unmasked = Boolean(False) - thread_has_masked = Boolean(False) - thread_is_valid = Boolean(False) - - # Each thread handles 1 row - q_idx = m_base + tidx - kv_idx = Int32(0) - if tidx < self.tile_mn[0] and q_idx < seqlen_q: - thread_is_valid = Boolean(True) - q_idx_ssa = ssa(q_idx) - - # Loop over all columns in this row - for c in cutlass.range(self.tile_mn[1], unroll_full=True): - kv_idx = n_base + c - kv_idx_ssa = ssa(kv_idx) - - # Only check elements within valid sequence bounds - if kv_idx < seqlen_k: - # Direct scalar call - mask_val = ssa_to_scalar( - self.mask_mod( - ssa(batch_idx), - ssa(head_idx), - q_idx_ssa, - kv_idx_ssa, - seqlen, - aux_tensors, - ) - ) - - # Update tracking flags - if mask_val: - thread_has_unmasked = Boolean(True) - else: - thread_has_masked = Boolean(True) - - # Block-level reduction to combine results across all threads - # Only count votes from threads that checked valid indices - warp_has_unmasked_mask = cute.arch.vote_any_sync( - thread_has_unmasked & thread_is_valid - ) - warp_has_masked_mask = cute.arch.vote_any_sync(thread_has_masked & thread_is_valid) - - # lane 0 writes the ballot mask to shared memory - lane_id = tidx % 32 - if lane_id == 0: - # Store as Int8 - reduction_buffer[warp_idx, 0] = Int8(1) if warp_has_unmasked_mask else Int8(0) - reduction_buffer[warp_idx, 1] = Int8(1) if warp_has_masked_mask else Int8(0) - - cute.arch.sync_threads() - - # Thread 0 ORs all warp results together - has_unmasked = Boolean(False) - has_masked = Boolean(False) - if tidx == 0: - for w in cutlass.range(self.num_warps): - if reduction_buffer[w, 0]: - has_unmasked = Boolean(True) - if reduction_buffer[w, 1]: - has_masked = Boolean(True) - - # Only thread 0 updates the output arrays (common to both paths) - if tidx == 0: - # Block classification based on what we found: - # - If has_masked and has_unmasked: partial block (needs masking) - # - If only has_unmasked: full block (no masking needed) - # - If only has_masked: skip this block entirely - is_partial = Boolean(has_masked and has_unmasked) - is_full = Boolean(has_unmasked and (not has_masked)) - - if is_partial: - mask_idx[batch_idx, head_idx, m_block, num_mask_blocks] = n_block - num_mask_blocks += 1 - elif is_full and const_expr(self.compute_full_blocks): - full_idx[batch_idx, head_idx, m_block, num_full_blocks] = n_block - num_full_blocks += 1 - - # Only thread 0 writes back the counts - if tidx == 0: - mask_cnt[batch_idx, head_idx, m_block] = num_mask_blocks - if const_expr(self.compute_full_blocks): - full_cnt[batch_idx, head_idx, m_block] = num_full_blocks - - -def compute_block_sparsity( - tile_m, - tile_n, - batch_size, - num_heads, - seqlen_q, - seqlen_k, - mask_mod: Callable, - aux_tensors: Optional[list], # list[cute.Tensor] - device, - compute_full_blocks: bool = True, - use_fast_sampling: bool = False, -) -> Tuple[BlockSparseTensors, BlockSparseTensorsTorch]: - """ - Computes block sparsity for a given `mask_mod`. - - Args: - tile_m: The tile size for the m dimension. - tile_n: The tile size for the n dimension. - batch_size: The batch size. - num_heads: The number of heads. - seqlen_q: The sequence length for the query. - seqlen_k: The sequence length for the key. - mask_mod: The `mask_mod` callable to use. - aux_tensors: A list of auxiliary tensors. - device: The device to use. - compute_full_blocks: Whether to compute full blocks. If False, only partially-masked blocks are computed. - use_fast_sampling: Whether to use 5-point sampling (4 corners + center). This is much faster, but only suitable for masks where this check is sufficient. - - Returns: - A tuple of `BlockSparseTensors` and `BlockSparseTensorsTorch`. - """ - # Check if mask_mod is marked as suitable for 5-point fast sampling - use_fast_sampling = getattr(mask_mod, "use_fast_sampling", use_fast_sampling) - - num_m_blocks = (seqlen_q + tile_m - 1) // tile_m - num_n_blocks = (seqlen_k + tile_n - 1) // tile_n - - mask_block_cnt = torch.zeros( - (batch_size, num_heads, num_m_blocks), device=device, dtype=torch.int32 - ) - mask_block_idx = torch.zeros( - (batch_size, num_heads, num_m_blocks, num_n_blocks), device=device, dtype=torch.int32 - ) - full_block_cnt = ( - torch.zeros((batch_size, num_heads, num_m_blocks), device=device, dtype=torch.int32) - if compute_full_blocks - else None - ) - full_block_idx = ( - torch.zeros( - (batch_size, num_heads, num_m_blocks, num_n_blocks), device=device, dtype=torch.int32 - ) - if compute_full_blocks - else None - ) - - blocksparse_tensors_torch = BlockSparseTensorsTorch( - mask_block_cnt=mask_block_cnt, - mask_block_idx=mask_block_idx, - full_block_cnt=full_block_cnt, - full_block_idx=full_block_idx, - ) - - mask_mod_hash = hash_callable(mask_mod) - blocksparse_tensors = to_cute_block_sparse_tensors( - blocksparse_tensors_torch, enable_tvm_ffi=True - ) - - compile_key = ( - tile_m, - tile_n, - mask_mod_hash, - compute_full_blocks, - aux_tensors is not None, - use_fast_sampling, - ) - if compile_key not in compute_block_sparsity.compile_cache: - kernel = BlockSparsityKernel( - mask_mod, - tile_mn=(tile_m, tile_n), - compute_full_blocks=compute_full_blocks, - use_aux_tensors=aux_tensors is not None, - use_fast_sampling=use_fast_sampling, - ) - - compute_block_sparsity.compile_cache[compile_key] = cute.compile( - kernel, blocksparse_tensors, seqlen_q, seqlen_k, aux_tensors, options="--enable-tvm-ffi" - ) - - compute_block_sparsity.compile_cache[compile_key]( - blocksparse_tensors_torch, - seqlen_q, - seqlen_k, - aux_tensors, - ) - - return blocksparse_tensors, blocksparse_tensors_torch - - -compute_block_sparsity.compile_cache = {} diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/copy_utils.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/copy_utils.py deleted file mode 100644 index cfdcbdb80a09..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/copy_utils.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright (c) 2025, Wentao Guo, Ted Zadouri, Tri Dao. - -import math -from typing import Optional, Type, Callable - -import cutlass -import cutlass.cute as cute -from cutlass import Float32, Int32, const_expr -from cutlass.cute.nvgpu import cpasync -import cutlass.utils.blackwell_helpers as sm100_utils -from cutlass.cutlass_dsl import T, dsl_user_op -from cutlass._mlir.dialects import llvm -import cutlass.pipeline - - -@dsl_user_op -def cvt_copy( - atom: cute.CopyAtom, - src: cute.Tensor, - dst: cute.Tensor, - *, - pred: Optional[cute.Tensor] = None, - loc=None, - ip=None, - **kwargs, -) -> None: - assert isinstance(src.iterator, cute.Pointer) and src.memspace == cute.AddressSpace.rmem - if const_expr(src.element_type != dst.element_type): - src_cvt = cute.make_fragment_like(src, dst.element_type, loc=loc, ip=ip) - src_cvt.store(src.load().to(dst.element_type)) - src = src_cvt - cute.copy(atom, src, dst, pred=pred, loc=loc, ip=ip, **kwargs) - - -@dsl_user_op -def load_s2r(src: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: - dst = cute.make_fragment_like(src, src.element_type, loc=loc, ip=ip) - cute.autovec_copy(src, dst, loc=loc, ip=ip) - return dst - - -@dsl_user_op -def get_copy_atom( - dtype: Type[cutlass.Numeric], num_copy_elems: int, is_async: bool = False, *, loc=None, ip=None -) -> cute.CopyAtom: - num_copy_bits = const_expr(min(128, num_copy_elems * dtype.width)) - copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp() - return cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits) - - -@dsl_user_op -def make_tmem_copy( - tmem_copy_atom: cute.CopyAtom, num_wg: int = 1, *, loc=None, ip=None -) -> cute.CopyAtom: - num_dp, num_bits, num_rep, _ = sm100_utils.get_tmem_copy_properties(tmem_copy_atom) - assert num_dp == 32 - assert num_bits == 32 - tiler_mn = (cute.make_layout((128 * num_rep * num_wg // 32, 32), stride=(32, 1)),) - layout_tv = cute.make_layout( - ((32, 4, num_wg), (num_rep, 32)), stride=((0, 1, 4 * num_rep), (4, 4 * num_rep * num_wg)) - ) - return cute.make_tiled_copy(tmem_copy_atom, layout_tv, tiler_mn) - - -@dsl_user_op -def copy( - src: cute.Tensor, - dst: cute.Tensor, - *, - pred: Optional[cute.Tensor] = None, - num_copy_elems: int = 1, - is_async: bool = False, - loc=None, - ip=None, - **kwargs, -) -> None: - copy_atom = get_copy_atom(src.element_type, num_copy_elems, is_async) - cute.copy(copy_atom, src, dst, pred=pred, loc=loc, ip=ip, **kwargs) - - -def tiled_copy_1d( - dtype: Type[cutlass.Numeric], num_threads: int, num_copy_elems: int = 1, is_async: bool = False -) -> cute.TiledCopy: - num_copy_bits = num_copy_elems * dtype.width - copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp() - copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits) - thr_layout = cute.make_layout(num_threads) - val_layout = cute.make_layout(num_copy_elems) - return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout) - - -def tiled_copy_2d( - dtype: Type[cutlass.Numeric], major_mode_size: int, num_threads: int, is_async: bool = False -) -> cute.TiledCopy: - num_copy_bits = math.gcd(major_mode_size, 128 // dtype.width) * dtype.width - copy_elems = num_copy_bits // dtype.width - copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp() - copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits) - gmem_threads_per_row = major_mode_size // copy_elems - assert num_threads % gmem_threads_per_row == 0 - thr_layout = cute.make_ordered_layout( - (num_threads // gmem_threads_per_row, gmem_threads_per_row), - order=(1, 0), - ) - val_layout = cute.make_layout((1, copy_elems)) - return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout) - - -@dsl_user_op -def atomic_add_fp32x4( - a: Float32, b: Float32, c: Float32, d: Float32, gmem_ptr: cute.Pointer, *, loc=None, ip=None -) -> None: - gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value() - # cache_hint = cutlass.Int64(0x12F0000000000000) - llvm.inline_asm( - None, - [ - gmem_ptr_i64, - Float32(a).ir_value(loc=loc, ip=ip), - Float32(b).ir_value(loc=loc, ip=ip), - Float32(c).ir_value(loc=loc, ip=ip), - Float32(d).ir_value(loc=loc, ip=ip), - ], - # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip), cache_hint.ir_value()], - "{\n\t" - # ".reg .b128 abcd;\n\t" - # "mov.b128 abcd, {$1, $2, $3, $4};\n\t" - ".reg .v4 .f32 abcd;\n\t" - # "mov.b128 abcd, {$1, $2, $3, $4};\n\t" - "mov.f32 abcd.x, $1;\n\t" - "mov.f32 abcd.y, $2;\n\t" - "mov.f32 abcd.z, $3;\n\t" - "mov.f32 abcd.w, $4;\n\t" - "red.global.add.v4.f32 [$0], abcd;\n\t" - # "red.global.add.L2::cache_hint.v4.f32 [$0], abcd, 0x14F0000000000000;\n\t" - "}\n", - # "red.global.add.L2::cache_hint.f32 [$0], $1, 0x12F0000000000000;", - # "red.global.add.L2::cache_hint.f32 [$0], $1, $2;", - "l,f,f,f,f", - # "l,f,l", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@dsl_user_op -def set_block_rank( - smem_ptr: cute.Pointer, peer_cta_rank_in_cluster: Int32, *, loc=None, ip=None -) -> Int32: - """Map the given smem pointer to the address at another CTA rank in the cluster.""" - smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value() - return Int32( - llvm.inline_asm( - T.i32(), - [smem_ptr_i32, peer_cta_rank_in_cluster.ir_value()], - "mapa.shared::cluster.u32 $0, $1, $2;", - "=r,r,r", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@dsl_user_op -def store_shared_remote_fp32x4( - a: Float32, - b: Float32, - c: Float32, - d: Float32, - smem_ptr: cute.Pointer, - mbar_ptr: cute.Pointer, - peer_cta_rank_in_cluster: Int32, - *, - loc=None, - ip=None, -) -> None: - remote_smem_ptr_i32 = set_block_rank( - smem_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip - ).ir_value() - remote_mbar_ptr_i32 = set_block_rank( - mbar_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip - ).ir_value() - llvm.inline_asm( - None, - [ - remote_smem_ptr_i32, - remote_mbar_ptr_i32, - Float32(a).ir_value(loc=loc, ip=ip), - Float32(b).ir_value(loc=loc, ip=ip), - Float32(c).ir_value(loc=loc, ip=ip), - Float32(d).ir_value(loc=loc, ip=ip), - ], - "{\n\t" - ".reg .v4 .f32 abcd;\n\t" - "mov.f32 abcd.x, $2;\n\t" - "mov.f32 abcd.y, $3;\n\t" - "mov.f32 abcd.z, $4;\n\t" - "mov.f32 abcd.w, $5;\n\t" - "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [$0], abcd, [$1];\n\t" - "}\n", - "r,r,f,f,f,f", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@dsl_user_op -def cpasync_bulk_g2s( - gmem_ptr: cute.Pointer, - smem_ptr: cute.Pointer, - tma_bar_ptr: cute.Pointer, - size: int | Int32, - *, - loc=None, - ip=None, -): - gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value() - smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value() - mbar_ptr_i32 = tma_bar_ptr.toint(loc=loc, ip=ip).ir_value() - llvm.inline_asm( - None, - [gmem_ptr_i64, smem_ptr_i32, mbar_ptr_i32, Int32(size).ir_value()], - "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [$1], [$0], $3, [$2];", - "l,r,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -@dsl_user_op -def cpasync_reduce_bulk_add_f32( - smem_ptr: cute.Pointer, - gmem_ptr: cute.Pointer, - store_bytes: int | Int32, - *, - loc=None, - ip=None, -): - smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value() - # cache_hint = cutlass.Int64(0x14F0000000000000) # EVICT_LAST - llvm.inline_asm( - None, - [gmem_ptr.llvm_ptr, smem_ptr_i32, Int32(store_bytes).ir_value()], - "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [$0], [$1], $2;", - "l,r,r", - # [gmem_ptr.llvm_ptr, smem_ptr_i32, Int32(store_bytes).ir_value(), cache_hint.ir_value()], - # "cp.reduce.async.bulk.global.shared::cta.bulk_group.L2::cache_hint.add.f32 [$0], [$1], $2, $3;", - # "l,r,r,l", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - -def cpasync_bulk_get_copy_fn( - src_tensor: cute.Tensor, - dst_tensor: cute.Tensor, - single_stage: bool = False, - **kwargs, -) -> Callable: - # src_is_smem = const_expr( - # isinstance(src_tensor.iterator, cute.Pointer) - # and src_tensor.memspace == cute.AddressSpace.smem - # ) - group_rank_src = const_expr(cute.rank(src_tensor) - (1 if not single_stage else 0)) - group_rank_dst = const_expr(cute.rank(dst_tensor) - (1 if not single_stage else 0)) - # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK) - src = cute.group_modes(src_tensor, 0, group_rank_src) - dst = cute.group_modes(dst_tensor, 0, group_rank_dst) - - def copy_bulk(src_idx, dst_idx, **new_kwargs): - size = const_expr(cute.size(src.shape[:-1]) * src.element_type.width // 8) - cpasync_bulk_g2s( - src[None, src_idx].iterator, - dst[None, dst_idx].iterator, - size=size, - **new_kwargs, - **kwargs, - ) - - def copy_bulk_single_stage(**new_kwargs): - size = const_expr(cute.size(src.shape) * src.element_type.width // 8) - cpasync_bulk_g2s(src.iterator, dst.iterator, size=size, **new_kwargs, **kwargs) - - return copy_bulk if const_expr(not single_stage) else copy_bulk_single_stage - - -def tma_get_copy_fn( - atom: cute.CopyAtom, - cta_coord: cute.Coord, - cta_layout: cute.Layout, - src_tensor: cute.Tensor, - dst_tensor: cute.Tensor, - filter_zeros: bool = False, - single_stage: bool = False, - **kwargs, -) -> Callable: - src_is_smem = const_expr( - isinstance(src_tensor.iterator, cute.Pointer) - and src_tensor.memspace == cute.AddressSpace.smem - ) - smem_tensor, gmem_tensor = (src_tensor, dst_tensor) if src_is_smem else (dst_tensor, src_tensor) - group_rank_smem = const_expr(cute.rank(smem_tensor) - (1 if not single_stage else 0)) - group_rank_gmem = const_expr(cute.rank(gmem_tensor) - (1 if not single_stage else 0)) - # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK) - s, g = cpasync.tma_partition( - atom, - cta_coord, - cta_layout, - cute.group_modes(smem_tensor, 0, group_rank_smem), - cute.group_modes(gmem_tensor, 0, group_rank_gmem), - ) - if const_expr(filter_zeros): - s = cute.filter_zeros(s) - g = cute.filter_zeros(g) - src, dst = (s, g) if src_is_smem else (g, s) - - def copy_tma(src_idx, dst_idx, **new_kwargs): - cute.copy(atom, src[None, src_idx], dst[None, dst_idx], **new_kwargs, **kwargs) - - def copy_tma_single_stage(**new_kwargs): - cute.copy(atom, src, dst, **new_kwargs, **kwargs) - - return (copy_tma if const_expr(not single_stage) else copy_tma_single_stage), s, g - - -def tma_producer_copy_fn(copy: Callable, pipeline: cutlass.pipeline.PipelineAsync): - def copy_fn(src_idx, producer_state: cutlass.pipeline.PipelineState, **new_kwargs): - copy( - src_idx=src_idx, - dst_idx=producer_state.index, - tma_bar_ptr=pipeline.producer_get_barrier(producer_state), - **new_kwargs, - ) - - return copy_fn diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/cute_dsl_utils.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/cute_dsl_utils.py deleted file mode 100644 index 25398abf8737..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/cute_dsl_utils.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -import os -import pathlib -from typing import Tuple -from functools import partial, lru_cache -from dataclasses import dataclass, fields - -import torch - -try: - from triton.tools.disasm import extract -except ImportError: - extract = None - -import cutlass -import cutlass.cute as cute -from cutlass.base_dsl.typing import JitArgument -from cutlass.cutlass_dsl import NumericMeta -from cutlass.cute.runtime import from_dlpack - -StaticTypes = (cutlass.Constexpr, NumericMeta, int, bool, str, float, type(None)) - - -load_cubin_module_data_og = cutlass.base_dsl.runtime.cuda.load_cubin_module_data -cute_compile_og = cute.compile - - -torch2cute_dtype_map = { - torch.float16: cutlass.Float16, - torch.bfloat16: cutlass.BFloat16, - torch.float32: cutlass.Float32, -} - - -@lru_cache -def get_max_active_clusters(cluster_size): - return cutlass.utils.HardwareInfo().get_max_active_clusters(cluster_size=cluster_size) - - -@lru_cache -def get_device_capacity(device: torch.device = None) -> Tuple[int, int]: - return torch.cuda.get_device_capability(device) - - -@dataclass -class ParamsBase: - def __extract_mlir_values__(self): - all_fields = [getattr(self, field.name) for field in fields(self)] - non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)] - values, self._values_pos = [], [] - for obj in non_constexpr_fields: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - all_fields = {field.name: getattr(self, field.name) for field in fields(self)} - constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, StaticTypes)} - non_constexpr_fields = { - n: f for n, f in all_fields.items() if not isinstance(f, StaticTypes) - } - for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos): - non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items]) - values = values[n_items:] - return self.__class__(**non_constexpr_fields, **constexpr_fields) - - -@dataclass -class ArgumentsBase(JitArgument): - def __c_pointers__(self): - all_fields = [getattr(self, field.name) for field in fields(self)] - non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)] - c_ptrs = [] - for obj in non_constexpr_fields: - if hasattr(obj, "__c_pointers__"): - c_ptrs.extend(obj.__c_pointers__()) - return c_ptrs - - def __get_mlir_types__(self): - all_fields = [getattr(self, field.name) for field in fields(self)] - non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)] - types, self._values_pos = [], [] - for obj in non_constexpr_fields: - if hasattr(obj, "__get_mlir_types__"): - obj_types = obj.__get_mlir_types__() - types.extend(obj_types) - self._values_pos.append(len(obj_types)) - else: - self._values_pos.append(0) - return types - - def __new_from_mlir_values__(self, values): - all_fields = {field.name: getattr(self, field.name) for field in fields(self)} - constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, StaticTypes)} - non_constexpr_fields = { - n: f for n, f in all_fields.items() if not isinstance(f, StaticTypes) - } - for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos): - non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items]) - values = values[n_items:] - return self.__class__(**non_constexpr_fields, **constexpr_fields) - - -def load_cubin_module_data_patched(cubin_data, filepath): - pathlib.Path(filepath).write_bytes(cubin_data) - return load_cubin_module_data_og(cubin_data) - - -class _CuteCompilePatched: - """Drop-in replacement for cute.compile that dumps SASS when CUTE_CUBIN_PATH is set. - - Must be a class (not a function) because cute.compile is a CompileCallable - supporting both `cute.compile(...)` and `cute.compile[opts](...)`. The latter - form is used by FlashInfer's mamba SSD kernels, so __getitem__ is preserved. - """ - - def __init__(self, original=None): - self._original = original or cute_compile_og - - def __getitem__(self, item): - return _CuteCompilePatched(self._original[item]) - - def __call__(self, *args, **kwargs): - cubin_path = os.getenv("CUTE_CUBIN_PATH", None) - if cubin_path is not None: - cutlass.base_dsl.runtime.cuda.load_cubin_module_data = partial( - load_cubin_module_data_patched, filepath=cubin_path - ) - output = self._original(*args, **kwargs) - if cubin_path is not None: - cutlass.base_dsl.runtime.cuda.load_cubin_module_data = load_cubin_module_data_og - if extract is not None: - sass = extract(cubin_path, None) - pathlib.Path(cubin_path).with_suffix(".annotated.sass").write_text(sass) - return output - - -cute_compile_patched = _CuteCompilePatched() - - -def to_cute_tensor(t, assumed_align=16, leading_dim=-1, fully_dynamic=False, enable_tvm_ffi=True): - """Convert torch tensor to cute tensor for TVM FFI. leading_dim=-1 defaults to t.ndim-1.""" - tensor = from_dlpack(t.detach(), assumed_align=assumed_align, enable_tvm_ffi=enable_tvm_ffi) - if fully_dynamic: - return tensor.mark_layout_dynamic() - if leading_dim == -1: - leading_dim = t.ndim - 1 - return tensor.mark_layout_dynamic(leading_dim=leading_dim) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/fast_math.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/fast_math.py deleted file mode 100644 index c56ea89e7988..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/fast_math.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -import cutlass -import cutlass.cute as cute -from cutlass import Int32 - - -@cute.jit -def clz(x: Int32) -> Int32: - # for i in cutlass.range_constexpr(32): - # if (1 << (31 - i)) & x: - # return Int32(i) - # return Int32(32) - # Early exit is not supported yet - res = Int32(32) - done = False - for i in cutlass.range(32): - if ((1 << (31 - i)) & x) and not done: - res = Int32(i) - done = True - return res diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd.py deleted file mode 100644 index 86f19357f76a..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd.py +++ /dev/null @@ -1,1583 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# A reimplementation of https://github.com/Dao-AILab/flash-attention/blob/main/hopper/mainloop_bwd_sm80.hpp -# from Cutlass C++ to Cute-DSL. -import math -from types import SimpleNamespace -from typing import Type, Callable, Optional -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass.cute.nvgpu import cpasync, warp -from cutlass import Float32, Int32 -import cutlass.utils as utils_basic - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.ampere_helpers as sm80_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .mask import AttentionMask -from .seqlen_info import SeqlenInfoQK -from .tile_scheduler import ( - ParamsBase, - SingleTileScheduler, - SingleTileVarlenScheduler, - TileSchedulerArguments, -) - - -class FlashAttentionBackwardSm80: - def __init__( - self, - dtype: Type[cutlass.Numeric], - head_dim: int, - head_dim_v: Optional[int] = None, - qhead_per_kvhead: int = 1, - m_block_size: int = 64, - n_block_size: int = 128, - num_stages_Q: int = 2, - num_stages_dO: int = 2, - num_threads: int = 256, - pack_gqa: bool = False, - is_causal: bool = False, - SdP_swapAB: bool = False, - dKV_swapAB: bool = False, - dQ_swapAB: bool = False, - AtomLayoutMSdP: int = 1, - AtomLayoutNdKV: int = 8, - AtomLayoutMdQ: int = 1, - V_in_regs: bool = False, - ): - """Initializes the configuration for a flash attention v2 kernel. - - All contiguous dimensions must be at least 16 bytes aligned which indicates the head dimension - should be a multiple of 8. - - :param head_dim: head dimension - :type head_dim: int - :param m_block_size: m block size - :type m_block_size: int - :param n_block_size: n block size - :type n_block_size: int - :param num_threads: number of threads - :type num_threads: int - :param is_causal: is causal - """ - self.dtype = dtype - # padding head_dim to a multiple of 16 as k_block_size - hdim_multiple_of = 32 - self.head_dim_padded = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - head_dim_v = head_dim_v if head_dim_v is not None else head_dim - self.same_hdim_kv = head_dim == head_dim_v - self.head_dim_v_padded = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of) - # Can save registers (and hence be faster) if we don't have to check hdim predication - self.check_hdim_oob = head_dim != self.head_dim_padded - self.check_hdim_v_oob = head_dim_v != self.head_dim_v_padded - self.qhead_per_kvhead = qhead_per_kvhead - self.m_block_size = m_block_size - self.n_block_size = n_block_size - self.num_threads = num_threads - self.pack_gqa = pack_gqa - self.is_causal = is_causal - self.num_stages_Q = num_stages_Q - self.num_stages_dO = num_stages_dO - self.SdP_swapAB = SdP_swapAB - self.dKV_swapAB = dKV_swapAB - self.dQ_swapAB = dQ_swapAB - self.AtomLayoutMSdP = AtomLayoutMSdP - self.AtomLayoutNdKV = AtomLayoutNdKV - self.AtomLayoutMdQ = AtomLayoutMdQ - num_mma_warps = self.num_threads // cute.arch.WARP_SIZE - self.Mma_dKV_is_RS = ( - AtomLayoutMSdP == 1 - and AtomLayoutNdKV == num_mma_warps - and SdP_swapAB - and not dKV_swapAB - ) - self.V_in_regs = V_in_regs - self.share_QV_smem = V_in_regs - - @staticmethod - def can_implement( - dtype, - head_dim, - head_dim_v, - m_block_size, - n_block_size, - num_stages_Q, - num_stages_dO, - num_threads, - is_causal, - V_in_regs=False, - ) -> bool: - """Check if the kernel can be implemented with the given parameters. - - :param dtype: data type - :type dtype: cutlass.Numeric - :param head_dim: head dimension - :type head_dim: int - :param m_block_size: m block size - :type m_block_size: int - :param n_block_size: n block size - :type n_block_size: int - :param num_threads: number of threads - :type num_threads: int - :param is_causal: is causal - :type is_causal: bool - - :return: True if the kernel can be implemented, False otherwise - :rtype: bool - """ - if dtype not in [cutlass.Float16, cutlass.BFloat16]: - return False - if head_dim % 8 != 0: - return False - if head_dim_v % 8 != 0: - return False - if n_block_size % 16 != 0: - return False - if num_threads % 32 != 0: - return False - # Check if block size setting is out of shared memory capacity - # Shared memory usage: Q tile + (K tile + V tile) where K and V use the same tile size - smem_usage_Q = m_block_size * head_dim * num_stages_Q * 2 - smem_usage_dO = m_block_size * head_dim_v * num_stages_dO * 2 - smem_usage_K = n_block_size * head_dim * 2 - smem_usage_V = n_block_size * head_dim_v * 2 - smem_usage_QV = ( - (smem_usage_Q + smem_usage_V) if not V_in_regs else max(smem_usage_Q, smem_usage_V) - ) - smem_usage = smem_usage_QV + smem_usage_dO + smem_usage_K - smem_capacity = utils_basic.get_smem_capacity_in_bytes("sm_80") - if smem_usage > smem_capacity: - return False - return True - - def _check_type( - self, - mQ_type: Type[cutlass.Numeric], - mK_type: Type[cutlass.Numeric], - mV_type: Type[cutlass.Numeric], - mdO_type: Type[cutlass.Numeric], - mLSE_type: Type[cutlass.Numeric], - mdPsum_type: Type[cutlass.Numeric], - mdQaccum_type: Type[cutlass.Numeric], - mdK_type: Type[cutlass.Numeric], - mdV_type: Type[cutlass.Numeric], - mCuSeqlensQ_type: Type[cutlass.Numeric] | None, - mCuSeqlensK_type: Type[cutlass.Numeric] | None, - mSeqUsedQ_type: Type[cutlass.Numeric] | None, - mSeqUsedK_type: Type[cutlass.Numeric] | None, - ): - if cutlass.const_expr(not (mQ_type == mK_type == mV_type == mdO_type)): - raise TypeError("All tensors must have the same data type") - if cutlass.const_expr(self.qhead_per_kvhead == 1): - if cutlass.const_expr(not (mdK_type == mdV_type == mQ_type)): - raise TypeError("mdK and mdV tensors must have the same data type as mQ") - else: - if cutlass.const_expr(not (mdK_type == mdV_type == cutlass.Float32)): - raise TypeError("mdKaccum and mdVaccum tensors must have the data type Float32") - if cutlass.const_expr(mQ_type not in [cutlass.Float16, cutlass.BFloat16]): - raise TypeError("Only Float16 or BFloat16 is supported") - if cutlass.const_expr(mLSE_type not in [cutlass.Float32]): - raise TypeError("LSE tensor must be Float32") - if cutlass.const_expr(mdPsum_type not in [cutlass.Float32]): - raise TypeError("dPsum tensor must be Float32") - if cutlass.const_expr(mdQaccum_type not in [cutlass.Float32]): - raise TypeError("dQaccum tensor must be Float32") - if cutlass.const_expr(mCuSeqlensQ_type not in [None, cutlass.Int32]): - raise TypeError("cuSeqlensQ tensor must be Int32") - if cutlass.const_expr(mCuSeqlensK_type not in [None, cutlass.Int32]): - raise TypeError("cuSeqlensK tensor must be Int32") - if cutlass.const_expr(mSeqUsedQ_type not in [None, cutlass.Int32]): - raise TypeError("SeqUsedQ tensor must be Int32") - if cutlass.const_expr(mSeqUsedK_type not in [None, cutlass.Int32]): - raise TypeError("SeqUsedK tensor must be Int32") - assert mQ_type == self.dtype - - def _setup_attributes(self): - # /////////////////////////////////////////////////////////////////////////////// - # Shared memory layout: Q/K/V - # /////////////////////////////////////////////////////////////////////////////// - sQ_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, self.head_dim_padded) - self.sQ_layout = cute.tile_to_shape( - sQ_layout_atom, - (self.m_block_size, self.head_dim_padded, self.num_stages_Q), - (0, 1, 2), - ) - sK_layout_atom = sQ_layout_atom - self.sK_layout = cute.tile_to_shape( - sK_layout_atom, - (self.n_block_size, self.head_dim_padded), - (0, 1), - ) - sV_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, self.head_dim_v_padded) - self.sV_layout = cute.tile_to_shape( - sV_layout_atom, - (self.n_block_size, self.head_dim_v_padded), - (0, 1), - ) - sdO_layout_atom = sV_layout_atom - self.sdO_layout = cute.tile_to_shape( - sdO_layout_atom, - (self.m_block_size, self.head_dim_v_padded, self.num_stages_dO), - (0, 1, 2), - ) - # TODO: do we set swizzle to be 3 here explicitly? - sPdS_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, self.n_block_size) - self.sPdS_layout = cute.tile_to_shape( - sPdS_layout_atom, - (self.m_block_size, self.n_block_size), - (0, 1), - ) - # We set stride to be multiple of 64 so that if ShuffleLSE, even if threads read from sLSE but out of bounds, - # it's still a valid smem address. - self.sLSE_layout = cute.make_layout( - (self.m_block_size, self.num_stages_Q), - stride=(1, cute.round_up(self.m_block_size, 64)), - ) - sLSEMma_layout = cute.make_layout( - (self.m_block_size, self.n_block_size, self.num_stages_Q), - stride=(1, 0, cute.round_up(self.m_block_size, 64)), - ) - sLSEMma_layout_transposed = cute.make_layout( - (self.n_block_size, self.m_block_size, self.num_stages_Q), - stride=(0, 1, cute.round_up(self.m_block_size, 64)), - ) - self.sLSEMma_layout = sLSEMma_layout if not self.SdP_swapAB else sLSEMma_layout_transposed - - # /////////////////////////////////////////////////////////////////////////////// - # GMEM Tiled copy: - # /////////////////////////////////////////////////////////////////////////////// - # Thread layouts for copies - universal_copy_bits = 128 - async_copy_elems = universal_copy_bits // self.dtype.width - # atom_async_copy: async copy atom for QKV load - atom_async_copy = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - self.dtype, - num_bits_per_copy=universal_copy_bits, - ) - # atom_universal_copy: universal copy atom for O store - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dtype, - num_bits_per_copy=universal_copy_bits, - ) - # tQK_layout: thread layout for QK load - tQK_shape_dim_1 = sQ_layout_atom.outer.shape[1] // async_copy_elems - assert self.num_threads % tQK_shape_dim_1 == 0, ( - "num_threads must be divisible by tQK_shape_dim_1" - ) - tQK_layout = cute.make_ordered_layout( - (self.num_threads // tQK_shape_dim_1, tQK_shape_dim_1), - order=(1, 0), - ) - # Do we need to check if we overshot kBlockM when we load Q? - self.is_even_m_smem_q = self.m_block_size % tQK_layout.shape[0] == 0 - # Do we need to check if we overshot kBlockN when we load K? - self.is_even_n_smem_k = self.n_block_size % tQK_layout.shape[0] == 0 - tVdO_shape_dim_1 = sV_layout_atom.outer.shape[1] // async_copy_elems - assert self.num_threads % tVdO_shape_dim_1 == 0, ( - "num_threads must be divisible by tVdO_shape_dim_1" - ) - tVdO_layout = cute.make_ordered_layout( - (self.num_threads // tVdO_shape_dim_1, tVdO_shape_dim_1), - order=(1, 0), - ) - # Do we need to check if we overshot kBlockN when we load V? - self.is_even_n_smem_v = self.n_block_size % tVdO_layout.shape[0] == 0 - self.is_even_m_smem_do = self.m_block_size % tVdO_layout.shape[0] == 0 - - # Value layouts for copies - vQKVdO_layout = cute.make_layout((1, async_copy_elems)) - - # gmem_tiled_copy_QK: tiled copy for QK load - self.gmem_tiled_copy_QK = cute.make_tiled_copy_tv( - atom_async_copy, tQK_layout, vQKVdO_layout - ) - self.gmem_tiled_copy_VdO = cute.make_tiled_copy_tv( - atom_async_copy, tVdO_layout, vQKVdO_layout - ) - self.gmem_tiled_copy_dK = cute.make_tiled_copy_tv( - atom_universal_copy, tQK_layout, vQKVdO_layout - ) - self.gmem_tiled_copy_dV = cute.make_tiled_copy_tv( - atom_universal_copy, tVdO_layout, vQKVdO_layout - ) - async_copy_elems_accum = universal_copy_bits // cutlass.Float32.width - - # I think we wouldn't require this with smarter padding - if cutlass.const_expr(not self.varlen_q): - async_copy_elems_accum = universal_copy_bits // cutlass.Float32.width - atom_async_copy_accum = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - cutlass.Float32, - num_bits_per_copy=universal_copy_bits, - ) - else: - async_copy_elems_accum = 1 - atom_async_copy_accum = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.Float32, - num_bits_per_copy=cutlass.Float32.width, - ) - self.gmem_tiled_copy_LSE = cute.make_tiled_copy_tv( - atom_async_copy_accum, - cute.make_layout(self.num_threads), - cute.make_layout(async_copy_elems_accum), - ) - self.gmem_tiled_copy_dQaccum = cute.make_tiled_copy_tv( - cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.Float32, - num_bits_per_copy=cutlass.Float32.width, - ), - cute.make_layout(self.num_threads), - cute.make_layout(1), - ) - if cutlass.const_expr(self.qhead_per_kvhead > 1): - self.gmem_tiled_copy_dK = self.gmem_tiled_copy_dQaccum - self.gmem_tiled_copy_dV = self.gmem_tiled_copy_dQaccum - - def _get_tiled_mma(self): - num_mma_warps = self.num_threads // 32 - AtomLayoutSdP = ( - (self.AtomLayoutMSdP, num_mma_warps // self.AtomLayoutMSdP, 1) - if cutlass.const_expr(not self.SdP_swapAB) - else (num_mma_warps // self.AtomLayoutMSdP, self.AtomLayoutMSdP, 1) - ) - tiled_mma_sdp = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, cutlass.Float32, (16, 8, 16)), - AtomLayoutSdP, - permutation_mnk=(AtomLayoutSdP[0] * 16, AtomLayoutSdP[1] * 16, 16), - ) - AtomLayoutdKV = ( - (self.AtomLayoutNdKV, num_mma_warps // self.AtomLayoutNdKV, 1) - if cutlass.const_expr(not self.dKV_swapAB) - else (num_mma_warps // self.AtomLayoutNdKV, self.AtomLayoutNdKV, 1) - ) - tiled_mma_dkv = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, cutlass.Float32, (16, 8, 16)), - AtomLayoutdKV, - permutation_mnk=(AtomLayoutdKV[0] * 16, AtomLayoutdKV[1] * 16, 16), - ) - AtomLayoutdQ = ( - (self.AtomLayoutMdQ, num_mma_warps // self.AtomLayoutMdQ, 1) - if cutlass.const_expr(not self.dQ_swapAB) - else (num_mma_warps // self.AtomLayoutMdQ, self.AtomLayoutMdQ, 1) - ) - tiled_mma_dq = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, cutlass.Float32, (16, 8, 16)), - AtomLayoutdQ, - permutation_mnk=(AtomLayoutdQ[0] * 16, AtomLayoutdQ[1] * 16, 16), - ) - return tiled_mma_sdp, tiled_mma_dkv, tiled_mma_dq - - def _get_shared_storage_cls(self): - sQ_struct, sK_struct, sV_struct, sdO_struct = [ - cute.struct.Align[cute.struct.MemRange[self.dtype, cute.cosize(layout)], 1024] - for layout in (self.sQ_layout, self.sK_layout, self.sV_layout, self.sdO_layout) - ] - cosize_sQV = max(cute.cosize(self.sQ_layout), cute.cosize(self.sV_layout)) - sQV_struct = cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sQV], 1024] - sLSE_struct, sdPsum_struct = [ - cute.struct.Align[cute.struct.MemRange[cutlass.Float32, cute.cosize(layout)], 128] - for layout in (self.sLSE_layout, self.sLSE_layout) - ] - sP_struct, sdS_struct = [ - cute.struct.Align[cute.struct.MemRange[self.dtype, cute.cosize(layout)], 128] - for layout in (self.sPdS_layout, self.sPdS_layout) - ] - - @cute.struct - class SharedStorageSeparateQV: - sK: sK_struct - sV: sV_struct - sQ: sQ_struct - sdO: sdO_struct - sLSE: sLSE_struct - sdPsum: sdPsum_struct - sP: sP_struct - sdS: sdS_struct - # TODO: the case where there's no sP - - @cute.struct - class SharedStorageSharedQV: - sK: sK_struct - sV: sV_struct - sQ: sQV_struct - sdO: sdO_struct - sLSE: sLSE_struct - sdPsum: sdPsum_struct - sP: sP_struct - sdS: sdS_struct - - return ( - SharedStorageSeparateQV - if cutlass.const_expr(not self.share_QV_smem) - else SharedStorageSharedQV - ) - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdQaccum: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - softmax_scale: cutlass.Float32, - stream: cuda.CUstream, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - softcap: Float32 | float | None = None, - window_size_left: Int32 | int | None = None, - window_size_right: Int32 | int | None = None, - mdQ_semaphore: Optional[cute.Tensor] = None, - ): - assert mdQ_semaphore is None, "semaphore not supported yet" - # Get the data type and check if it is fp16 or bf16 - self._check_type( - *( - t.element_type if t is not None else None - for t in ( - mQ, - mK, - mV, - mdO, - mLSE, - mdPsum, - mdQaccum, - mdK, - mdV, - mCuSeqlensQ, - mCuSeqlensK, - mSeqUsedQ, - mSeqUsedK, - ) - ) - ) - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mQ, mK, mV, mdO, mLSE, mdPsum, mdQaccum, mdK, mdV = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - if t is not None - else None - for t in (mQ, mK, mV, mdO, mLSE, mdPsum, mdQaccum, mdK, mdV) - ] - self.varlen_q = mCuSeqlensQ is not None - self._setup_attributes() - SharedStorage = self._get_shared_storage_cls() - tiled_mma_sdp, tiled_mma_dkv, tiled_mma_dq = self._get_tiled_mma() - - num_head = mQ.shape[1] if cutlass.const_expr(mCuSeqlensQ is not None) else mQ.shape[2] - - if cutlass.const_expr(mCuSeqlensK is not None): - TileScheduler = SingleTileVarlenScheduler - num_batch = mCuSeqlensK.shape[0] - 1 - else: - TileScheduler = SingleTileScheduler - num_batch = mK.shape[0] - - # Uses seqlen k, etc. since main bwd kernel's blocks are over n - tile_sched_args = TileSchedulerArguments( - num_block=cute.ceil_div(mK.shape[1], self.n_block_size), - num_head=num_head, - num_batch=num_batch, - num_splits=1, - seqlen_k=0, - headdim=mK.shape[2], - headdim_v=mV.shape[2], - total_q=mK.shape[0], - tile_shape_mn=(self.n_block_size, self.m_block_size), - qhead_per_kvhead_packgqa=self.qhead_per_kvhead - if cutlass.const_expr(self.pack_gqa) - else 1, - mCuSeqlensQ=mCuSeqlensK, - mSeqUsedQ=mSeqUsedK, - ) - - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - - softmax_scale_log2 = softmax_scale * math.log2(math.e) - self.kernel( - mQ, - mK, - mV, - mdO, - mLSE, - mdPsum, - mdQaccum, - mdK, - mdV, - mCuSeqlensQ, - mCuSeqlensK, - mSeqUsedQ, - mSeqUsedK, - softmax_scale, - softmax_scale_log2, - self.sQ_layout, - self.sK_layout, - self.sV_layout, - self.sdO_layout, - self.sPdS_layout, - self.sLSE_layout, - self.sLSEMma_layout, - self.gmem_tiled_copy_QK, - self.gmem_tiled_copy_VdO, - self.gmem_tiled_copy_dK, - self.gmem_tiled_copy_dV, - self.gmem_tiled_copy_LSE, - self.gmem_tiled_copy_dQaccum, - tiled_mma_sdp, - tiled_mma_dkv, - tiled_mma_dq, - SharedStorage, - tile_sched_params, - TileScheduler, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - smem=SharedStorage.size_in_bytes(), - stream=stream, - ) - - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdQaccum: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - mCuSeqlensQ: Optional[cute.Tensor], - mCuSeqlensK: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - mSeqUsedK: Optional[cute.Tensor], - softmax_scale: cutlass.Float32, - softmax_scale_log2: cutlass.Float32, - sQ_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sdO_layout: cute.ComposedLayout, - sPdS_layout: cute.ComposedLayout, - sLSE_layout: cute.Layout, - sLSEMma_layout: cute.Layout, - gmem_tiled_copy_QK: cute.TiledCopy, - gmem_tiled_copy_VdO: cute.TiledCopy, - gmem_tiled_copy_dK: cute.TiledCopy, - gmem_tiled_copy_dV: cute.TiledCopy, - gmem_tiled_copy_LSE: cute.TiledCopy, - gmem_tiled_copy_dQaccum: cute.TiledCopy, - tiled_mma_sdp: cute.TiledMma, - tiled_mma_dkv: cute.TiledMma, - tiled_mma_dq: cute.TiledMma, - SharedStorage: cutlass.Constexpr, - tile_sched_params: ParamsBase, - TileScheduler: cutlass.Constexpr[Callable], - ): - # Thread index, block index - tidx, _, _ = cute.arch.thread_idx() - - tile_scheduler = TileScheduler.create(tile_sched_params) - work_tile = tile_scheduler.initial_work_tile_info() - - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - - if work_tile.is_valid_tile: - seqlen = SeqlenInfoQK.create( - batch_idx, - mQ.shape[1], - mK.shape[1], - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=mCuSeqlensK, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=mSeqUsedK, - ) - - m_block_max = cute.ceil_div(seqlen.seqlen_q, self.m_block_size) - m_block_min = 0 - if cutlass.const_expr(self.is_causal): - m_block_min = max( - (n_block * self.n_block_size + seqlen.seqlen_q - seqlen.seqlen_k) - // self.m_block_size, - m_block_min, - ) - # TODO: return early if m_block_max == 0 - - # /////////////////////////////////////////////////////////////////////////////// - # Get the appropriate tiles for this thread block. - # /////////////////////////////////////////////////////////////////////////////// - blkQ_shape = (self.m_block_size, self.head_dim_padded) - blkK_shape = (self.n_block_size, self.head_dim_padded) - blkV_shape = (self.n_block_size, self.head_dim_v_padded) - blkdO_shape = (self.m_block_size, self.head_dim_v_padded) - - if cutlass.const_expr(not seqlen.has_cu_seqlens_q): - mQ_cur = mQ[batch_idx, None, head_idx, None] - mLSE_cur = mLSE[batch_idx, head_idx, None] - mdO_cur = mdO[batch_idx, None, head_idx, None] - mdPsum_cur = mdPsum[batch_idx, head_idx, None] - mdQaccum_cur = mdQaccum[batch_idx, head_idx, None] - else: - padded_offset_q = seqlen.offset_q + batch_idx * self.m_block_size - mQ_cur = cute.domain_offset((seqlen.offset_q, 0), mQ[None, head_idx, None]) - mLSE_cur = cute.domain_offset((padded_offset_q,), mLSE[head_idx, None]) - mdO_cur = cute.domain_offset((seqlen.offset_q, 0), mdO[None, head_idx, None]) - mdPsum_cur = cute.domain_offset((padded_offset_q,), mdPsum[head_idx, None]) - mdQaccum_cur = cute.domain_offset( - (padded_offset_q * self.head_dim_padded,), mdQaccum[head_idx, None] - ) - head_idx_kv = ( - head_idx // self.qhead_per_kvhead - if cutlass.const_expr(not self.pack_gqa) - else head_idx - ) - - if cutlass.const_expr(not seqlen.has_cu_seqlens_k): - mK_cur, mV_cur = [t[batch_idx, None, head_idx_kv, None] for t in (mK, mV)] - else: - mK_cur, mV_cur = [ - cute.domain_offset((seqlen.offset_k, 0), t[None, head_idx_kv, None]) - for t in (mK, mV) - ] - - # (m_block_size, head_dim, m_block) - gQ = cute.local_tile(mQ_cur, blkQ_shape, (None, 0)) - # (n_block_size, head_dim) - gK = cute.local_tile(mK_cur, blkK_shape, (n_block, 0)) - # (n_block_size, head_dim_v) - gV = cute.local_tile(mV_cur, blkV_shape, (n_block, 0)) - # (m_block_size, head_dim_v, m_block) - gdO = cute.local_tile(mdO_cur, blkdO_shape, (None, 0)) - gLSE = cute.local_tile(mLSE_cur, (self.m_block_size,), (None,)) - gdPsum = cute.local_tile(mdPsum_cur, (self.m_block_size,), (None,)) - gdQaccum = cute.local_tile( - mdQaccum_cur, (self.m_block_size * self.head_dim_padded,), (None,) - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Get shared memory buffer - # /////////////////////////////////////////////////////////////////////////////// - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage) - sQ = storage.sQ.get_tensor(sQ_layout) - sK = storage.sK.get_tensor(sK_layout) - if cutlass.const_expr(not self.share_QV_smem): - sV = storage.sV.get_tensor(sV_layout) - else: - sV = cute.make_tensor(cute.recast_ptr(sQ.iterator, dtype=self.dtype), sV_layout) - sdO = storage.sdO.get_tensor(sdO_layout) - sP = storage.sP.get_tensor(sPdS_layout) - sdS = storage.sdS.get_tensor(sPdS_layout) - sLSE = storage.sLSE.get_tensor(sLSE_layout) - sdPsum = storage.sdPsum.get_tensor(sLSE_layout) - sLSEMma = storage.sLSE.get_tensor(sLSEMma_layout) - sdPsumMma = storage.sdPsum.get_tensor(sLSEMma_layout) - - # Transpose view of tensors for tiled mma - sQt, sdOt, sKt, sPt, sdSt = [utils.transpose_view(t) for t in (sQ, sdO, sK, sP, sdS)] - - gmem_thr_copy_QK = gmem_tiled_copy_QK.get_slice(tidx) - gmem_thr_copy_VdO = gmem_tiled_copy_VdO.get_slice(tidx) - gmem_thr_copy_lse = gmem_tiled_copy_LSE.get_slice(tidx) - gmem_thr_copy_dQaccum = gmem_tiled_copy_dQaccum.get_slice(tidx) - # (CPY_Atom, CPY_M, CPY_K, m_block) - tQgQ = gmem_thr_copy_QK.partition_S(gQ) - tQsQ = gmem_thr_copy_QK.partition_D(sQ) - # (CPY_Atom, CPY_N, CPY_K) - tKgK = gmem_thr_copy_QK.partition_S(gK) - tKsK = gmem_thr_copy_QK.partition_D(sK) - # (CPY_Atom, CPY_N, CPY_K) - tVgV = gmem_thr_copy_VdO.partition_S(gV) - tVsV = gmem_thr_copy_VdO.partition_D(sV) - # (CPY_Atom, CPY_M, CPY_K, m_block) - tdOgdO = gmem_thr_copy_VdO.partition_S(gdO) - tdOsdO = gmem_thr_copy_VdO.partition_D(sdO) - tLSEgLSE = gmem_thr_copy_lse.partition_S(gLSE) - tLSEsLSE = gmem_thr_copy_lse.partition_D(sLSE) - tLSEgdPsum = gmem_thr_copy_lse.partition_S(gdPsum) - tLSEsdPsum = gmem_thr_copy_lse.partition_D(sdPsum) - tdQgdQaccum = gmem_thr_copy_dQaccum.partition_S(gdQaccum) - - # /////////////////////////////////////////////////////////////////////////////// - # Tile MMA compute thread partitions and allocate accumulators - # /////////////////////////////////////////////////////////////////////////////// - thr_mma_sdp = tiled_mma_sdp.get_slice(tidx) - thr_mma_dkv = tiled_mma_dkv.get_slice(tidx) - thr_mma_dq = tiled_mma_dq.get_slice(tidx) - acc_shape_dK = thr_mma_dkv.partition_shape_C((self.n_block_size, self.head_dim_padded)) - acc_shape_dV = thr_mma_dkv.partition_shape_C( - (self.n_block_size, self.head_dim_v_padded) - ) - acc_dK = cute.make_fragment(acc_shape_dK, cutlass.Float32) - acc_dV = cute.make_fragment(acc_shape_dV, cutlass.Float32) - acc_dK.fill(0.0) - acc_dV.fill(0.0) - - tSrQ = utils.mma_make_fragment_A(sQ[None, None, 0], thr_mma_sdp, swapAB=self.SdP_swapAB) - tSrK = utils.mma_make_fragment_B(sK, thr_mma_sdp, swapAB=self.SdP_swapAB) - tdPrdO = utils.mma_make_fragment_A( - sdO[None, None, 0], thr_mma_sdp, swapAB=self.SdP_swapAB - ) - tdPrV = utils.mma_make_fragment_B(sV, thr_mma_sdp, swapAB=self.SdP_swapAB) - tdVrP = utils.mma_make_fragment_A(sPt, thr_mma_dkv, swapAB=self.dKV_swapAB) - tdVrdO = utils.mma_make_fragment_B( - sdOt[None, None, 0], thr_mma_dkv, swapAB=self.dKV_swapAB - ) - tdKrdS = utils.mma_make_fragment_A(sdSt, thr_mma_dkv, swapAB=self.dKV_swapAB) - tdKrQ = utils.mma_make_fragment_B( - sQt[None, None, 0], thr_mma_dkv, swapAB=self.dKV_swapAB - ) - tdQrdS = utils.mma_make_fragment_A(sdS, thr_mma_dq, swapAB=self.dQ_swapAB) - tdQrK = utils.mma_make_fragment_B(sKt, thr_mma_dq, swapAB=self.dQ_swapAB) - - LSEslice = ( - (None, 0, None) if cutlass.const_expr(not self.SdP_swapAB) else (0, None, None) - ) - tSsLSEMma = utils.make_acc_tensor_mn_view(thr_mma_sdp.partition_C(sLSEMma))[LSEslice] - tSsdPsumMma = utils.make_acc_tensor_mn_view(thr_mma_sdp.partition_C(sdPsumMma))[ - LSEslice - ] - - # /////////////////////////////////////////////////////////////////////////////// - # Smem copy atom tiling - # /////////////////////////////////////////////////////////////////////////////// - smem_copy_atom = cute.make_copy_atom( - warp.LdMatrix8x8x16bOp(transpose=False, num_matrices=4), - self.dtype, - ) - smem_copy_atom_transposed = cute.make_copy_atom( - warp.LdMatrix8x8x16bOp(transpose=True, num_matrices=4), - self.dtype, - ) - smem_thr_copy_QdO = utils.make_tiled_copy_A( - smem_copy_atom, tiled_mma_sdp, swapAB=self.SdP_swapAB - ).get_slice(tidx) - smem_thr_copy_KV = utils.make_tiled_copy_B( - smem_copy_atom, tiled_mma_sdp, swapAB=self.SdP_swapAB - ).get_slice(tidx) - # TODO: should this be smem_copy_atom_transposed? - smem_thr_copy_PdSt = utils.make_tiled_copy_A( - smem_copy_atom_transposed, tiled_mma_dkv, swapAB=self.dKV_swapAB - ).get_slice(tidx) - smem_thr_copy_QdOt = utils.make_tiled_copy_B( - smem_copy_atom_transposed, tiled_mma_dkv, swapAB=self.dKV_swapAB - ).get_slice(tidx) - smem_thr_copy_dS = utils.make_tiled_copy_A( - smem_copy_atom, tiled_mma_dq, swapAB=self.dQ_swapAB - ).get_slice(tidx) - smem_thr_copy_Kt = utils.make_tiled_copy_B( - smem_copy_atom_transposed, tiled_mma_dq, swapAB=self.dQ_swapAB - ).get_slice(tidx) - # TODO: what's the number of bits? What if SdP_swapAB - r2s_thr_copy_PdS = cute.make_tiled_copy_C( - cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), self.dtype, num_bits_per_copy=2 * self.dtype.width - ), - tiled_mma_sdp, - ).get_slice(tidx) - - tSsQ = smem_thr_copy_QdO.partition_S(sQ) - tdPsdO = smem_thr_copy_QdO.partition_S(sdO) - tSsK = smem_thr_copy_KV.partition_S(sK) - tdPsV = smem_thr_copy_KV.partition_S(sV) - tdVsPt = smem_thr_copy_PdSt.partition_S(sPt) - tdKsdSt = smem_thr_copy_PdSt.partition_S(sdSt) - tdVsdOt = smem_thr_copy_QdOt.partition_S(sdOt) - tdKsQt = smem_thr_copy_QdOt.partition_S(sQt) - tdQsdS = smem_thr_copy_dS.partition_S(sdS) - tdQsKt = smem_thr_copy_Kt.partition_S(sKt) - tPsP = r2s_thr_copy_PdS.partition_D(sP) - tdSsdS = r2s_thr_copy_PdS.partition_D(sdS) - - # /////////////////////////////////////////////////////////////////////////////// - # Predicate: Mark indices that need to copy when problem_shape isn't a multiple - # of tile_shape - # /////////////////////////////////////////////////////////////////////////////// - # Construct identity layout for KV - cQ = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded)) - tQcQ = gmem_thr_copy_QK.partition_S(cQ) - t0QcQ = gmem_thr_copy_QK.get_slice(0).partition_S(cQ) - if cutlass.const_expr(self.head_dim_padded == self.head_dim_v_padded): - tdOcdO = tQcQ - t0dOcdO = t0QcQ - else: - cdO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded)) - tdOcdO = gmem_thr_copy_VdO.partition_S(cdO) - t0dOcdO = gmem_thr_copy_VdO.get_slice(0).partition_S(cdO) - cLSE = cute.make_identity_tensor((self.m_block_size,)) - tLSEcLSE = gmem_thr_copy_lse.partition_S(cLSE) - - # Allocate predicate tensors for m and n, here we only allocate the tile of k, and - # use "if" on the mn dimension. - # This is to reduce register pressure and gets 2-3% performance gain. - - d_head = mQ.shape[cute.rank(mQ) - 1] - d_head_v = mdO.shape[cute.rank(mdO) - 1] - - tQpQ = utils.predicate_k(tQcQ, limit=d_head) - if cutlass.const_expr(self.same_hdim_kv): - tdOpdO = tQpQ - else: - tdOpdO = utils.predicate_k(tdOcdO, limit=d_head_v) - - # group parameters for compute_one_m_block - mma_params = SimpleNamespace( - thr_mma_sdp=thr_mma_sdp, - thr_mma_dkv=thr_mma_dkv, - thr_mma_dq=thr_mma_dq, - tSrQ=tSrQ, - tSrK=tSrK, - tdPrdO=tdPrdO, - tdPrV=tdPrV, - tdVrP=tdVrP, - tdVrdO=tdVrdO, - tdKrdS=tdKrdS, - tdKrQ=tdKrQ, - tdQrdS=tdQrdS, - tdQrK=tdQrK, - acc_dK=acc_dK, - acc_dV=acc_dV, - ) - smem_copy_params = SimpleNamespace( - smem_thr_copy_QdO=smem_thr_copy_QdO, - smem_thr_copy_KV=smem_thr_copy_KV, - smem_thr_copy_PdSt=smem_thr_copy_PdSt, - smem_thr_copy_QdOt=smem_thr_copy_QdOt, - smem_thr_copy_dS=smem_thr_copy_dS, - smem_thr_copy_Kt=smem_thr_copy_Kt, - r2s_thr_copy_PdS=r2s_thr_copy_PdS, - tSsQ=tSsQ, - tSsK=tSsK, - tdPsdO=tdPsdO, - tdPsV=tdPsV, - tSsLSEMma=tSsLSEMma, - tSsdPsumMma=tSsdPsumMma, - tPsP=tPsP, - tdSsdS=tdSsdS, - tdVsPt=tdVsPt, - tdVsdOt=tdVsdOt, - tdKsdSt=tdKsdSt, - tdKsQt=tdKsQt, - tdQsdS=tdQsdS, - tdQsKt=tdQsKt, - ) - gmem_copy_params = SimpleNamespace( - gmem_thr_copy_dQaccum=gmem_thr_copy_dQaccum, tdQgdQaccum=tdQgdQaccum - ) - load_Q_LSE = partial( - self.load_Q_LSE, - gmem_tiled_copy_QK, - gmem_tiled_copy_LSE, - tQgQ, - tQsQ, - tQcQ, - t0QcQ, - tQpQ, - tLSEgLSE, - tLSEsLSE, - tLSEcLSE, - seqlen=seqlen.seqlen_q, - ) - load_dO_dPsum = partial( - self.load_dO_dPsum, - gmem_tiled_copy_VdO, - gmem_tiled_copy_LSE, - tdOgdO, - tdOsdO, - tdOcdO, - t0dOcdO, - tdOpdO, - tLSEgdPsum, - tLSEsdPsum, - tLSEcLSE, - seqlen=seqlen.seqlen_q, - ) - compute_one_m_block = partial( - self.compute_one_m_block, - mma_params=mma_params, - smem_copy_params=smem_copy_params, - gmem_copy_params=gmem_copy_params, - load_Q_LSE=load_Q_LSE, - load_dO_dPsum=load_dO_dPsum, - m_block_max=m_block_max, - softmax_scale_log2=softmax_scale_log2, - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Prologue - # /////////////////////////////////////////////////////////////////////////////// - # Start async loads of the last mn-tile, where we take care of the mn residue - self.load_V( - gmem_thr_copy_VdO, tVgV, tVsV, n_block, seqlen=seqlen.seqlen_k, headdim=d_head_v - ) - if cutlass.const_expr(self.V_in_regs): - cute.arch.cp_async_commit_group() - self.load_K( - gmem_thr_copy_QK, tKgK, tKsK, n_block, seqlen=seqlen.seqlen_k, headdim=d_head - ) - cute.arch.cp_async_commit_group() - - if cutlass.const_expr(self.V_in_regs): - cute.arch.cp_async_wait_group(1) - cute.arch.barrier() - tdPrV_copy_view = smem_thr_copy_KV.retile(tdPrV) - cute.copy(smem_thr_copy_KV, tdPsV, tdPrV_copy_view) - # Sync to avoid loading Q to smem_q, which overlaps with smem_v - cute.arch.barrier() - - m_block = m_block_min - assert self.num_stages_Q >= self.num_stages_dO - for stage in cutlass.range_constexpr(self.num_stages_Q): - if cutlass.const_expr(self.num_stages_Q == 1 or stage < self.num_stages_Q - 1): - if stage == 0 or m_block + stage < m_block_max: - load_Q_LSE(m_block + stage, smem_pipe_write_q=stage) - cute.arch.cp_async_commit_group() - if cutlass.const_expr(stage < self.num_stages_dO): - if stage == 0 or m_block + stage < m_block_max: - load_dO_dPsum(m_block + stage, smem_pipe_write_q=stage) - cute.arch.cp_async_commit_group() - - # /////////////////////////////////////////////////////////////////////////////// - # Mainloop - # /////////////////////////////////////////////////////////////////////////////// - # Start processing of the first n-block. - mask = AttentionMask( - self.m_block_size, self.n_block_size, seqlen.seqlen_q, seqlen.seqlen_k - ) - mask_fn = partial( - mask.apply_mask, - n_block=n_block, - thr_mma=thr_mma_sdp, - mask_seqlen=True, - mask_causal=self.is_causal, - ) - smem_pipe_read_q = cutlass.Int32(0) - smem_pipe_read_do = cutlass.Int32(0) - smem_pipe_write_q = cutlass.Int32(self.num_stages_Q - 1) - smem_pipe_write_do = cutlass.Int32(0) - for m_tile in cutlass.range(m_block_min, m_block_max, unroll=1): - compute_one_m_block( - m_tile, - smem_pipe_read_q, - smem_pipe_read_do, - smem_pipe_write_q, - smem_pipe_write_do, - mask_fn=mask_fn, - ) - smem_pipe_read_q = self.advance_pipeline(smem_pipe_read_q, self.num_stages_Q) - smem_pipe_read_do = self.advance_pipeline(smem_pipe_read_do, self.num_stages_dO) - smem_pipe_write_q = self.advance_pipeline(smem_pipe_write_q, self.num_stages_Q) - smem_pipe_write_do = self.advance_pipeline(smem_pipe_write_do, self.num_stages_dO) - - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue - # /////////////////////////////////////////////////////////////////////////////// - # If GQA, we scale dK in the postprocessing kernel instead - if cutlass.const_expr(self.qhead_per_kvhead == 1): - acc_dK.store(acc_dK.load() * softmax_scale) - # reuse sK and sV data iterator - sdK = cute.make_tensor(sK.iterator, sK_layout) - sdV = cute.make_tensor(sV.iterator, sV_layout) - self.epilogue( - acc_dK, - acc_dV, - mdK, - mdV, - sdK, - sdV, - gmem_tiled_copy_dK, - gmem_tiled_copy_dV, - tiled_mma_dkv, - tidx, - n_block, - head_idx, - batch_idx, - seqlen, - d_head, - d_head_v, - ) - - @cute.jit - def compute_one_m_block( - self, - m_block: cutlass.Int32, - smem_pipe_read_q: cutlass.Int32, - smem_pipe_read_do: cutlass.Int32, - smem_pipe_write_q: cutlass.Int32, - smem_pipe_write_do: cutlass.Int32, - mma_params: SimpleNamespace, - smem_copy_params: SimpleNamespace, - gmem_copy_params: SimpleNamespace, - load_Q_LSE: Callable, - load_dO_dPsum: Callable, - m_block_max: cutlass.Int32, - softmax_scale_log2: cutlass.Float32, - mask_fn: Optional[Callable] = None, - ): - def load_Q_next(): - m_block_next = m_block + ( - self.num_stages_Q - 1 if cutlass.const_expr(self.num_stages_Q > 1) else 1 - ) - if m_block_next < m_block_max: - load_Q_LSE(m_block_next, smem_pipe_write_q) - cute.arch.cp_async_commit_group() - - def load_dO_next(): - if m_block + self.num_stages_dO < m_block_max: - load_dO_dPsum(m_block + self.num_stages_dO, smem_pipe_write_do) - cute.arch.cp_async_commit_group() - - # MMA S - acc_shape_SdP = mma_params.thr_mma_sdp.partition_shape_C( - (self.m_block_size, self.n_block_size) - if cutlass.const_expr(not self.SdP_swapAB) - else (self.n_block_size, self.m_block_size) - ) - acc_S = cute.make_fragment(acc_shape_SdP, cutlass.Float32) - acc_S.fill(0.0) - cute.arch.cp_async_wait_group(1 if cutlass.const_expr(self.num_stages_Q > 1) else 0) - cute.arch.barrier() - sm80_utils.gemm( - mma_params.thr_mma_sdp, - acc_S, - mma_params.tSrQ, - mma_params.tSrK, - smem_copy_params.tSsQ[ - None, - None, - None, - smem_pipe_read_q if cutlass.const_expr(self.num_stages_Q > 1) else 0, - ], - smem_copy_params.tSsK, - smem_copy_params.smem_thr_copy_QdO, - smem_copy_params.smem_thr_copy_KV, - swap_AB=self.SdP_swapAB, - ) - tLSErLSE = cute.make_fragment_like(smem_copy_params.tSsLSEMma[None, 0]) - cute.autovec_copy( - smem_copy_params.tSsLSEMma[ - None, smem_pipe_read_q if cutlass.const_expr(self.num_stages_Q > 1) else 0 - ], - tLSErLSE, - ) - if cutlass.const_expr(mask_fn is not None): - mask_fn(acc_S, m_block=m_block) - acc_S_mn = utils.make_acc_tensor_mn_view(acc_S) - bidx = 0 - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(acc_S_mn) - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == 1: cute.print_tensor(tLSErLSE) - assert cute.size(acc_S_mn, mode=[0]) == cute.size(tLSErLSE) - for r in cutlass.range(cute.size(acc_S_mn, mode=[0]), unroll_full=True): - acc_S_mn[r, None].store( - utils.exp2f(acc_S_mn[r, None].load() * softmax_scale_log2 - tLSErLSE[r]) - ) - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(acc_S_mn) - - # MMA dP - acc_dP = cute.make_fragment(acc_shape_SdP, cutlass.Float32) - acc_dP.fill(0.0) - cute.arch.cp_async_wait_group(1 if cutlass.const_expr(self.num_stages_dO > 1) else 0) - cute.arch.barrier() - sm80_utils.gemm( - mma_params.thr_mma_sdp, - acc_dP, - mma_params.tdPrdO, - mma_params.tdPrV, - smem_copy_params.tdPsdO[ - None, - None, - None, - smem_pipe_read_do if cutlass.const_expr(self.num_stages_dO > 1) else 0, - ], - smem_copy_params.tdPsV, - smem_copy_params.smem_thr_copy_QdO, - smem_copy_params.smem_thr_copy_KV, - hook_fn=load_Q_next if cutlass.const_expr(self.num_stages_Q > 1) else None, - swap_AB=self.SdP_swapAB, - ) - tLSErdPsum = cute.make_fragment_like(smem_copy_params.tSsdPsumMma[None, 0]) - cute.autovec_copy( - smem_copy_params.tSsdPsumMma[ - None, smem_pipe_read_do if cutlass.const_expr(self.num_stages_dO > 1) else 0 - ], - tLSErdPsum, - ) - acc_dP_mn = utils.make_acc_tensor_mn_view(acc_dP) - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(acc_dP_mn) - assert cute.size(acc_dP_mn, mode=[0]) == cute.size(tLSErdPsum) - for r in cutlass.range(cute.size(acc_dP_mn, mode=[0]), unroll_full=True): - acc_dP_mn[r, None].store( - acc_S_mn[r, None].load() * (acc_dP_mn[r, None].load() - tLSErdPsum[r]) - ) - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(acc_dP_mn) - rP = cute.make_fragment_like(acc_S, self.dtype) - rP.store(acc_S.load().to(self.dtype)) - if cutlass.const_expr(not self.Mma_dKV_is_RS): - tPrP = smem_copy_params.r2s_thr_copy_PdS.retile(rP) # ((Atom,AtomNum), MMA_N, MMA_N) - cute.copy(smem_copy_params.r2s_thr_copy_PdS, tPrP, smem_copy_params.tPsP) - rdS = cute.make_fragment_like(acc_dP, self.dtype) - rdS.store(acc_dP.load().to(self.dtype)) - if cutlass.const_expr(not self.Mma_dKV_is_RS): - cute.arch.barrier() # Make sure P is written - # For hdim 64, It's faster to write to smem_dS first before the dV gemm - if cutlass.const_expr(not self.Mma_dKV_is_RS): - tdSrdS = smem_copy_params.r2s_thr_copy_PdS.retile(rdS) - cute.copy(smem_copy_params.r2s_thr_copy_PdS, tdSrdS, smem_copy_params.tdSsdS) - if cutlass.const_expr(self.Mma_dKV_is_RS): - tdVrP = cute.make_tensor(rP.iterator, utils.convert_layout_acc_frgA(rP.layout)) - else: - tdVrP = mma_params.tdVrP - - # MMA dK - sm80_utils.gemm( - mma_params.thr_mma_dkv, - mma_params.acc_dV, - tdVrP, - mma_params.tdVrdO, - smem_copy_params.tdVsPt, - smem_copy_params.tdVsdOt[ - None, - None, - None, - smem_pipe_read_do if cutlass.const_expr(self.num_stages_dO > 1) else 0, - ], - smem_copy_params.smem_thr_copy_PdSt, - smem_copy_params.smem_thr_copy_QdOt, - A_in_regs=self.Mma_dKV_is_RS, - swap_AB=self.dKV_swapAB, - ) - # if cute.arch.thread_idx()[0] == 0 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(mma_params.acc_dV) - cute.arch.barrier() # Make sure dS is written - - # MMA dQ - def dQ_mma(hook_fn): - acc_shape_dQ = mma_params.thr_mma_dq.partition_shape_C( - (self.m_block_size, self.head_dim_padded) - if cutlass.const_expr(not self.dQ_swapAB) - else (self.head_dim_padded, self.m_block_size) - ) - acc_dQ = cute.make_fragment(acc_shape_dQ, cutlass.Float32) - acc_dQ.fill(0.0) - sm80_utils.gemm( - mma_params.thr_mma_dq, - acc_dQ, - mma_params.tdQrdS, - mma_params.tdQrK, - smem_copy_params.tdQsdS, - smem_copy_params.tdQsKt, - smem_copy_params.smem_thr_copy_dS, - smem_copy_params.smem_thr_copy_Kt, - swap_AB=self.dQ_swapAB, - hook_fn=hook_fn, - ) - # ((1, 1), num_elements) - acc_dQ_atomic = gmem_copy_params.gmem_thr_copy_dQaccum.retile(acc_dQ) - tdQgdQaccum_atomic = gmem_copy_params.tdQgdQaccum[None, None, m_block] - assert cute.size(acc_dQ_atomic) == cute.size(tdQgdQaccum_atomic) - for i in cutlass.range(cute.size(acc_dQ_atomic), unroll_full=True): - utils.atomic_add_fp32(acc_dQ_atomic[i], utils.elem_pointer(tdQgdQaccum_atomic, i)) - # utils.atomic_add_fp32(acc_dQ[i], tdQgdQaccum_atomic.iterator + i * tdQgdQaccum_atomic.stride[1]) - # if cute.arch.thread_idx()[0] == 64 and cute.arch.block_idx()[0] == bidx: cute.print_tensor(acc_dQ) - - # If num_stages_Q == 1, we want to do Mma_dK first so we can start loading Q for the next iteration - if cutlass.const_expr(self.num_stages_Q > 1): - dQ_mma(load_dO_next) - - # MMA dK - if cutlass.const_expr(self.Mma_dKV_is_RS): - tdKrdS = cute.make_tensor(rdS.iterator, utils.convert_layout_acc_frgA(rdS.layout)) - else: - tdKrdS = mma_params.tdKrdS - sm80_utils.gemm( - mma_params.thr_mma_dkv, - mma_params.acc_dK, - tdKrdS, - mma_params.tdKrQ, - smem_copy_params.tdKsdSt, - smem_copy_params.tdKsQt[ - None, - None, - None, - smem_pipe_read_q if cutlass.const_expr(self.num_stages_Q > 1) else 0, - ], - smem_copy_params.smem_thr_copy_PdSt, - smem_copy_params.smem_thr_copy_QdOt, - A_in_regs=self.Mma_dKV_is_RS, - swap_AB=self.dKV_swapAB, - hook_fn=load_dO_next if cutlass.const_expr(self.num_stages_Q == 1) else None, - ) - # if cute.arch.thread_idx()[0] == 0: cute.print_tensor(mma_params.acc_dK) - if cutlass.const_expr(self.num_stages_Q == 1): - cute.arch.barrier() - dQ_mma(load_Q_next) - - @cute.jit - def epilogue( - self, - acc_dK: cute.Tensor, - acc_dV: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - sdK: cute.Tensor, - sdV: cute.Tensor, - gmem_tiled_copy_dK: cute.TiledCopy, - gmem_tiled_copy_dV: cute.TiledCopy, - tiled_mma: cute.TiledMma, - tidx: cutlass.Int32, - n_block: cutlass.Int32, - num_head: cutlass.Int32, - batch_size: cutlass.Int32, - seqlen: SeqlenInfoQK, - d_head: cutlass.Int32, - d_head_v: cutlass.Int32, - ): - rdV = cute.make_fragment_like(acc_dV, self.dtype) - rdV.store(acc_dV.load().to(self.dtype)) - rdK = cute.make_fragment_like(acc_dK, self.dtype) - rdK.store(acc_dK.load().to(self.dtype)) - gmem_thr_copy_dK = gmem_tiled_copy_dK.get_slice(tidx) - gmem_thr_copy_dV = gmem_tiled_copy_dV.get_slice(tidx) - - batch_idx = batch_size - head_idx_kv = ( - num_head // self.qhead_per_kvhead if cutlass.const_expr(not self.pack_gqa) else num_head - ) - - if cutlass.const_expr(self.qhead_per_kvhead == 1): - # Make sure all threads have finished reading K and V, otherwise we get racy dQ - # because smem_q could be changed. - cute.arch.barrier() - # smem copy atom for dKV - smem_copy_atom_dKV = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), self.dtype, num_bits_per_copy=2 * self.dtype.width - ) - smem_thr_copy_dKV = cute.make_tiled_copy_C(smem_copy_atom_dKV, tiled_mma).get_slice( - tidx - ) - taccdVrdV = smem_thr_copy_dKV.retile(rdV) - taccdKrdK = smem_thr_copy_dKV.retile(rdK) - taccdVsdV = smem_thr_copy_dKV.partition_D(sdV) - taccdKsdK = smem_thr_copy_dKV.partition_D(sdK) - # copy acc O from rmem to smem with the smem copy atom - cute.copy(smem_copy_atom_dKV, taccdVrdV, taccdVsdV) - cute.copy(smem_copy_atom_dKV, taccdKrdK, taccdKsdK) - - if cutlass.const_expr(not seqlen.has_cu_seqlens_k): - mdK_cur, mdV_cur = [t[batch_idx, None, head_idx_kv, None] for t in (mdK, mdV)] - else: - mdK_cur, mdV_cur = [ - cute.domain_offset((seqlen.offset_k, 0), t[None, head_idx_kv, None]) - for t in (mdK, mdV) - ] - - blkdK_shape = (self.n_block_size, self.head_dim_padded) - blkdV_shape = (self.n_block_size, self.head_dim_v_padded) - gdK = cute.local_tile(mdK_cur, blkdK_shape, (n_block, 0)) - gdV = cute.local_tile(mdV_cur, blkdV_shape, (n_block, 0)) - tdKsdK = gmem_thr_copy_dK.partition_S(sdK) - tdKgdK = gmem_thr_copy_dK.partition_D(gdK) - tdVsdV = gmem_thr_copy_dV.partition_S(sdV) - tdVgdV = gmem_thr_copy_dV.partition_D(gdV) - tdKrdK = cute.make_fragment_like(tdKgdK, self.dtype) - tdVrdV = cute.make_fragment_like(tdVgdV, self.dtype) - # sync before all smem stores are done. - cute.arch.barrier() - # load acc dK and dV from smem to rmem for wider vectorization - # Need to check OOB when reading from smem if kBlockN isn't evenly tiled - # TODO - cute.autovec_copy(tdKsdK, tdKrdK) - cute.autovec_copy(tdVsdV, tdVrdV) - - cdK = cute.make_identity_tensor((self.n_block_size, self.head_dim_padded)) - tdKcdK = gmem_thr_copy_dK.partition_S(cdK) - t0dKcdK = gmem_tiled_copy_dK.get_slice(0).partition_S(cdK) - if cutlass.const_expr(self.head_dim_padded == self.head_dim_v_padded): - tdVcdV = tdKcdK - t0dVcdV = t0dKcdK - else: - cdV = cute.make_identity_tensor((self.n_block_size, self.head_dim_v_padded)) - tdVcdV = gmem_thr_copy_dV.partition_S(cdV) - t0dVcdV = gmem_tiled_copy_dV.get_slice(0).partition_S(cdV) - tdKpdK = utils.predicate_k(tdKcdK, limit=d_head) - if cutlass.const_expr(self.same_hdim_kv): - tdVpdV = tdKpdK - else: - tdVpdV = utils.predicate_k(tdVcdV, limit=d_head_v) - # copy acc dK and acc_dV from rmem to gmem - for rest_m in cutlass.range_constexpr(cute.size(tdKrdK.shape[1])): - if ( - t0dKcdK[0, rest_m, 0][0] - < seqlen.seqlen_k - n_block * self.n_block_size - tdKcdK[0][0] - ): - cute.copy( - gmem_tiled_copy_dK, - tdKrdK[None, rest_m, None], - tdKgdK[None, rest_m, None], - pred=tdKpdK[None, rest_m, None] - if cutlass.const_expr(self.check_hdim_oob) - else None, - ) - for rest_m in cutlass.range_constexpr(cute.size(tdVrdV.shape[1])): - if ( - t0dVcdV[0, rest_m, 0][0] - < seqlen.seqlen_k - n_block * self.n_block_size - tdVcdV[0][0] - ): - cute.copy( - gmem_tiled_copy_dV, - tdVrdV[None, rest_m, None], - tdVgdV[None, rest_m, None], - pred=tdVpdV[None, rest_m, None] - if cutlass.const_expr(self.check_hdim_v_oob) - else None, - ) - - else: # qhead_per_kvhead > 1, do atomic add - # For Sm90, we need to sync to avoid racy writes to smem_q - # For Sm80, we don't need to sync since we're not touching smem - head_idx_kv = ( - num_head // self.qhead_per_kvhead - if cutlass.const_expr(not self.pack_gqa) - else num_head - ) - - if cutlass.const_expr(not seqlen.has_cu_seqlens_k): - mdK_cur, mdV_cur = [t[batch_idx, head_idx_kv, None] for t in (mdK, mdV)] - else: - padded_offset_k = seqlen.offset_k + batch_idx * self.n_block_size - mdK_cur = cute.domain_offset( - (padded_offset_k * self.head_dim_padded,), mdK[head_idx_kv, None] - ) - mdV_cur = cute.domain_offset( - (padded_offset_k * self.head_dim_v_padded,), mdV[head_idx_kv, None] - ) - - gdV = cute.local_tile( - mdV_cur, (self.n_block_size * self.head_dim_v_padded,), (n_block,) - ) - gdK = cute.local_tile(mdK_cur, (self.n_block_size * self.head_dim_padded,), (n_block,)) - tdVgdVaccum = gmem_thr_copy_dV.partition_S(gdV) - tdKgdKaccum = gmem_thr_copy_dK.partition_S(gdK) - acc_dV_atomic = gmem_thr_copy_dV.retile(acc_dV) - acc_dK_atomic = gmem_thr_copy_dK.retile(acc_dK) - assert cute.size(acc_dV_atomic) == cute.size(tdVgdVaccum) - assert cute.size(acc_dK_atomic) == cute.size(tdKgdKaccum) - for i in cutlass.range(cute.size(acc_dV_atomic), unroll_full=True): - utils.atomic_add_fp32(acc_dV_atomic[i], utils.elem_pointer(tdVgdVaccum, i)) - for i in cutlass.range(cute.size(acc_dK_atomic), unroll_full=True): - utils.atomic_add_fp32(acc_dK_atomic[i], utils.elem_pointer(tdKgdKaccum, i)) - - @cute.jit - def advance_pipeline(self, pipeline_index, num_stages: cutlass.Constexpr): - return pipeline_index + 1 if pipeline_index < num_stages - 1 else 0 - - @cute.jit - def load_K( - self, - gmem_thr_copy: cute.TiledCopy, - tKgK: cute.Tensor, - tKsK: cute.Tensor, - block: cutlass.Int32, - seqlen: cutlass.Int32, - headdim: cutlass.Int32, - ): - cK = cute.make_identity_tensor((self.n_block_size, self.head_dim_padded)) - tKcK = gmem_thr_copy.partition_S(cK) - t0KcK = gmem_thr_copy.get_slice(0).partition_S(cK) - tKpK = utils.predicate_k(tKcK, limit=headdim) - for n in cutlass.range_constexpr(cute.size(tKsK.shape[1])): - # If kBlockN doesn't evenly divide the tiled copy, only the last `n` needs to be checked - if ( - self.is_even_n_smem_k - or n < cute.size(tKsK.shape[1]) - 1 - or tKcK[0, n, 0][0] < self.n_block_size - ): - # Instead of using tKcK, we using t0KcK and subtract the offset from the limit - # (seqlen - block * kBlockN). This is because the entries of t0KcK are known at compile time. - predicate_n = t0KcK[0, n, 0][0] < seqlen - block * self.n_block_size - tKcK[0][0] - predicate = cute.make_fragment_like(tKpK[None, 0, None]) - for k in cutlass.range_constexpr(cute.size(predicate.shape[1])): - for i in cutlass.range_constexpr(cute.size(predicate.shape[0])): - predicate[i, k] = ( - tKpK[i, n, k] if cutlass.const_expr(self.check_hdim_oob) else True - ) and predicate_n - cute.copy( - gmem_thr_copy, - tKgK[None, n, None], - tKsK[None, n, None], - pred=predicate, - ) - # We need to clear the sK smem tiles since we'll use sKt for mma_dq - - @cute.jit - def load_V( - self, - gmem_thr_copy: cute.TiledCopy, - tVgV: cute.Tensor, - tVsV: cute.Tensor, - block: cutlass.Int32, - seqlen: cutlass.Int32, - headdim: cutlass.Int32, - ): - cV = cute.make_identity_tensor((self.n_block_size, self.head_dim_v_padded)) - tVcV = gmem_thr_copy.partition_S(cV) - t0VcV = gmem_thr_copy.get_slice(0).partition_S(cV) - tVpV = utils.predicate_k(tVcV, limit=headdim) - for n in cutlass.range_constexpr(cute.size(tVsV.shape[1])): - # If kBlockN doesn't evenly divide the tiled copy, only the last `n` needs to be checked - if ( - self.is_even_n_smem_v - or n < cute.size(tVsV.shape[1]) - 1 - or tVcV[0, n, 0][0] < self.n_block_size - ): - # Instead of using tVcV, we using t0VcV and subtract the offset from the limit - # (seqlen - block * kBlockN). This is because the entries of t0VcV are known at compile time. - predicate_n = t0VcV[0, n, 0][0] < seqlen - block * self.n_block_size - tVcV[0][0] - predicate = cute.make_fragment_like(tVpV[None, 0, None]) - for k in cutlass.range_constexpr(cute.size(predicate.shape[1])): - for i in cutlass.range_constexpr(cute.size(predicate.shape[0])): - predicate[i, k] = ( - tVpV[i, n, k] if cutlass.const_expr(self.check_hdim_oob) else True - ) and predicate_n - cute.copy( - gmem_thr_copy, - tVgV[None, n, None], - tVsV[None, n, None], - pred=predicate, - ) - - @cute.jit - def load_Q_LSE( - self, - gmem_tiled_copy_Q: cute.TiledCopy, - gmem_tiled_copy_LSE: cute.TiledCopy, - tQgQ: cute.Tensor, - tQsQ: cute.Tensor, - tQcQ: cute.Tensor, - t0QcQ: cute.Tensor, - tQpQ: cute.Tensor, - tLSEgLSE: cute.Tensor, - tLSEsLSE: cute.Tensor, - tLSEcLSE: cute.Tensor, - block: cutlass.Int32, - smem_pipe_write_q: cutlass.Int32, - seqlen: cutlass.Int32, - ): - for m in cutlass.range_constexpr(cute.size(tQsQ.shape[1])): - # If kBlockM doesn't evenly divide the tiled copy, only the last `m` needs to be checked - if ( - self.is_even_m_smem_q - or m < cute.size(tQsQ.shape[1]) - 1 - or tQcQ[0, m, 0][0] < self.m_block_size - ): - # Instead of using tQcQ, we using t0QcQ and subtract the offset from the limit - # (seqlen - block * kBlockM). This is because the entries of t0QcQ are known at compile time. - predicate_m = t0QcQ[0, m, 0][0] < seqlen - block * self.m_block_size - tQcQ[0][0] - predicate = cute.make_fragment_like(tQpQ[None, 0, None]) - for k in cutlass.range_constexpr(cute.size(predicate.shape[1])): - for i in cutlass.range_constexpr(cute.size(predicate.shape[0])): - predicate[i, k] = ( - tQpQ[i, m, k] if cutlass.const_expr(self.check_hdim_oob) else True - ) and predicate_m - cute.copy( - gmem_tiled_copy_Q, - tQgQ[None, m, None, block], - tQsQ[ - None, - m, - None, - smem_pipe_write_q if cutlass.const_expr(self.num_stages_Q) > 1 else 0, - ], - pred=predicate, - ) - # We need to clear the sQ smem tiles since we'll use sQt for mma_dK - # We made sure LSE length is padded so we read `kBlockM` elements so that all - # elements in sLSE are filled. Without this we might have uninitialized sLSE values. - for m in cutlass.range_constexpr(cute.size(tLSEsLSE.shape[1])): - if tLSEcLSE[0, m][0] < self.m_block_size: - cute.copy( - gmem_tiled_copy_LSE, - tLSEgLSE[None, m, block], - tLSEsLSE[ - None, - m, - smem_pipe_write_q if cutlass.const_expr(self.num_stages_Q > 1) else 0, - ], - ) - - @cute.jit - def load_dO_dPsum( - self, - gmem_tiled_copy_dO: cute.TiledCopy, - gmem_tiled_copy_dPsum: cute.TiledCopy, - tdOgdO: cute.Tensor, - tdOsdO: cute.Tensor, - tdOcdO: cute.Tensor, - t0dOcdO: cute.Tensor, - tdOpdO: cute.Tensor, - tdPsumgdPsum: cute.Tensor, - tdPsumsdPsum: cute.Tensor, - tdPsumcdPsum: cute.Tensor, - block: cutlass.Int32, - smem_pipe_write_q: cutlass.Int32, - seqlen: cutlass.Int32, - ): - for m in cutlass.range_constexpr(cute.size(tdOsdO.shape[1])): - # If kBlockM doesn't evenly divide the tiled copy, only the last `m` needs to be checked - if ( - self.is_even_m_smem_do - or m < cute.size(tdOsdO.shape[1]) - 1 - or tdOcdO[0, m, 0][0] < self.m_block_size - ): - # Instead of using tdOcdO, we using t0dOcdO and subtract the offset from the limit - # (seqlen - block * kBlockM). This is because the entries of t0dOcdO are known at compile time. - predicate_m = ( - t0dOcdO[0, m, 0][0] < seqlen - block * self.m_block_size - tdOcdO[0][0] - ) - predicate = cute.make_fragment_like(tdOpdO[None, 0, None]) - for k in cutlass.range_constexpr(cute.size(predicate.shape[1])): - for i in cutlass.range_constexpr(cute.size(predicate.shape[0])): - predicate[i, k] = ( - tdOpdO[i, m, k] if cutlass.const_expr(self.check_hdim_oob) else True - ) and predicate_m - cute.copy( - gmem_tiled_copy_dO, - tdOgdO[None, m, None, block], - tdOsdO[ - None, - m, - None, - smem_pipe_write_q if cutlass.const_expr(self.num_stages_dO > 1) else 0, - ], - pred=predicate, - ) - # We need to clear the sQ smem tiles since we'll use sQt for mma_dK - # We made sure LSE length is padded so we read `kBlockM` elements so that all - # elements in sLSE are filled. Without this we might have uninitialized sLSE values. - for m in cutlass.range_constexpr(cute.size(tdPsumgdPsum.shape[1])): - if tdPsumcdPsum[0, m][0] < self.m_block_size: - cute.copy( - gmem_tiled_copy_dPsum, - tdPsumgdPsum[None, m, block], - tdPsumsdPsum[ - None, - m, - smem_pipe_write_q if cutlass.const_expr(self.num_stages_dO > 1) else 0, - ], - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_postprocess.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_postprocess.py deleted file mode 100644 index 9293a3ef24a5..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_postprocess.py +++ /dev/null @@ -1,463 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# A reimplementation of https://github.com/Dao-AILab/flash-attention/blob/main/hopper/flash_bwd_postprocess_kernel.h -# from Cutlass C++ to Cute-DSL. -import math -from typing import Callable, Optional, Type, Literal - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -import cutlass.utils.hopper_helpers as sm90_utils_basic -import cutlass.utils.blackwell_helpers as sm100_utils_basic -from cutlass.cute.nvgpu import cpasync, warp, warpgroup -from cutlass import Float32, const_expr -from cutlass.utils import LayoutEnum - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.ampere_helpers as sm80_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.hopper_helpers as sm90_utils -from .seqlen_info import SeqlenInfoQK -import cutlass.cute.nvgpu.tcgen05 as tcgen05 -from .tile_scheduler import ( - ParamsBase, - SingleTileScheduler, - SingleTileVarlenScheduler, - TileSchedulerArguments, -) - - -class FlashAttentionBackwardPostprocess: - def __init__( - self, - dtype: Type[cutlass.Numeric], - head_dim: int, - arch: Literal[80, 90, 100], - tile_m: int = 128, - num_threads: int = 256, - AtomLayoutMdQ: int = 1, - dQ_swapAB: bool = False, - ): - """ - :param head_dim: head dimension - :type head_dim: int - :param tile_m: m block size - :type tile_m: int - """ - self.dtype = dtype - self.tile_m = tile_m - assert arch in [80, 90, 100], ( - "Only Ampere (80), Hopper (90), and Blackwell (100) are supported" - ) - self.arch = arch - # padding head_dim to a multiple of 32 as k_block_size - hdim_multiple_of = 32 - self.tile_hdim = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - self.check_hdim_oob = head_dim != self.tile_hdim - self.num_threads = num_threads - self.AtomLayoutMdQ = AtomLayoutMdQ - self.dQ_swapAB = dQ_swapAB - - @staticmethod - def can_implement(dtype, head_dim, tile_m, num_threads) -> bool: - """Check if the kernel can be implemented with the given parameters. - - :param dtype: data type - :type dtype: cutlass.Numeric - :param head_dim: head dimension - :type head_dim: int - :param tile_m: m block size - :type tile_m: int - - :return: True if the kernel can be implemented, False otherwise - :rtype: bool - """ - if dtype not in [cutlass.Float16, cutlass.BFloat16]: - return False - if head_dim % 8 != 0: - return False - if num_threads % 32 != 0: - return False - return True - - def _get_tiled_mma(self): - if const_expr(self.arch == 80): - num_mma_warps = self.num_threads // 32 - atom_layout_dQ = ( - (self.AtomLayoutMdQ, num_mma_warps // self.AtomLayoutMdQ, 1) - if const_expr(not self.dQ_swapAB) - else (num_mma_warps // self.AtomLayoutMdQ, self.AtomLayoutMdQ, 1) - ) - tiled_mma = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, Float32, (16, 8, 16)), - atom_layout_dQ, - permutation_mnk=(atom_layout_dQ[0] * 16, atom_layout_dQ[1] * 16, 16), - ) - elif const_expr(self.arch == 90): - num_mma_warp_groups = self.num_threads // 128 - atom_layout_dQ = (self.AtomLayoutMdQ, num_mma_warp_groups // self.AtomLayoutMdQ) - tiler_mn_dQ = (self.tile_m // atom_layout_dQ[0], self.tile_hdim // atom_layout_dQ[1]) - tiled_mma = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K, # These don't matter, we only care about the accum - warpgroup.OperandMajorMode.K, - Float32, - atom_layout_mnk=(atom_layout_dQ if not self.dQ_swapAB else atom_layout_dQ[::-1]) - + (1,), - tiler_mn=tiler_mn_dQ if not self.dQ_swapAB else tiler_mn_dQ[::-1], - ) - else: - cta_group = tcgen05.CtaGroup.ONE - tiled_mma = sm100_utils_basic.make_trivial_tiled_mma( - self.dtype, - tcgen05.OperandMajorMode.MN, # dS_major_mode - tcgen05.OperandMajorMode.MN, # Kt_major_mode - Float32, - cta_group, - (self.tile_m, self.tile_hdim), - ) - if const_expr(self.arch in [80, 90]): - assert self.num_threads == tiled_mma.size - return tiled_mma - - def _setup_attributes(self): - # /////////////////////////////////////////////////////////////////////////////// - # GMEM Tiled copy: - # /////////////////////////////////////////////////////////////////////////////// - # Thread layouts for copies - universal_copy_bits = 128 - async_copy_elems_accum = universal_copy_bits // Float32.width - atom_async_copy_accum = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - Float32, - num_bits_per_copy=universal_copy_bits, - ) - # We don't do bound checking for the gmem -> smem load so we just assert here. - assert (self.tile_m * self.tile_hdim // async_copy_elems_accum) % self.num_threads == 0 - self.g2s_tiled_copy_dQaccum = cute.make_tiled_copy_tv( - atom_async_copy_accum, - cute.make_layout(self.num_threads), - cute.make_layout(async_copy_elems_accum), - ) - num_s2r_copy_elems = 1 if const_expr(self.arch == 80) else 4 - if const_expr(self.arch == 80): - self.s2r_tiled_copy_dQaccum = copy_utils.tiled_copy_1d( - Float32, self.num_threads, num_s2r_copy_elems - ) - self.sdQaccum_layout = cute.make_layout(self.tile_m * self.tile_hdim) - elif const_expr(self.arch == 90): - num_threads_per_warp_group = 128 - num_mma_warp_groups = self.num_threads // 128 - self.s2r_tiled_copy_dQaccum = cute.make_tiled_copy_tv( - cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128), - cute.make_layout((num_threads_per_warp_group, num_mma_warp_groups)), # thr_layout - cute.make_layout(128 // Float32.width), # val_layout - ) - self.sdQaccum_layout = cute.make_layout( - (self.tile_m * self.tile_hdim // num_mma_warp_groups, num_mma_warp_groups) - ) - else: - self.dQ_reduce_ncol = 32 - dQaccum_reduce_stage = self.tile_hdim // self.dQ_reduce_ncol - assert self.num_threads == 128 # TODO: currently hard-coded - self.s2r_tiled_copy_dQaccum = copy_utils.tiled_copy_1d( - Float32, self.num_threads, num_s2r_copy_elems - ) - self.sdQaccum_layout = cute.make_layout( - (self.tile_m * self.tile_hdim // dQaccum_reduce_stage, dQaccum_reduce_stage) - ) - - self.gmem_tiled_copy_dQ = copy_utils.tiled_copy_2d( - self.dtype, self.tile_hdim, self.num_threads - ) - # /////////////////////////////////////////////////////////////////////////////// - # Shared memory layout: dQ - # /////////////////////////////////////////////////////////////////////////////// - # We can't just use kHeadDim here. E.g. if MMA shape is 64 x 96 but split across 2 WGs, - # then setting kBlockKSmem to 32 will cause "Static shape_div failure". - # We want to treat it as 64 x 48, so kBlockKSmem should be 16. - mma_shape_n = self.tiled_mma.get_tile_size(1) - if const_expr(self.arch == 80): - sdQ_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, mma_shape_n) - self.sdQ_layout = cute.tile_to_shape( - sdQ_layout_atom, (self.tile_m, self.tile_hdim), (0, 1) - ) - elif const_expr(self.arch == 90): - self.sdQ_layout = sm90_utils.make_smem_layout( - self.dtype, LayoutEnum.ROW_MAJOR, (self.tile_m, self.tile_hdim) - ) - else: - # TODO: this is hard-coded for hdim 128 - self.sdQ_layout = sm100_utils_basic.make_smem_layout_epi( - self.dtype, LayoutEnum.ROW_MAJOR, (self.tile_m, self.tile_hdim), 1 - ) - - @cute.jit - def __call__( - self, - mdQaccum: cute.Tensor, - mdQ: cute.Tensor, - scale: cutlass.Float32, - mCuSeqlensQ: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - stream: cuda.CUstream, - ): - # Get the data type and check if it is fp16 or bf16 - if const_expr(mdQ.element_type not in [cutlass.Float16, cutlass.BFloat16]): - raise TypeError("Only Float16 or BFloat16 is supported") - if const_expr(mdQaccum is not None): - if const_expr(mdQaccum.element_type not in [cutlass.Float32]): - raise TypeError("dQaccum tensor must be Float32") - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mdQaccum, mdQ = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - for t in (mdQaccum, mdQ) - ] - - self.tiled_mma = self._get_tiled_mma() - self._setup_attributes() - - smem_size = max( - cute.size_in_bytes(cutlass.Float32, self.sdQaccum_layout), - cute.size_in_bytes(self.dtype, self.sdQ_layout), - ) - - if const_expr(mCuSeqlensQ is not None): - TileScheduler = SingleTileVarlenScheduler - num_head = mdQ.shape[1] - num_batch = mCuSeqlensQ.shape[0] - 1 - num_block = cute.ceil_div(mdQ.shape[0], self.tile_m) - else: - TileScheduler = SingleTileScheduler - num_head = mdQ.shape[2] - num_batch = mdQ.shape[0] - num_block = cute.ceil_div(mdQ.shape[1], self.tile_m) - - tile_sched_args = TileSchedulerArguments( - num_block=num_block, - num_head=num_head, - num_batch=num_batch, - num_splits=1, - seqlen_k=0, - headdim=mdQ.shape[2], - headdim_v=0, - total_q=mdQ.shape[0], - tile_shape_mn=(self.tile_m, 1), - mCuSeqlensQ=mCuSeqlensQ, - mSeqUsedQ=mSeqUsedQ, - ) - - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - - # grid_dim: (m_block, num_head, batch_size) - self.kernel( - mdQaccum, - mdQ, - mCuSeqlensQ, - mSeqUsedQ, - scale, - self.tiled_mma, - self.dQ_swapAB, - self.sdQaccum_layout, - self.sdQ_layout, - self.g2s_tiled_copy_dQaccum, - self.s2r_tiled_copy_dQaccum, - self.gmem_tiled_copy_dQ, - tile_sched_params, - TileScheduler, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - smem=smem_size, - stream=stream, - ) - - @cute.kernel - def kernel( - self, - mdQaccum: cute.Tensor, - mdQ: cute.Tensor, - mCuSeqlensQ: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - scale: cutlass.Float32, - tiled_mma: cute.TiledMma, - dQ_swapAB: cutlass.Constexpr, - sdQaccum_layout: cute.Layout, - sdQ_layout: cute.ComposedLayout, - g2s_tiled_copy_dQaccum: cute.TiledCopy, - s2r_tiled_copy_dQaccum: cute.TiledCopy, - gmem_tiled_copy_dQ: cute.TiledCopy, - tile_sched_params: ParamsBase, - TileScheduler: cutlass.Constexpr[Callable], - ): - # /////////////////////////////////////////////////////////////////////////////// - # Get shared memory buffer - # /////////////////////////////////////////////////////////////////////////////// - smem = cutlass.utils.SmemAllocator() - sdQaccum = smem.allocate_tensor(cutlass.Float32, sdQaccum_layout, byte_alignment=1024) - sdQaccum_flat = cute.make_tensor(sdQaccum.iterator, cute.make_layout(cute.size(sdQaccum))) - if const_expr(self.arch in [80, 90]): - sdQ = cute.make_tensor(cute.recast_ptr(sdQaccum.iterator, dtype=self.dtype), sdQ_layout) - else: - # extra stage dimension - sdQ = cute.make_tensor( - cute.recast_ptr(sdQaccum.iterator, sdQ_layout.inner, dtype=self.dtype), - sdQ_layout.outer, - )[None, None, 0] - sdQt = utils.transpose_view(sdQ) - - # Thread index, block index - tidx, _, _ = cute.arch.thread_idx() - - tile_scheduler = TileScheduler.create(tile_sched_params) - work_tile = tile_scheduler.initial_work_tile_info() - - m_block, head_idx, batch_idx, _ = work_tile.tile_idx - - if work_tile.is_valid_tile: - # /////////////////////////////////////////////////////////////////////////////// - # Get the appropriate tiles for this thread block. - # /////////////////////////////////////////////////////////////////////////////// - - seqlen = SeqlenInfoQK.create( - batch_idx, - mdQ.shape[1], - 0, - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=None, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=None, - ) - if const_expr(not seqlen.has_cu_seqlens_q): - mdQ_cur = mdQ[batch_idx, None, head_idx, None] - mdQaccum_cur = mdQaccum[batch_idx, head_idx, None] - head_dim = mdQ.shape[3] - else: - padded_offset_q = seqlen.offset_q + batch_idx * self.tile_m - if cutlass.const_expr(self.arch >= 90): - padded_offset_q = padded_offset_q // self.tile_m * self.tile_m - mdQ_cur = cute.domain_offset((seqlen.offset_q, 0), mdQ[None, head_idx, None]) - mdQaccum_cur = cute.domain_offset( - (padded_offset_q * self.tile_hdim,), mdQaccum[head_idx, None] - ) - head_dim = mdQ.shape[2] - - # HACK: Compiler doesn't seem to recognize that padding - # by padded_offset_q * self.tile_hdim keeps alignment - # since statically divisible by 4 - - mdQaccum_cur_ptr = cute.make_ptr( - dtype=mdQaccum_cur.element_type, - value=mdQaccum_cur.iterator.toint(), - mem_space=mdQaccum_cur.iterator.memspace, - assumed_align=mdQaccum.iterator.alignment, - ) - mdQaccum_cur = cute.make_tensor(mdQaccum_cur_ptr, mdQaccum_cur.layout) - - gdQaccum = cute.local_tile(mdQaccum_cur, (self.tile_m * self.tile_hdim,), (m_block,)) - gdQ = cute.local_tile(mdQ_cur, (self.tile_m, self.tile_hdim), (m_block, 0)) - - seqlen_q = seqlen.seqlen_q - seqlen_q_rounded = cute.round_up(seqlen_q, self.tile_m) - - # Step 1: load dQaccum from gmem to smem - g2s_thr_copy_dQaccum = g2s_tiled_copy_dQaccum.get_slice(tidx) - tdQgdQaccum = g2s_thr_copy_dQaccum.partition_S(gdQaccum) - tdQsdQaccumg2s = g2s_thr_copy_dQaccum.partition_D(sdQaccum_flat) - cute.copy(g2s_tiled_copy_dQaccum, tdQgdQaccum, tdQsdQaccumg2s) - cute.arch.cp_async_commit_group() - cute.arch.cp_async_wait_group(0) - cute.arch.barrier() - - # Step 2: load dQ from smem to rmem - s2r_thr_copy_dQaccum = s2r_tiled_copy_dQaccum.get_slice(tidx) - tdQsdQaccum = s2r_thr_copy_dQaccum.partition_S(sdQaccum) - tile_shape = (self.tile_m, self.tile_hdim) - acc = None - tiled_copy_t2r = None - if const_expr(self.arch in [80, 90]): - acc_shape = tiled_mma.partition_shape_C( - tile_shape if const_expr(not dQ_swapAB) else tile_shape[::-1] - ) - acc = cute.make_fragment(acc_shape, cutlass.Float32) - assert cute.size(acc) == cute.size(tdQsdQaccum) - else: - thr_mma = tiled_mma.get_slice(0) # 1-CTA - dQacc_shape = tiled_mma.partition_shape_C((self.tile_m, self.tile_hdim)) - tdQtdQ = tiled_mma.make_fragment_C(dQacc_shape) - tdQcdQ = thr_mma.partition_C( - cute.make_identity_tensor((self.tile_m, self.tile_hdim)) - ) - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(self.dQ_reduce_ncol)), Float32 - ) - tiled_copy_t2r = tcgen05.make_tmem_copy(tmem_load_atom, tdQtdQ) - thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) - tdQrdQ_t2r_shape = thr_copy_t2r.partition_D(tdQcdQ).shape - acc = cute.make_fragment(tdQrdQ_t2r_shape, Float32) - tdQrdQaccum = cute.make_tensor(acc.iterator, cute.make_layout(tdQsdQaccum.shape)) - cute.autovec_copy(tdQsdQaccum, tdQrdQaccum) - # Convert tdQrdQaccum from fp32 to fp16/bf16 - rdQ = cute.make_fragment_like(acc, self.dtype) - rdQ.store((acc.load() * scale).to(self.dtype)) - - # Step 3: Copy dQ from register to smem - cute.arch.barrier() # make sure all threads have finished loading dQaccum - if const_expr(self.arch in [80, 90]): - copy_atom_r2s_dQ = utils.get_smem_store_atom( - self.arch, self.dtype, transpose=self.dQ_swapAB - ) - tiled_copy_r2s_dQ = cute.make_tiled_copy_C(copy_atom_r2s_dQ, tiled_mma) - else: - # copy_atom_r2s_dQ = sm100_utils_basic.get_smem_store_op( - # LayoutEnum.ROW_MAJOR, self.dtype, Float32, tiled_copy_t2r, - # ) - # tiled_copy_r2s_dQ = cute.make_tiled_copy_D(copy_atom_r2s_dQ, tiled_copy_t2r) - thr_layout_r2s_dQ = cute.make_layout((self.num_threads, 1)) # 128 threads - val_layout_r2s_dQ = cute.make_layout((1, 128 // self.dtype.width)) - copy_atom_r2s_dQ = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dtype, - num_bits_per_copy=128, - ) - tiled_copy_r2s_dQ = cute.make_tiled_copy_tv( - copy_atom_r2s_dQ, thr_layout_r2s_dQ, val_layout_r2s_dQ - ) - thr_copy_r2s_dQ = tiled_copy_r2s_dQ.get_slice(tidx) - cdQ = cute.make_identity_tensor((self.tile_m, self.tile_hdim)) - if const_expr(self.arch in [80, 90]): - taccdQrdQ = thr_copy_r2s_dQ.retile(rdQ) - else: - taccdQcdQ_shape = thr_copy_r2s_dQ.partition_S(cdQ).shape - taccdQrdQ = cute.make_tensor(rdQ.iterator, taccdQcdQ_shape) - taccdQsdQ = thr_copy_r2s_dQ.partition_D(sdQ if const_expr(not self.dQ_swapAB) else sdQt) - cute.copy(thr_copy_r2s_dQ, taccdQrdQ, taccdQsdQ) - - # Step 4: Copy dQ from smem to register to prepare for coalesced write to gmem - cute.arch.barrier() # make sure all smem stores are done - gmem_thr_copy_dQ = gmem_tiled_copy_dQ.get_slice(tidx) - tdQgdQ = gmem_thr_copy_dQ.partition_S(gdQ) - tdQsdQ = gmem_thr_copy_dQ.partition_D(sdQ) - tdQrdQ = cute.make_fragment_like(tdQsdQ, self.dtype) - # TODO: check OOB when reading from smem if kBlockM isn't evenly tiled - cute.autovec_copy(tdQsdQ, tdQrdQ) - - # Step 5: Copy dQ from register to gmem - tdQcdQ = gmem_thr_copy_dQ.partition_S(cdQ) - tdQpdQ = utils.predicate_k(tdQcdQ, limit=head_dim) - for rest_m in cutlass.range(cute.size(tdQrdQ.shape[1]), unroll_full=True): - if tdQcdQ[0, rest_m, 0][0] < seqlen_q - m_block * self.tile_m: - cute.copy( - gmem_tiled_copy_dQ, - tdQrdQ[None, rest_m, None], - tdQgdQ[None, rest_m, None], - pred=tdQpdQ[None, rest_m, None], - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_preprocess.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_preprocess.py deleted file mode 100644 index d9fc22875240..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_preprocess.py +++ /dev/null @@ -1,365 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# A reimplementation of https://github.com/Dao-AILab/flash-attention/blob/main/hopper/flash_bwd_preprocess_kernel.h -# from Cutlass C++ to Cute-DSL. -import math -import operator -from typing import Callable, Type, Optional, Literal - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass import Float32 - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -from .seqlen_info import SeqlenInfoQK -from .tile_scheduler import ( - ParamsBase, - SingleTileScheduler, - SingleTileVarlenScheduler, - TileSchedulerArguments, -) - - -class FlashAttentionBackwardPreprocess: - def __init__( - self, - dtype: Type[cutlass.Numeric], - head_dim: int, - arch: Literal[80, 90, 100], - m_block_size: int = 128, - num_threads: int = 128, - ): - """ - All contiguous dimensions must be at least 16 bytes aligned which indicates the head dimension - should be a multiple of 8. - - :param head_dim: head dimension - :type head_dim: int - :param m_block_size: m block size - :type m_block_size: int - :param num_threads: number of threads - :type num_threads: int - """ - self.dtype = dtype - self.m_block_size = m_block_size - self.arch = arch - # padding head_dim to a multiple of 32 as k_block_size - hdim_multiple_of = 32 - self.head_dim_padded = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - self.check_hdim_oob = head_dim != self.head_dim_padded - self.num_threads = num_threads - - @staticmethod - def can_implement(dtype, head_dim, m_block_size, num_threads) -> bool: - """Check if the kernel can be implemented with the given parameters. - - :param dtype: data type - :type dtype: cutlass.Numeric - :param head_dim: head dimension - :type head_dim: int - :param m_block_size: m block size - :type m_block_size: int - :param num_threads: number of threads - :type num_threads: int - - :return: True if the kernel can be implemented, False otherwise - :rtype: bool - """ - if dtype not in [cutlass.Float16, cutlass.BFloat16]: - return False - if head_dim % 8 != 0: - return False - if num_threads % 32 != 0: - return False - if num_threads < m_block_size: # For multiplying lse with log2 - return False - return True - - def _setup_attributes(self): - # /////////////////////////////////////////////////////////////////////////////// - # GMEM Tiled copy: - # /////////////////////////////////////////////////////////////////////////////// - # Thread layouts for copies - # We want kBlockKGmem to be a power of 2 so that when we do the summing, - # it's just between threads in the same warp - gmem_k_block_size = ( - 128 - if self.head_dim_padded % 128 == 0 - else ( - 64 - if self.head_dim_padded % 64 == 0 - else (32 if self.head_dim_padded % 32 == 0 else 16) - ) - ) - self.gmem_tiled_copy_O = copy_utils.tiled_copy_2d( - self.dtype, gmem_k_block_size, self.num_threads - ) - universal_copy_bits = 128 - num_copy_elems_dQaccum = universal_copy_bits // Float32.width - assert ( - self.m_block_size * self.head_dim_padded // num_copy_elems_dQaccum - ) % self.num_threads == 0 - self.gmem_tiled_copy_dQaccum = copy_utils.tiled_copy_1d( - Float32, self.num_threads, num_copy_elems_dQaccum - ) - - @cute.jit - def __call__( - self, - mO: cute.Tensor, - mdO: cute.Tensor, - mdPsum: cute.Tensor, - mLSE: Optional[cute.Tensor], - mLSElog2: Optional[cute.Tensor], - mdQaccum: Optional[cute.Tensor], - mCuSeqlensQ: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - stream: cuda.CUstream, - ): - # Get the data type and check if it is fp16 or bf16 - if cutlass.const_expr(not (mO.element_type == mdO.element_type)): - raise TypeError("All tensors must have the same data type") - if cutlass.const_expr(mO.element_type not in [cutlass.Float16, cutlass.BFloat16]): - raise TypeError("Only Float16 or BFloat16 is supported") - if cutlass.const_expr(mdPsum.element_type not in [Float32]): - raise TypeError("dPsum tensor must be Float32") - if cutlass.const_expr(mdQaccum is not None): - if cutlass.const_expr(mdQaccum.element_type not in [Float32]): - raise TypeError("dQaccum tensor must be Float32") - if cutlass.const_expr(mLSE is not None): - assert mLSElog2 is not None, "If mLSE is provided, mLSElog2 must also be provided" - if cutlass.const_expr(mLSE.element_type not in [Float32]): - raise TypeError("LSE tensor must be Float32") - if cutlass.const_expr(mLSElog2.element_type not in [Float32]): - raise TypeError("LSElog2 tensor must be Float32") - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mO, mdO, mdQaccum = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - if t is not None - else None - for t in (mO, mdO, mdQaccum) - ] - - self._setup_attributes() - - if cutlass.const_expr(mCuSeqlensQ is not None): - TileScheduler = SingleTileVarlenScheduler - num_head = mO.shape[1] - num_batch = mCuSeqlensQ.shape[0] - 1 - else: - TileScheduler = SingleTileScheduler - num_head = mO.shape[2] - num_batch = mO.shape[0] - - tile_sched_args = TileSchedulerArguments( - num_block=cute.ceil_div(mO.shape[1], self.m_block_size), - num_head=num_head, - num_batch=num_batch, - num_splits=1, - seqlen_k=0, - headdim=0, - headdim_v=mO.shape[2], - total_q=mO.shape[0], - tile_shape_mn=(self.m_block_size, 1), - mCuSeqlensQ=mCuSeqlensQ, - mSeqUsedQ=mSeqUsedQ, - ) - - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - - self.kernel( - mO, - mdO, - mdPsum, - mLSE, - mLSElog2, - mdQaccum, - mCuSeqlensQ, - mSeqUsedQ, - self.gmem_tiled_copy_O, - self.gmem_tiled_copy_dQaccum, - tile_sched_params, - TileScheduler, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - stream=stream, - ) - - @cute.kernel - def kernel( - self, - mO: cute.Tensor, - mdO: cute.Tensor, - mdPsum: cute.Tensor, - mLSE: Optional[cute.Tensor], - mLSElog2: Optional[cute.Tensor], - mdQaccum: Optional[cute.Tensor], - mCuSeqlensQ: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - gmem_tiled_copy_O: cute.TiledCopy, - gmem_tiled_copy_dQaccum: cute.TiledCopy, - tile_sched_params: ParamsBase, - TileScheduler: cutlass.Constexpr[Callable], - ): - # Thread index, block index - tidx, _, _ = cute.arch.thread_idx() - - tile_scheduler = TileScheduler.create(tile_sched_params) - work_tile = tile_scheduler.initial_work_tile_info() - m_block, head_idx, batch_idx, _ = work_tile.tile_idx - - if work_tile.is_valid_tile: - # /////////////////////////////////////////////////////////////////////////////// - # Get the appropriate tiles for this thread block. - # /////////////////////////////////////////////////////////////////////////////// - seqlen = SeqlenInfoQK.create( - batch_idx, - mO.shape[1], - 0, - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=None, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=None, - ) - - if cutlass.const_expr(not seqlen.has_cu_seqlens_q): - mO_cur = mO[batch_idx, None, head_idx, None] - mdO_cur = mdO[batch_idx, None, head_idx, None] - mdPsum_cur = mdPsum[batch_idx, head_idx, None] - headdim_v = mO.shape[3] - else: - mO_cur = cute.domain_offset((seqlen.offset_q, 0), mO[None, head_idx, None]) - mdO_cur = cute.domain_offset((seqlen.offset_q, 0), mdO[None, head_idx, None]) - - padded_offset_q = seqlen.offset_q + batch_idx * self.m_block_size - if cutlass.const_expr(self.arch >= 90): - padded_offset_q = padded_offset_q // self.m_block_size * self.m_block_size - mdPsum_cur = cute.domain_offset((padded_offset_q,), mdPsum[head_idx, None]) - headdim_v = mO.shape[2] - - blkOdO_shape = (self.m_block_size, self.head_dim_padded) - # (m_block_size, head_dim) - gO = cute.local_tile(mO_cur, blkOdO_shape, (m_block, 0)) - gdO = cute.local_tile(mdO_cur, blkOdO_shape, (m_block, 0)) - - gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx) - # (CPY_Atom, CPY_M, CPY_K) - tOgO = gmem_thr_copy_O.partition_S(gO) - tOgdO = gmem_thr_copy_O.partition_S(gdO) - - # /////////////////////////////////////////////////////////////////////////////// - # Predicate: Mark indices that need to copy when problem_shape isn't a multiple - # of tile_shape - # /////////////////////////////////////////////////////////////////////////////// - # Construct identity layout for KV - cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded)) - tOcO = gmem_thr_copy_O.partition_S(cO) - t0OcO = gmem_thr_copy_O.get_slice(0).partition_S(cO) - tOpO = utils.predicate_k(tOcO, limit=headdim_v) - tOpdO = utils.predicate_k(tOcO, limit=headdim_v) - - seqlen_q = seqlen.seqlen_q - seqlen_q_rounded = cute.round_up(seqlen_q, self.m_block_size) - - if cutlass.const_expr(mLSE is not None): - if cutlass.const_expr(not seqlen.has_cu_seqlens_q): - mLSE_cur = mLSE[batch_idx, head_idx, None] - else: - mLSE_cur = cute.domain_offset((seqlen.offset_q,), mLSE[head_idx, None]) - - gLSE = cute.local_tile(mLSE_cur, (self.m_block_size,), (m_block,)) - lse = Float32.inf - if tidx < seqlen_q - m_block * self.m_block_size: - lse = gLSE[tidx] - - tOrO = cute.make_fragment_like(tOgO) - tOrdO = cute.make_fragment_like(tOgdO) - assert cute.size(tOgO, mode=[0]) == cute.size(tOgdO, mode=[0]) - assert cute.size(tOgO, mode=[1]) == cute.size(tOgdO, mode=[1]) - assert cute.size(tOgO, mode=[2]) == cute.size(tOgdO, mode=[2]) - for m in cutlass.range(cute.size(tOrO.shape[1]), unroll_full=True): - # Instead of using tOcO, we using t0OcO and subtract the offset from the limit - # (seqlen_q - m_block * kBlockM). This is because the entries of t0OcO are known at compile time. - if t0OcO[0, m, 0][0] < seqlen_q - m_block * self.m_block_size - tOcO[0][0]: - cute.copy( - gmem_thr_copy_O, - tOgO[None, m, None], - tOrO[None, m, None], - pred=tOpO[None, m, None] - if cutlass.const_expr(self.check_hdim_oob) - else None, - ) - cute.copy( - gmem_thr_copy_O, - tOgdO[None, m, None], - tOrdO[None, m, None], - pred=tOpdO[None, m, None] - if cutlass.const_expr(self.check_hdim_oob) - else None, - ) - # Sum across the "k" dimension - dpsum = (tOrO.load().to(Float32) * tOrdO.load().to(Float32)).reduce( - cute.ReductionOp.ADD, init_val=0.0, reduction_profile=(0, None, 1) - ) - threads_per_row = gmem_tiled_copy_O.layout_src_tv_tiled[0].shape[0] - assert cute.arch.WARP_SIZE % threads_per_row == 0 - dpsum = utils.warp_reduce(dpsum, operator.add, width=threads_per_row) - dP_sum = cute.make_fragment(cute.size(tOrO, mode=[1]), Float32) - dP_sum.store(dpsum) - - # Write dPsum from rmem -> gmem - gdPsum = cute.local_tile(mdPsum_cur, (self.m_block_size,), (m_block,)) - # Only the thread corresponding to column 0 writes out the dPsum to gmem - if tOcO[0, 0, 0][1] == 0: - for m in cutlass.range(cute.size(dP_sum), unroll_full=True): - row = tOcO[0, m, 0][0] - gdPsum[row] = dP_sum[m] if row < seqlen_q - m_block * self.m_block_size else 0.0 - - # Clear dQaccum - if cutlass.const_expr(mdQaccum is not None): - if cutlass.const_expr(not seqlen.has_cu_seqlens_q): - mdQaccum_cur = mdQaccum[batch_idx, head_idx, None] - else: - mdQaccum_cur = cute.domain_offset( - (padded_offset_q * self.head_dim_padded,), mdQaccum[head_idx, None] - ) - - # HACK: Compiler doesn't seem to recognize that padding - # by padded_offset_q * self.head_dim_padded keeps alignment - # since statically divisible by 4 - - mdQaccum_cur_ptr = cute.make_ptr( - dtype=mdQaccum_cur.element_type, - value=mdQaccum_cur.iterator.toint(), - mem_space=mdQaccum_cur.iterator.memspace, - assumed_align=mdQaccum.iterator.alignment, - ) - mdQaccum_cur = cute.make_tensor(mdQaccum_cur_ptr, mdQaccum_cur.layout) - - blkdQaccum_shape = (self.m_block_size * self.head_dim_padded,) - gdQaccum = cute.local_tile(mdQaccum_cur, blkdQaccum_shape, (m_block,)) - gmem_thr_copy_dQaccum = gmem_tiled_copy_dQaccum.get_slice(tidx) - tdQgdQaccum = gmem_thr_copy_dQaccum.partition_S(gdQaccum) - zero = cute.make_fragment_like(tdQgdQaccum) - zero.fill(0.0) - cute.copy(gmem_tiled_copy_dQaccum, zero, tdQgdQaccum) - - if cutlass.const_expr(mLSE is not None): - if cutlass.const_expr(not seqlen.has_cu_seqlens_q): - mLSElog2_cur = mLSElog2[batch_idx, head_idx, None] - else: - mLSElog2_cur = cute.domain_offset((padded_offset_q,), mLSElog2[head_idx, None]) - - gLSElog2 = cute.local_tile(mLSElog2_cur, (self.m_block_size,), (m_block,)) - LOG2_E = math.log2(math.e) - if tidx < seqlen_q_rounded - m_block * self.m_block_size: - gLSElog2[tidx] = lse * LOG2_E if lse != -Float32.inf else 0.0 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm100.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm100.py deleted file mode 100644 index bfd9d8850ff7..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm100.py +++ /dev/null @@ -1,2942 +0,0 @@ -# Copyright (c) 2025, Ted Zadouri, Markus Hoehnerbach, Jay Shah, Tri Dao. -import math -from typing import Callable, Optional -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass.cute import FastDivmodDivisor -from cutlass import Float32, Int32, const_expr -from cutlass.utils import LayoutEnum -from cutlass.cute.nvgpu import cpasync, tcgen05 -import cutlass.utils.blackwell_helpers as sm100_utils_basic -from cutlass.pipeline import PipelineAsync, PipelineConsumer - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.pipeline as pipeline -from .blackwell_helpers import gemm_w_idx, gemm_ptx_w_idx # noqa -from .mask import AttentionMask -from .seqlen_info import SeqlenInfoQK -from .block_info import BlockInfo -from .tile_scheduler import ( - TileSchedulerArguments, - SingleTileScheduler, - SingleTileLPTBwdScheduler, # noqa - SingleTileVarlenScheduler, - ParamsBase, -) - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.barrier as barrier -from .named_barrier import NamedBarrierBwdSm100 -from .softmax import apply_score_mod_inner, apply_score_mod_bwd_inner -from .block_sparsity import BlockSparseTensors -from .block_sparse_utils import ( - get_total_q_block_count_bwd, - get_block_sparse_iteration_info_bwd, - get_m_block_from_iter_bwd, - produce_block_sparse_q_loads_bwd_sm100, -) - - -class FlashAttentionBackwardSm100: - arch = 100 - - def __init__( - self, - head_dim: int, - head_dim_v: Optional[int] = None, - is_causal: bool = False, - is_local: bool = False, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, - tile_m: int = 128, - tile_n: int = 128, - is_persistent: bool = False, - deterministic: bool = False, - cluster_size: int = 1, - score_mod: cutlass.Constexpr | None = None, - score_mod_bwd: cutlass.Constexpr | None = None, - mask_mod: cutlass.Constexpr | None = None, - has_aux_tensors: cutlass.Constexpr = False, - subtile_factor: cutlass.Constexpr[int] = 1, - ): - # padding head_dim to a multiple of 16 as k_block_size - hdim_multiple_of = 16 - self.tile_hdim = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - head_dim_v = head_dim_v if head_dim_v is not None else head_dim - self.same_hdim_kv = head_dim == head_dim_v - assert head_dim == head_dim_v, "head_dim and head_dim_v must be the same for now" - self.tile_hdimv = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of) - assert self.tile_hdim == self.tile_hdimv, ( - "tile_hdim and tile_hdimv must be the same for now" - ) - self.check_hdim_oob = head_dim != self.tile_hdim - self.check_hdim_v_oob = head_dim_v != self.tile_hdimv - - self.tile_m = tile_m - self.tile_n = tile_n - - # CTA tiler - self.cta_tiler = (tile_n, tile_m, self.tile_hdim) - # S = K @ Q.T - self.mma_tiler_kq = (tile_n, tile_m, self.tile_hdim) - # dP = V @ dO.T - self.mma_tiler_vdo = (tile_n, tile_m, self.tile_hdimv) - # dV = P.T @ dO - self.mma_tiler_pdo = (tile_n, self.tile_hdimv, tile_m) - # dK = dS.T @ Q (N, M) (M, D) - self.mma_tiler_dsq = (tile_n, self.tile_hdimv, tile_m) - # dQ = dS @ K - self.mma_tiler_dsk = (tile_m, self.tile_hdimv, tile_n) - - self.acc_dtype = Float32 - - assert cluster_size in (1, 2), "Only cluster_size=1 or 2 is supported" - self.cluster_shape_mn = (cluster_size, 1) - self.is_persistent = is_persistent - self.is_causal = is_causal - self.is_local = is_local - self.qhead_per_kvhead = qhead_per_kvhead - self.pack_gqa = False - self.deterministic = deterministic - - # Score mod and mask mod support - self.score_mod = score_mod - self.score_mod_bwd = score_mod_bwd - self.mask_mod = mask_mod - self.has_aux_tensors = has_aux_tensors - self.subtile_factor = subtile_factor - # For score_mod, use vec_size=1 (like forward) to handle per-element indices - if cutlass.const_expr(has_aux_tensors): - self.vec_size: cutlass.Constexpr = 1 - else: - self.vec_size: cutlass.Constexpr = 4 - self.qk_acc_dtype = Float32 - - # Speed optimizations, does not affect correctness - self.shuffle_LSE = False - self.shuffle_dPsum = False - self.use_smem_dS_for_mma_dK = self.deterministic and self.is_causal - - self.reduce_warp_ids = (0, 1, 2, 3) - self.compute_warp_ids = (4, 5, 6, 7, 8, 9, 10, 11) - self.mma_warp_id = 12 - self.load_warp_id = 13 - self.epi_warp_id = 14 - self.empty_warp_id = 15 - - # 16 warps -> 512 threads - self.threads_per_cta = cute.arch.WARP_SIZE * len( - ( - *self.reduce_warp_ids, - *self.compute_warp_ids, - self.mma_warp_id, - self.load_warp_id, - self.epi_warp_id, - self.empty_warp_id, - ) - ) - - # NamedBarrier - self.compute_sync_barrier = cutlass.pipeline.NamedBarrier( - barrier_id=int(NamedBarrierBwdSm100.Compute), - num_threads=len(self.compute_warp_ids) * cute.arch.WARP_SIZE, - ) - # self.epilogue_sync_barrier = pipeline.NamedBarrier( - # barrier_id=2, - # num_threads=self.num_compute_warps * self.threads_per_warp, - # ) - self.reduce_sync_barrier = cutlass.pipeline.NamedBarrier( - barrier_id=int(NamedBarrierBwdSm100.dQaccReduce), - num_threads=len(self.reduce_warp_ids) * cute.arch.WARP_SIZE, - ) - - # TMEM setup - SM100_TMEM_CAPACITY_COLUMNS = 512 - self.tmem_alloc_cols = SM100_TMEM_CAPACITY_COLUMNS - - # self.tmem_dK_offset = 0 - # self.tmem_dV_offset = self.tmem_dK_offset + self.tile_hdim - # self.tmem_dQ_offset = self.tmem_dV_offset + self.tile_hdimv - # self.tmem_dP_offset = self.tmem_dQ_offset # overlap with dQ - # self.tmem_S_offset = self.tmem_dQ_offset + max(self.tile_m, self.tile_hdim) - # self.tmem_P_offset = self.tmem_S_offset # overlap with S - # self.tmem_total = self.tmem_S_offset + self.tile_n - # assert self.tmem_total <= self.tmem_alloc_cols - - self.tmem_S_offset = 0 - self.tmem_P_offset = 0 # overlap with S - self.tmem_dV_offset = self.tmem_S_offset + self.tile_n - self.tmem_dP_offset = self.tmem_dV_offset + self.tile_hdimv - self.tmem_dQ_offset = self.tmem_dP_offset # overlap with dP - self.tmem_dK_offset = self.tmem_dP_offset + self.tile_m - self.tmem_dS_offset = self.tmem_dP_offset # overlap with dP - - if (not is_causal and not is_local) or deterministic: - self.num_regs_reduce = 152 - self.num_regs_compute = 136 - else: - self.num_regs_reduce = 136 - self.num_regs_compute = 144 - self.num_regs_other = 96 - 8 - self.num_regs_empty = 24 - assert self.num_regs_reduce + self.num_regs_compute * 2 + self.num_regs_other <= 512 - - self.buffer_align_bytes = 1024 - - def _setup_attributes(self): - self.Q_stage = 2 - self.dO_stage = 1 - # LSE_stage = Q_stage and dPsum_stage = dO_stage - # self.sdKVaccum_stage = 2 - # number of tma reduce adds per dQacc mma - self.dQ_reduce_ncol = 32 - self.sdQaccum_stage = 64 // self.dQ_reduce_ncol - assert self.tile_hdim % self.dQ_reduce_ncol == 0 - self.dQaccum_reduce_stage = self.tile_hdim // self.dQ_reduce_ncol - self.cluster_reduce_dQ = False and cute.size(self.cluster_shape_mn) > 1 - # number of tma reduce adds for dKacc and dVacc epilogue - self.dK_reduce_ncol = 32 - - def _get_tiled_mma(self): - cta_group = tcgen05.CtaGroup.ONE - # S = K @ Q.T - tiled_mma_S = sm100_utils_basic.make_trivial_tiled_mma( - self.q_dtype, - tcgen05.OperandMajorMode.K, - tcgen05.OperandMajorMode.K, - self.acc_dtype, - cta_group, - self.mma_tiler_kq[:2], - ) - # dP = V @ dO.T - tiled_mma_dP = sm100_utils_basic.make_trivial_tiled_mma( - self.do_dtype, - tcgen05.OperandMajorMode.K, - tcgen05.OperandMajorMode.K, - self.acc_dtype, - cta_group, - self.mma_tiler_vdo[:2], - ) - # dV += P @ dO --> (K, MN) major - tiled_mma_dV = sm100_utils_basic.make_trivial_tiled_mma( - self.do_dtype, - tcgen05.OperandMajorMode.K, # P_major_mode - tcgen05.OperandMajorMode.MN, # dO_major_mode - self.acc_dtype, - cta_group, - self.mma_tiler_pdo[:2], - a_source=tcgen05.OperandSource.TMEM, - ) - # dK += dS.T @ Q - if const_expr(self.use_smem_dS_for_mma_dK): - mma_dK_a_src = tcgen05.OperandSource.SMEM - else: - mma_dK_a_src = tcgen05.OperandSource.TMEM - tiled_mma_dK = sm100_utils_basic.make_trivial_tiled_mma( - self.do_dtype, - tcgen05.OperandMajorMode.K, # dS_major_mode - tcgen05.OperandMajorMode.MN, # Q_major_mode - self.acc_dtype, - cta_group, - self.mma_tiler_dsq[:2], - a_source=mma_dK_a_src, - ) - # dQ = dS @ K - tiled_mma_dQ = sm100_utils_basic.make_trivial_tiled_mma( - self.k_dtype, - tcgen05.OperandMajorMode.MN, # dS_major_mode - tcgen05.OperandMajorMode.MN, # Kt_major_mode - self.acc_dtype, - cta_group, - self.mma_tiler_dsk[:2], - ) - return tiled_mma_S, tiled_mma_dP, tiled_mma_dK, tiled_mma_dV, tiled_mma_dQ - - def _setup_smem_layout(self): - # S = K @ Q.T - sK_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_S, - self.mma_tiler_kq, - self.k_dtype, - 1, - ) - self.sK_layout = cute.slice_(sK_layout, (None, None, None, 0)) - self.sQ_layout = sm100_utils_basic.make_smem_layout_b( - self.tiled_mma_S, - self.mma_tiler_kq, - self.q_dtype, - self.Q_stage, - ) - # dP = V @ dO.T - sV_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_dP, - self.mma_tiler_vdo, - self.v_dtype, - 1, - ) - self.sV_layout = cute.slice_(sV_layout, (None, None, None, 0)) - self.sdOt_layout = sm100_utils_basic.make_smem_layout_b( - self.tiled_mma_dP, - self.mma_tiler_vdo, - self.do_dtype, - self.dO_stage, - ) - # dV += P @ dO - tP_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_dV, - self.mma_tiler_pdo, - self.do_dtype, - 1, - ) - self.tP_layout = cute.slice_(tP_layout, (None, None, None, 0)) - self.sdO_layout = sm100_utils_basic.make_smem_layout_b( - self.tiled_mma_dV, - self.mma_tiler_pdo, - self.do_dtype, - self.dO_stage, - ) - # dK += dS.T @ Q - sdSt_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_dK, - self.mma_tiler_dsq, - self.ds_dtype, - 1, - ) - self.sdSt_layout = cute.slice_(sdSt_layout, (None, None, None, 0)) - tdS_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_dK, - self.mma_tiler_dsq, - self.ds_dtype, - 1, - ) - self.tdS_layout = cute.slice_(tdS_layout, (None, None, None, 0)) - self.sQt_layout = sm100_utils_basic.make_smem_layout_b( - self.tiled_mma_dK, - self.mma_tiler_dsq, - self.q_dtype, - self.Q_stage, - ) - # dQ = dS @ K - sdS_layout = sm100_utils_basic.make_smem_layout_a( - self.tiled_mma_dQ, - self.mma_tiler_dsk, - self.ds_dtype, - 1, - ) - self.sdS_layout = cute.slice_(sdS_layout, (None, None, None, 0)) - sKt_layout = sm100_utils_basic.make_smem_layout_b( - self.tiled_mma_dQ, - self.mma_tiler_dsk, - self.k_dtype, - 1, - ) - self.sKt_layout = cute.slice_(sKt_layout, (None, None, None, 0)) - self.sdQaccum_layout = cute.make_layout( - (self.tile_m * self.dQ_reduce_ncol, self.sdQaccum_stage) - ) - self.sLSE_layout = cute.make_layout( - shape=(self.tile_m, self.Q_stage), - stride=(1, cute.round_up(self.tile_m, 64)), - ) - self.sdPsum_layout = cute.make_layout( - shape=(self.tile_m, self.dO_stage), - stride=(1, cute.round_up(self.tile_m, 64)), - ) - self.sdKV_epi_tile = ( - self.tile_n, - min(128 // (self.dk_dtype.width // 8), self.tile_hdim // 2), # 64 or 32 - ) # subtiles mma_tiler_dsq[:2] = mma_tiler_pdo[:2] - # headdim_64 gets 1 stage - self.num_epi_stages = max(1, (self.tile_hdim // 2) // self.sdKV_epi_tile[1]) - self.sdKV_flat_epi_tile = self.tile_n * (self.tile_hdim // 2) // self.num_epi_stages - # TODO: dK and dV could have different shapes - if const_expr(not self.dKV_postprocess): - self.sdKV_layout = sm100_utils_basic.make_smem_layout_epi( - self.dk_dtype, - LayoutEnum.ROW_MAJOR, - self.sdKV_epi_tile, - 2, # num compute wgs - ) - else: - self.sdKV_layout = cute.make_layout((self.tile_n * self.dK_reduce_ncol, 2)) - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdQaccum: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - softmax_scale: Float32, - stream: cuda.CUstream, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - softcap: Float32 | float | None = None, - window_size_left: Int32 | int | None = None, - window_size_right: Int32 | int | None = None, - mdQ_semaphore: Optional[cute.Tensor] = None, - mdK_semaphore: Optional[cute.Tensor] = None, - mdV_semaphore: Optional[cute.Tensor] = None, - aux_tensors: Optional[list] = None, - # Block-sparse tensors (Q direction - for iterating m_blocks per n_block): - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - self.q_dtype = mQ.element_type - self.k_dtype = mK.element_type - self.v_dtype = mV.element_type - self.do_dtype = mdO.element_type - self.lse_dtype = mLSE.element_type - self.dpsum_dtype = mdPsum.element_type - self.dqaccum_dtype = mdQaccum.element_type - self.dk_dtype = mdK.element_type - self.dv_dtype = mdV.element_type - self.ds_dtype = self.q_dtype - - self.is_varlen_k = mCuSeqlensK is not None or mSeqUsedK is not None - self.is_varlen_q = mCuSeqlensQ is not None or mSeqUsedQ is not None - self.use_tma_store = not (self.qhead_per_kvhead == 1 and mCuSeqlensK is not None) - self.dKV_postprocess = self.qhead_per_kvhead > 1 - - if const_expr(self.dKV_postprocess): - assert self.dk_dtype.width == 32, "Must accumulate dK in float precision for GQA" - assert self.dv_dtype.width == 32, "Must accumulate dV in float precision for GQA" - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - ( - mdQaccum, - mdK, - mdV, - ) = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - if t is not None - else None - for t in ( - mdQaccum, - mdK, - mdV, - ) - ] - - # (b, s, n, h) --> (s, h, n, b) or (t, n, h) -> (t, h, n) - QO_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1] - mQ, mdO = [utils.select(t, mode=QO_layout_transpose) for t in (mQ, mdO)] - - KV_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensK is None) else [0, 2, 1] - mK, mV = [utils.select(t, mode=KV_layout_transpose) for t in (mK, mV)] - - # (b, n, s) --> (s, n, b) or (n, t) --> (t, n) - LSE_dPsum_dQaccum_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0] - mLSE, mdPsum, mdQaccum = [ - utils.select(t, mode=LSE_dPsum_dQaccum_transpose) for t in (mLSE, mdPsum, mdQaccum) - ] - - if const_expr(not self.dKV_postprocess): - layout_dKV_transpose = KV_layout_transpose - else: - layout_dKV_transpose = [2, 1, 0] if const_expr(mCuSeqlensK is None) else [1, 0] - mdK, mdV = [utils.select(t, mode=layout_dKV_transpose) for t in (mdK, mdV)] - # (s, h, n, b) --> (h, s, n, b) or (t, h, n) -> (h, t, b) - dO_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensQ is None) else [1, 0, 2] - mdO = utils.select(mdO, mode=dO_transpose) - - # (b, n, block, stage) -> (block, stage, n, b) - semaphore_transpose = [2, 3, 1, 0] - if const_expr(self.deterministic): - assert mdQ_semaphore is not None - mdQ_semaphore = utils.select(mdQ_semaphore, mode=semaphore_transpose) - - if const_expr(self.deterministic and self.qhead_per_kvhead > 1): - assert mdK_semaphore is not None - assert mdV_semaphore is not None - mdK_semaphore, mdV_semaphore = [ - utils.select(t, mode=semaphore_transpose) for t in (mdK_semaphore, mdV_semaphore) - ] - else: - mdK_semaphore = None - mdV_semaphore = None - - self._setup_attributes() - ( - self.tiled_mma_S, - self.tiled_mma_dP, - self.tiled_mma_dK, - self.tiled_mma_dV, - self.tiled_mma_dQ, - ) = self._get_tiled_mma() - self._setup_smem_layout() - - cta_group = tcgen05.CtaGroup.ONE - - self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) - self.cluster_layout_vmnk = cute.tiled_divide( - cute.make_layout(self.cluster_shape_mnk), - (self.tiled_mma_S.thr_id.shape,), - ) - self.num_mcast_ctas_b = cute.size(self.cluster_layout_vmnk.shape[1]) - self.is_q_do_mcast = self.num_mcast_ctas_b > 1 - - if const_expr(not self.dKV_postprocess): - self.mdK_layout_enum = LayoutEnum.from_tensor(mdK) - self.mdV_layout_enum = LayoutEnum.from_tensor(mdV) - dK_major_mode = self.mdK_layout_enum.mma_major_mode() - dV_major_mode = self.mdV_layout_enum.mma_major_mode() - if const_expr(dK_major_mode != tcgen05.OperandMajorMode.K): - raise RuntimeError("The layout of mdK is wrong") - if const_expr(dV_major_mode != tcgen05.OperandMajorMode.K): - raise RuntimeError("The layout of mdV is wrong") - - if const_expr(self.use_tma_store and not self.dKV_postprocess): - tma_copy_op_dKV = cpasync.CopyBulkTensorTileS2GOp() - tma_atom_dK, mdK_tma_tensor = cpasync.make_tiled_tma_atom( - tma_copy_op_dKV, - mdK, - cute.select(self.sdKV_layout, mode=[0, 1]), - self.sdKV_epi_tile, - 1, # no mcast - ) - tma_atom_dV, mdV_tma_tensor = cpasync.make_tiled_tma_atom( - tma_copy_op_dKV, - mdV, - cute.select(self.sdKV_layout, mode=[0, 1]), - self.sdKV_epi_tile, - 1, # no mcast - ) - else: - mdV_tma_tensor = mdV - mdK_tma_tensor = mdK - tma_atom_dV = None - tma_atom_dK = None - - if const_expr(not self.dKV_postprocess): - thr_layout_r2s_dKV = cute.make_ordered_layout((128, 1), order=(1, 0)) # 128 threads - val_layout_r2s_dKV = cute.make_ordered_layout( - (1, 128 // self.dk_dtype.width), order=(1, 0) - ) # 4 or 8 vals for 16 byte store - copy_atom_r2s_dKV = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dk_dtype, - num_bits_per_copy=128, - ) - tiled_copy_r2s_dKV = cute.make_tiled_copy_tv( - copy_atom_r2s_dKV, thr_layout_r2s_dKV, val_layout_r2s_dKV - ) - else: - tiled_copy_r2s_dKV = copy_utils.tiled_copy_1d( - Float32, 128, num_copy_elems=128 // Float32.width - ) - - tma_load_op = cpasync.CopyBulkTensorTileG2SOp(cta_group) - tma_load_op_multicast = cpasync.CopyBulkTensorTileG2SMulticastOp(cta_group) - - # S.T = K @ Q.T - tma_atom_K, tma_tensor_K = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - mK, - cute.select(self.sK_layout, mode=[0, 1, 2]), - self.mma_tiler_kq, - self.tiled_mma_S, - self.cluster_layout_vmnk.shape, - ) - Q_tma_op = sm100_utils_basic.cluster_shape_to_tma_atom_B( - self.cluster_shape_mnk, self.tiled_mma_S.thr_id - ) - tma_atom_Q, tma_tensor_Q = cute.nvgpu.make_tiled_tma_atom_B( - # tma_load_op if const_expr(self.cluster_shape_mnk[0] == 1) else tma_load_op_multicast, - Q_tma_op, - mQ, - cute.select(self.sQ_layout, mode=[0, 1, 2]), - self.mma_tiler_kq, - self.tiled_mma_S, - self.cluster_layout_vmnk.shape, - ) - # dP.T = V @ dO.T - tma_atom_V, tma_tensor_V = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - mV, - cute.select(self.sV_layout, mode=[0, 1, 2]), - self.mma_tiler_vdo, - self.tiled_mma_dP, - self.cluster_layout_vmnk.shape, - ) - dO_tma_op = sm100_utils_basic.cluster_shape_to_tma_atom_B( - self.cluster_shape_mnk, self.tiled_mma_dV.thr_id - ) - tma_atom_dO, tma_tensor_dO = cute.nvgpu.make_tiled_tma_atom_B( - # tma_load_op if const_expr(self.cluster_shape_mnk[0] == 1) else tma_load_op_multicast, - dO_tma_op, - mdO, - cute.select(self.sdO_layout, mode=[0, 1, 2]), - self.mma_tiler_pdo, - self.tiled_mma_dV, - self.cluster_layout_vmnk.shape, - ) - - self.tma_copy_bytes = { - name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1, 2])) - for name, mX, layout in [ - ("Q", mQ, self.sQ_layout), - ("K", mK, self.sK_layout), - ("V", mV, self.sV_layout), - ("dO", mdO, self.sdO_layout), - ] - } - self.tma_copy_bytes["LSE"] = self.tile_m * Float32.width // 8 - self.tma_copy_bytes["dPsum"] = self.tile_m * Float32.width // 8 - self.tma_copy_bytes["dQ"] = self.tile_m * self.dQ_reduce_ncol * Float32.width // 8 - self.tma_copy_bytes["dKacc"] = self.tile_n * self.dK_reduce_ncol * Float32.width // 8 - - # TileScheduler = SingleTileScheduler - if const_expr(self.is_varlen_k): - TileScheduler = SingleTileVarlenScheduler - elif const_expr(self.deterministic): - TileScheduler = SingleTileLPTBwdScheduler - else: - TileScheduler = SingleTileScheduler - # reads n_blocks right-to-left - self.spt = (self.is_causal or self.is_local) and self.deterministic - tile_sched_args = TileSchedulerArguments( - cute.ceil_div(cute.size(mK.shape[0]), self.cta_tiler[0]), # num_blocks - cute.size(mQ.shape[2]), # num_heads = num_query_heads - cute.size(mK.shape[3]) - if const_expr(mCuSeqlensK is None) - else cute.size(mCuSeqlensK.shape[0] - 1), # num_batches - 1, # num_splits - cute.size(mQ.shape[0]), # pass seqlen_q or total_q for seqlen_k - mQ.shape[1], # headdim - mV.shape[1], # headdim_v - total_q=cute.size(mK.shape[0]) # pass total_k for total_q - if const_expr(mCuSeqlensK is not None) - else cute.size(mK.shape[0]) * cute.size(mK.shape[3]), - tile_shape_mn=self.cta_tiler[:2], # (tile_n, tile_m) - cluster_shape_mn=self.cluster_shape_mnk[:2], - mCuSeqlensQ=mCuSeqlensK, - mSeqUsedQ=mSeqUsedK, - qhead_per_kvhead_packgqa=1, # pack_gqa disabled for bwd - element_size=self.k_dtype.width // 8, - is_persistent=self.is_persistent, # persistent mode not tested - lpt=self.spt, - head_swizzle=self.deterministic, - ) - - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - self.tile_scheduler_cls = TileScheduler - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - # cute.printf("grid_dim = {}", grid_dim) - - # Compute allocation sizes for shared buffers that are reused - # sQ is reused for sdK, sdO is reused for sdV - sQ_alloc_bytes = max( - cute.size_in_bytes(self.q_dtype, self.sQ_layout), - cute.size_in_bytes(self.dk_dtype, self.sdKV_layout), - ) - sdO_alloc_bytes = max( - cute.size_in_bytes(self.dv_dtype, self.sdKV_layout), - cute.size_in_bytes(self.do_dtype, self.sdO_layout), - ) - # Sanity check that layouts fit in allocation - sdV_bytes = cute.size_in_bytes(self.dv_dtype, self.sdKV_layout) - sdK_bytes = cute.size_in_bytes(self.dk_dtype, self.sdKV_layout) - assert sdV_bytes <= sdO_alloc_bytes, "sdV doesn't fit in sdO storage allocation" - assert sdK_bytes <= sQ_alloc_bytes, "sdK doesn't fit in sQ storage allocation" - - @cute.struct - class SharedStorage: - Q_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * self.Q_stage] - dO_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * self.dO_stage] - LSE_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * self.Q_stage] - dPsum_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * self.dO_stage] - S_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * 1] - dP_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * 1] - dS_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * 1] - dKV_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2 * 2] - dQ_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 2] - dQ_cluster_full_mbar_ptr: cute.struct.MemRange[ - cutlass.Int64, self.dQaccum_reduce_stage // 2 - ] - dQ_cluster_empty_mbar_ptr: cute.struct.MemRange[ - cutlass.Int64, self.dQaccum_reduce_stage // 2 - ] - tmem_holding_buf: Int32 - tmem_dealloc_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 1] - - # Smem tensors - - # sQ is reused for sdK which in the non-MHA case needs float32 - sQ: cute.struct.Align[ - cute.struct.MemRange[cute.Uint8, sQ_alloc_bytes], - self.buffer_align_bytes, - ] - sK: cute.struct.Align[ - cute.struct.MemRange[self.k_dtype, cute.cosize(self.sK_layout)], - self.buffer_align_bytes, - ] - sV: cute.struct.Align[ - cute.struct.MemRange[self.v_dtype, cute.cosize(self.sV_layout)], - self.buffer_align_bytes, - ] - # sdO is reused for sdV which in the non-MHA case needs float32 - sdO: cute.struct.Align[ - cute.struct.MemRange[cute.Uint8, sdO_alloc_bytes], - self.buffer_align_bytes, - ] - sdS: cute.struct.Align[ - cute.struct.MemRange[self.ds_dtype, cute.cosize(self.sdSt_layout)], - 128, - ] - sLSE: cute.struct.Align[ - cute.struct.MemRange[self.lse_dtype, cute.cosize(self.sLSE_layout)], - 128, - ] - sdPsum: cute.struct.Align[ - cute.struct.MemRange[self.dpsum_dtype, cute.cosize(self.sdPsum_layout)], - 128, - ] - sdQaccum: cute.struct.Align[ - cute.struct.MemRange[self.dqaccum_dtype, cute.cosize(self.sdQaccum_layout)], - self.buffer_align_bytes, - ] - - self.shared_storage = SharedStorage - - LOG2_E = math.log2(math.e) - if const_expr(self.score_mod is None): - # Without score_mod: bake scale into log2 - softmax_scale_log2 = softmax_scale * LOG2_E - else: - # With score_mod: score_mod applied to S * softmax_scale, then use LOG2_E only - softmax_scale_log2 = LOG2_E - - if const_expr(window_size_left is not None): - window_size_left = Int32(window_size_left) - if const_expr(window_size_right is not None): - window_size_right = Int32(window_size_right) - - fastdiv_mods = None - if const_expr(aux_tensors is not None): - seqlen_q = cute.size(mQ.shape[0]) // ( - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1 - ) - seqlen_k = cute.size(mK.shape[0]) - seqlen_q_divmod = FastDivmodDivisor(seqlen_q) - seqlen_k_divmod = FastDivmodDivisor(seqlen_k) - fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod) - self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None) - - if const_expr(self.use_block_sparsity or aux_tensors is not None): - assert all(x is None for x in (mCuSeqlensQ, mCuSeqlensK, mSeqUsedQ, mSeqUsedK)), ( - "Variable sequence length is not supported yet for blocksparse or aux tensors in bwd" - ) - - self.kernel( - tma_tensor_Q, - tma_tensor_K, - tma_tensor_V, - mLSE, - mdPsum, - tma_tensor_dO, - mdV, - mdK, - mdQaccum, - mdV_tma_tensor, - mdK_tma_tensor, - mdQ_semaphore, - mdK_semaphore, - mdV_semaphore, - mCuSeqlensQ, - mCuSeqlensK, - mSeqUsedQ, - mSeqUsedK, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_dO, - tma_atom_dV, - tma_atom_dK, - self.sQ_layout, - self.sQt_layout, - self.sK_layout, - self.sV_layout, - self.sLSE_layout, - self.sdPsum_layout, - self.sdO_layout, - self.sdOt_layout, - self.sdSt_layout, - self.sdS_layout, - self.sKt_layout, - self.sdQaccum_layout, - self.sdKV_layout, - self.tP_layout, - self.tdS_layout, - self.tiled_mma_S, - self.tiled_mma_dP, - self.tiled_mma_dV, - self.tiled_mma_dK, - self.tiled_mma_dQ, - tiled_copy_r2s_dKV, - softmax_scale, - softmax_scale_log2, - window_size_left, - window_size_right, - tile_sched_params, - aux_tensors, - fastdiv_mods, - blocksparse_tensors, - ).launch( - grid=grid_dim, - block=[self.threads_per_cta, 1, 1], - cluster=self.cluster_shape_mnk if cute.size(self.cluster_shape_mnk) > 1 else None, - smem=self.shared_storage.size_in_bytes(), - stream=stream, - min_blocks_per_mp=1, - ) - - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdO: cute.Tensor, - mdV: cute.Tensor, - mdK: cute.Tensor, - mdQaccum: cute.Tensor, - mdV_tma_tensor: Optional[cute.Tensor], - mdK_tma_tensor: Optional[cute.Tensor], - mdQ_semaphore: Optional[cute.Tensor], - mdK_semaphore: Optional[cute.Tensor], - mdV_semaphore: Optional[cute.Tensor], - mCuSeqlensQ: Optional[cute.Tensor], - mCuSeqlensK: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - mSeqUsedK: Optional[cute.Tensor], - tma_atom_Q: cute.CopyAtom, - tma_atom_K: cute.CopyAtom, - tma_atom_V: cute.CopyAtom, - tma_atom_dO: cute.CopyAtom, - tma_atom_dV: Optional[cute.CopyAtom], - tma_atom_dK: Optional[cute.CopyAtom], - sQ_layout: cute.ComposedLayout, - sQt_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sLSE_layout: cute.Layout, - sdPsum_layout: cute.Layout, - sdO_layout: cute.ComposedLayout, - sdOt_layout: cute.ComposedLayout, - sdSt_layout: cute.ComposedLayout, - sdS_layout: cute.ComposedLayout, - sKt_layout: cute.ComposedLayout, - sdQaccum_layout: cute.Layout, - sdKV_layout: cute.ComposedLayout | cute.Layout, - tP_layout: cute.ComposedLayout, - tdS_layout: cute.ComposedLayout, - tiled_mma_S: cute.TiledMma, - tiled_mma_dP: cute.TiledMma, - tiled_mma_dV: cute.TiledMma, - tiled_mma_dK: cute.TiledMma, - tiled_mma_dQ: cute.TiledMma, - tiled_copy_r2s_dKV: cute.TiledCopy, - softmax_scale: cutlass.Float32, - softmax_scale_log2: cutlass.Float32, - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - tile_sched_params: ParamsBase, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - - # Prefetch tma descriptor - if warp_idx == self.load_warp_id: - with cute.arch.elect_one(): - cpasync.prefetch_descriptor(tma_atom_Q) - cpasync.prefetch_descriptor(tma_atom_K) - cpasync.prefetch_descriptor(tma_atom_V) - cpasync.prefetch_descriptor(tma_atom_dO) - if const_expr(tma_atom_dV is not None): - cpasync.prefetch_descriptor(tma_atom_dV) - if const_expr(tma_atom_dK is not None): - cpasync.prefetch_descriptor(tma_atom_dK) - - cluster_layout_vmnk = cute.tiled_divide( - cute.make_layout(self.cluster_shape_mnk), - (tiled_mma_S.thr_id.shape,), - ) - - # Alloc - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(self.shared_storage) - - tmem_dealloc_mbar_ptr = storage.tmem_dealloc_mbar_ptr.data_ptr() - dQ_cluster_full_mbar_ptr = storage.dQ_cluster_full_mbar_ptr.data_ptr() - dQ_cluster_empty_mbar_ptr = storage.dQ_cluster_empty_mbar_ptr.data_ptr() - - if warp_idx == 1: - cute.arch.mbarrier_init( - tmem_dealloc_mbar_ptr, cute.arch.WARP_SIZE * len(self.compute_warp_ids) - ) - if const_expr(self.cluster_reduce_dQ): - if warp_idx == 4: - for i in range(self.dQaccum_reduce_stage // 2): - cute.arch.mbarrier_init(dQ_cluster_full_mbar_ptr + i, 1) - cute.arch.mbarrier_init(dQ_cluster_empty_mbar_ptr + i, 1) - - # UMMA producers and AsyncThread consumers - pipeline_producer_group_MMA_AsyncThread = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len([self.mma_warp_id]) - ) - # Only 1 thread per warp will signal - pipeline_consumer_group_MMA_AsyncThread = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len(self.compute_warp_ids) - ) - pipeline_S_P = cutlass.pipeline.PipelineUmmaAsync.create( - num_stages=1, - producer_group=pipeline_producer_group_MMA_AsyncThread, - consumer_group=pipeline_consumer_group_MMA_AsyncThread, - barrier_storage=storage.S_mbar_ptr.data_ptr(), - ) - pipeline_dP = cutlass.pipeline.PipelineUmmaAsync.create( - num_stages=1, - producer_group=pipeline_producer_group_MMA_AsyncThread, - consumer_group=pipeline_consumer_group_MMA_AsyncThread, - barrier_storage=storage.dP_mbar_ptr.data_ptr(), - ) - pipeline_dKV = cutlass.pipeline.PipelineUmmaAsync.create( - num_stages=2, - producer_group=pipeline_producer_group_MMA_AsyncThread, - consumer_group=pipeline_consumer_group_MMA_AsyncThread, - barrier_storage=storage.dKV_mbar_ptr.data_ptr(), - ) - pipeline_consumer_group_MMA_AsyncThread_dQ = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, - len(self.reduce_warp_ids), - ) # Compute - pipeline_dQ = cutlass.pipeline.PipelineUmmaAsync.create( - num_stages=1, - producer_group=pipeline_producer_group_MMA_AsyncThread, - consumer_group=pipeline_consumer_group_MMA_AsyncThread_dQ, - barrier_storage=storage.dQ_mbar_ptr.data_ptr(), - ) - - # AsyncThread producers and UMMA consumers - # Only 1 thread per warp will signal - pipeline_PdS_producer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len(self.compute_warp_ids) - ) # Compute - pipeline_PdS_consumer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len([self.mma_warp_id]) - ) # MMA - pipeline_dS = cutlass.pipeline.PipelineAsyncUmma.create( - num_stages=1, - producer_group=pipeline_PdS_producer_group, - consumer_group=pipeline_PdS_consumer_group, - barrier_storage=storage.dS_mbar_ptr.data_ptr(), - ) - - # TMA producer and UMMA consumers - pipeline_producer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len([self.load_warp_id]) - ) - # The arrive count is the number of mcast size - pipeline_consumer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len([self.mma_warp_id]) * self.num_mcast_ctas_b - ) - pipeline_consumer_group_compute = cutlass.pipeline.CooperativeGroup( - # cutlass.pipeline.Agent.Thread, len(self.compute_warp_ids) * self.num_mcast_ctas_b - cutlass.pipeline.Agent.Thread, - len(self.compute_warp_ids) * 1, - ) - pipeline_LSE = cutlass.pipeline.PipelineTmaAsync.create( - barrier_storage=storage.LSE_mbar_ptr.data_ptr(), - num_stages=self.Q_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group_compute, - tx_count=self.tma_copy_bytes["LSE"], - # cta_layout_vmnk=cluster_layout_vmnk, - # init_wait=False, - ) - pipeline_dPsum = cutlass.pipeline.PipelineTmaAsync.create( - barrier_storage=storage.dPsum_mbar_ptr.data_ptr(), - num_stages=self.dO_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group_compute, - tx_count=self.tma_copy_bytes["dPsum"], - # cta_layout_vmnk=cluster_layout_vmnk, - # init_wait=False, - ) - pipeline_Q = pipeline.PipelineTmaUmma.create( - barrier_storage=storage.Q_mbar_ptr.data_ptr(), - num_stages=self.Q_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group, - tx_count=self.tma_copy_bytes["Q"], - cta_layout_vmnk=cluster_layout_vmnk, - init_wait=False, - ) - pipeline_dO = pipeline.PipelineTmaUmma.create( - barrier_storage=storage.dO_mbar_ptr.data_ptr(), - num_stages=self.dO_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group, - tx_count=self.tma_copy_bytes["dO"], - cta_layout_vmnk=cluster_layout_vmnk, - init_wait=True, - ) - - sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner, dtype=self.q_dtype) - sQt = cute.make_tensor( - cute.recast_ptr(sQ.iterator, sQt_layout.inner, dtype=self.q_dtype), sQt_layout.outer - ) - sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner) - sKt = cute.make_tensor(cute.recast_ptr(sK.iterator, sKt_layout.inner), sKt_layout.outer) - sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner) - sdSt = storage.sdS.get_tensor(sdSt_layout.outer, swizzle=sdSt_layout.inner) - sdS = cute.make_tensor(cute.recast_ptr(sdSt.iterator, sdS_layout.inner), sdS_layout.outer) - sdO = storage.sdO.get_tensor( - sdO_layout.outer, swizzle=sdO_layout.inner, dtype=self.do_dtype - ) - sdOt = cute.make_tensor( - cute.recast_ptr(sdO.iterator, sdOt_layout.inner, dtype=self.do_dtype), sdOt_layout.outer - ) - sLSE = storage.sLSE.get_tensor(sLSE_layout) - sdPsum = storage.sdPsum.get_tensor(sdPsum_layout) - if const_expr(not self.dKV_postprocess): - sdV = storage.sdO.get_tensor( - sdKV_layout.outer, swizzle=sdKV_layout.inner, dtype=self.dv_dtype - ) - sdK = storage.sQ.get_tensor( - sdKV_layout.outer, swizzle=sdKV_layout.inner, dtype=self.dk_dtype - ) - else: - sdV = storage.sdO.get_tensor(sdKV_layout, dtype=self.dv_dtype) - sdK = storage.sQ.get_tensor(sdKV_layout, dtype=self.dk_dtype) - - # Buffer sizing is guaranteed by max(...) in SharedStorage declarations - # for both sQ (reused as sdK) and sdO (reused as sdV) - - sdQaccum = storage.sdQaccum.get_tensor(sdQaccum_layout) - - # TMEM - # This is a fake tensor, by right need to retrieve tmem_ptr. But we know that we always - # request 512 columns of tmem, so we know that it starts at 0. - tmem_ptr = cute.make_ptr(Float32, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16) - # S - thr_mma_S = tiled_mma_S.get_slice(0) - Sacc_shape = thr_mma_S.partition_shape_C(self.mma_tiler_kq[:2]) # (M, N) - tStS = thr_mma_S.make_fragment_C(Sacc_shape) - # (MMA, MMA_M, MMA_N) - tStS = cute.make_tensor(tmem_ptr + self.tmem_S_offset, tStS.layout) - # dP - thr_mma_dP = tiled_mma_dP.get_slice(0) - dPacc_shape = thr_mma_dP.partition_shape_C(self.mma_tiler_vdo[:2]) - tdPtdP = thr_mma_dP.make_fragment_C(dPacc_shape) - tdPtdP = cute.make_tensor(tmem_ptr + self.tmem_dP_offset, tdPtdP.layout) - # dV - thr_mma_dV = tiled_mma_dV.get_slice(0) - dvacc_shape = thr_mma_dV.partition_shape_C(self.mma_tiler_pdo[:2]) - tdVtdV = thr_mma_dV.make_fragment_C(dvacc_shape) - tdVtdV = cute.make_tensor(tmem_ptr + self.tmem_dV_offset, tdVtdV.layout) - tP = cute.make_tensor( - cute.recast_ptr(tmem_ptr + self.tmem_P_offset, dtype=self.do_dtype), tP_layout.outer - ) - # dK - thr_mma_dK = tiled_mma_dK.get_slice(0) - dkacc_shape = thr_mma_dK.partition_shape_C(self.mma_tiler_dsq[:2]) - tdKtdK = thr_mma_dK.make_fragment_C(dkacc_shape) - tdKtdK = cute.make_tensor(tmem_ptr + self.tmem_dK_offset, tdKtdK.layout) - tdS = cute.make_tensor( - cute.recast_ptr(tmem_ptr + self.tmem_dS_offset, dtype=self.ds_dtype), tdS_layout.outer - ) - # dQ - thr_mma_dQ = tiled_mma_dQ.get_slice(0) - dQacc_shape = thr_mma_dQ.partition_shape_C(self.mma_tiler_dsk[:2]) - tdQtdQ = thr_mma_dQ.make_fragment_C(dQacc_shape) - tdQtdQ = cute.make_tensor(tmem_ptr + self.tmem_dQ_offset, tdQtdQ.layout) - - block_info = BlockInfo( - self.tile_m, - # self.tile_n, - self.tile_n * self.cluster_shape_mnk[0], # careful, this case is not very well-tested - self.is_causal, - self.is_local, - False, # is_split_kv - window_size_left, - window_size_right, - qhead_per_kvhead_packgqa=1, - ) - SeqlenInfoCls = partial( - SeqlenInfoQK.create, - seqlen_q_static=mQ.shape[0], - seqlen_k_static=mK.shape[0], - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=mCuSeqlensK, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=mSeqUsedK, - tile_m=self.tile_m, - tile_n=self.tile_n, - ) - TileSchedulerCls = partial(self.tile_scheduler_cls.create, tile_sched_params) - - AttentionMaskCls = partial( - AttentionMask, - self.tile_m, - self.tile_n, - swap_AB=True, - window_size_left=window_size_left, - window_size_right=window_size_right, - ) - - # EMPTY - # (15) - if warp_idx == self.empty_warp_id: - cute.arch.warpgroup_reg_dealloc(self.num_regs_empty) - - # EPI - # (14) - if warp_idx == self.epi_warp_id: - # currently no-op, could use for tma store/reduce - cute.arch.warpgroup_reg_dealloc(self.num_regs_empty) - - # LOAD - # (13) - if warp_idx == self.load_warp_id: - cute.arch.warpgroup_reg_dealloc(self.num_regs_other) - self.load( - thr_mma_S, - thr_mma_dP, - thr_mma_dV, - mQ, - mK, - mV, - mLSE, - mdPsum, - mdO, - sQ, - sK, - sV, - sLSE, - sdPsum, - sdO, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_dO, - pipeline_Q, - pipeline_dO, - pipeline_LSE, - pipeline_dPsum, - cluster_layout_vmnk, - block_info, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - should_load_Q=True, - should_load_dO=True, - ) - - # MMA - # (12) - if warp_idx == self.mma_warp_id: - cute.arch.warpgroup_reg_dealloc(self.num_regs_other) - - # Alloc tmem buffer - tmem_alloc_cols = Int32(self.tmem_alloc_cols) - cute.arch.alloc_tmem(tmem_alloc_cols, storage.tmem_holding_buf) - cute.arch.sync_warp() - - self.mma( - tiled_mma_S, - tiled_mma_dP, - tiled_mma_dV, - tiled_mma_dK, - tiled_mma_dQ, - sQ, - sQt, - sK, - sV, - sdO, - sdOt, - sdSt, - sdS, - sKt, - tP, - tdS, - tStS, - tdPtdP, - tdVtdV, - tdKtdK, - tdQtdQ, - pipeline_Q.make_consumer(), - pipeline_dO, - pipeline_S_P, - pipeline_dS, - pipeline_dKV, - pipeline_dP, - pipeline_dQ, - block_info, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - ) - cute.arch.relinquish_tmem_alloc_permit() - tmem_ptr = cute.arch.retrieve_tmem_ptr( - Float32, alignment=16, ptr_to_buffer_holding_addr=storage.tmem_holding_buf - ) - - cute.arch.mbarrier_wait(tmem_dealloc_mbar_ptr, 0) - tmem_alloc_cols = Int32(self.tmem_alloc_cols) - cute.arch.dealloc_tmem(tmem_ptr, tmem_alloc_cols, is_two_cta=False) - - # Compute - # (4, 5, 6, 7, 8, 9, 10, 11) --> 8 warps - if warp_idx >= self.compute_warp_ids[0] and warp_idx <= self.compute_warp_ids[-1]: - cute.arch.warpgroup_reg_alloc(self.num_regs_compute) # 8 warps - self.compute_loop( - thr_mma_S, - thr_mma_dP, - thr_mma_dV, - thr_mma_dK, - tStS, - sLSE, - sdPsum, - tdVtdV, - tdKtdK, - mdV, - mdK, - sdS, - tdPtdP, - pipeline_LSE, - pipeline_dPsum, - pipeline_S_P, - pipeline_dS, - pipeline_dKV, - pipeline_dP, - softmax_scale, - softmax_scale_log2, - block_info, - SeqlenInfoCls, - AttentionMaskCls, - TileSchedulerCls, - sdV, - sdK, - mdV_tma_tensor, - mdK_tma_tensor, - tma_atom_dV, - tma_atom_dK, - tiled_copy_r2s_dKV, - mdK_semaphore, - mdV_semaphore, - aux_tensors, - fastdiv_mods, - blocksparse_tensors, - ) - cute.arch.mbarrier_arrive(tmem_dealloc_mbar_ptr) - - # Reduce - # (0, 1, 2, 3) - dQ - if warp_idx >= self.reduce_warp_ids[0] and warp_idx <= self.reduce_warp_ids[-1]: - cute.arch.warpgroup_reg_alloc(self.num_regs_reduce) - self.dQacc_reduce( - mdQaccum, - sdQaccum, - thr_mma_dQ, - tdQtdQ, - pipeline_dQ, - block_info, - SeqlenInfoCls, - TileSchedulerCls, - mdQ_semaphore, - blocksparse_tensors, - ) - - return - - @cute.jit - def load( - self, - thr_mma_S: cute.core.ThrMma, - thr_mma_dP: cute.core.ThrMma, - thr_mma_dV: cute.core.ThrMma, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdO: cute.Tensor, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - sLSE: cute.Tensor, - sdPsum: cute.Tensor, - sdO: cute.Tensor, - tma_atom_Q: cute.CopyAtom, - tma_atom_K: cute.CopyAtom, - tma_atom_V: cute.CopyAtom, - tma_atom_dO: cute.CopyAtom, - pipeline_Q: PipelineAsync, - pipeline_dO: PipelineAsync, - pipeline_LSE: PipelineAsync, - pipeline_dPsum: PipelineAsync, - cluster_layout_vmnk: cute.Layout, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - should_load_Q: bool = True, - should_load_dO: bool = True, - ): - producer_state_Q_LSE = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.Q_stage - ) - producer_state_dO_dPsum = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.dO_stage - ) - - # Compute multicast mask for Q & dO buffer full - cta_rank_in_cluster = cute.arch.make_warp_uniform(cute.arch.block_idx_in_cluster()) - block_in_cluster_coord_vmnk = cluster_layout_vmnk.get_flat_coord(cta_rank_in_cluster) - q_do_mcast_mask = None - if const_expr(self.is_q_do_mcast): - q_do_mcast_mask = cpasync.create_tma_multicast_mask( - cluster_layout_vmnk, block_in_cluster_coord_vmnk, mcast_mode=1 - ) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - m_block_min, m_block_max = block_info.get_m_block_min_max( - seqlen, n_block // self.cluster_shape_mnk[0] - ) - head_idx_kv = head_idx // self.qhead_per_kvhead - mQ_cur = seqlen.offset_batch_Q(mQ, batch_idx, dim=3)[None, None, head_idx] - mK_cur = seqlen.offset_batch_K(mK, batch_idx, dim=3)[None, None, head_idx_kv] - mV_cur = seqlen.offset_batch_K(mV, batch_idx, dim=3)[None, None, head_idx_kv] - if const_expr(not seqlen.has_cu_seqlens_q): - mdO_cur = mdO[None, None, head_idx, batch_idx] - else: - mdO_cur = cute.domain_offset((0, seqlen.offset_q), mdO[None, None, head_idx]) - mLSE_cur = seqlen.offset_batch_Q(mLSE, batch_idx, dim=2, padded=True)[None, head_idx] - mdPsum_cur = seqlen.offset_batch_Q(mdPsum, batch_idx, dim=2, padded=True)[ - None, head_idx - ] - - gK = cute.local_tile(mK_cur, cute.select(self.mma_tiler_kq, mode=[0, 2]), (n_block, 0)) - tSgK = thr_mma_S.partition_A(gK) - gV = cute.local_tile(mV_cur, cute.select(self.mma_tiler_vdo, mode=[0, 2]), (n_block, 0)) - tdPgV = thr_mma_dP.partition_A(gV) - gQ = cute.local_tile(mQ_cur, cute.select(self.mma_tiler_kq, mode=[1, 2]), (None, 0)) - tSgQ = thr_mma_S.partition_B(gQ) - gLSE = cute.local_tile(mLSE_cur, (self.tile_m,), (None,)) - gdPsum = cute.local_tile(mdPsum_cur, (self.tile_m,), (None,)) - gdO = cute.local_tile(mdO_cur, cute.select(self.mma_tiler_pdo, mode=[1, 2]), (0, None)) - tdPgdO = thr_mma_dV.partition_B(gdO) - - load_K, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_K, 0, cute.make_layout(1), tSgK, sK, single_stage=True - ) - load_V, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_V, - 0, - cute.make_layout(1), - tdPgV, - sV, - single_stage=True, - ) - b_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, None, 0, 0)).shape) - load_Q, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_Q, - cta_coord=block_in_cluster_coord_vmnk[1], - cta_layout=b_cta_layout, - src_tensor=tSgQ, - dst_tensor=sQ, - mcast_mask=q_do_mcast_mask, - ) - load_Q = copy_utils.tma_producer_copy_fn(load_Q, pipeline_Q) - load_dO, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_dO, - cta_coord=block_in_cluster_coord_vmnk[1], - cta_layout=b_cta_layout, - src_tensor=tdPgdO, - dst_tensor=sdO, - mcast_mask=q_do_mcast_mask, - ) - load_dO = copy_utils.tma_producer_copy_fn(load_dO, pipeline_dO) - copy_atom_stats = cute.make_copy_atom(cpasync.CopyBulkG2SOp(), Float32) - copy_stats = partial(cute.copy, copy_atom_stats) - # copy_atom_stats = cute.make_copy_atom(cpasync.CopyBulkG2SMulticastOp(), Float32) - # sLSE = cute.logical_divide(sLSE, (64,))[(None, block_in_cluster_coord_vmnk[1]), None] - # gLSE = cute.logical_divide(gLSE, (64,))[(None, block_in_cluster_coord_vmnk[1]), None] - # sdPsum = cute.logical_divide(sdPsum, (64,))[(None, block_in_cluster_coord_vmnk[1]), None] - # gdPsum = cute.logical_divide(gdPsum, (64,))[(None, block_in_cluster_coord_vmnk[1]), None] - # copy_stats = partial(cute.copy, copy_atom_stats, mcast_mask=q_do_mcast_mask) - - # some tiles might be empty due to block sparsity - if const_expr(self.use_block_sparsity): - total_m_block_cnt = get_total_q_block_count_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = total_m_block_cnt > Int32(0) - else: - process_tile = ( - const_expr(not self.is_local and not self.is_varlen_q) - or m_block_min < m_block_max - ) - - if process_tile: - if const_expr(self.use_block_sparsity): - producer_state_Q_LSE, producer_state_dO_dPsum = ( - produce_block_sparse_q_loads_bwd_sm100( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - producer_state_Q_LSE, - producer_state_dO_dPsum, - pipeline_Q, - pipeline_LSE, - pipeline_dO, - pipeline_dPsum, - load_K, - load_V, - load_Q, - load_dO, - copy_stats, - gLSE, - sLSE, - gdPsum, - sdPsum, - self.tma_copy_bytes["K"], - self.tma_copy_bytes["V"], - should_load_Q=should_load_Q, - should_load_dO=should_load_dO, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - ) - else: - first_m_block = m_block_min - - # First iteration: load K together w Q & LSE, then V together w dO & dPsum - if const_expr(should_load_Q): - pipeline_Q.producer_acquire( - producer_state_Q_LSE, extra_tx_count=self.tma_copy_bytes["K"] - ) - load_K(tma_bar_ptr=pipeline_Q.producer_get_barrier(producer_state_Q_LSE)) - load_Q(first_m_block, producer_state=producer_state_Q_LSE) - pipeline_Q.producer_commit(producer_state_Q_LSE) - pipeline_LSE.producer_acquire(producer_state_Q_LSE) - with cute.arch.elect_one(): - copy_stats( - gLSE[None, first_m_block], - sLSE[None, producer_state_Q_LSE.index], - mbar_ptr=pipeline_LSE.producer_get_barrier(producer_state_Q_LSE), - ) - producer_state_Q_LSE.advance() - if const_expr(should_load_dO): - pipeline_dO.producer_acquire( - producer_state_dO_dPsum, extra_tx_count=self.tma_copy_bytes["V"] - ) - load_V( - tma_bar_ptr=pipeline_dO.producer_get_barrier(producer_state_dO_dPsum) - ) - load_dO(first_m_block, producer_state=producer_state_dO_dPsum) - pipeline_dO.producer_commit(producer_state_dO_dPsum) - pipeline_dPsum.producer_acquire(producer_state_dO_dPsum) - with cute.arch.elect_one(): - copy_stats( - gdPsum[None, first_m_block], - sdPsum[None, producer_state_dO_dPsum.index], - mbar_ptr=pipeline_dPsum.producer_get_barrier( - producer_state_dO_dPsum - ), - ) - producer_state_dO_dPsum.advance() - - # Dense path: iterate from m_block_min+1 to m_block_max - for m_block in cutlass.range(m_block_min + 1, m_block_max, unroll=1): - if const_expr(should_load_Q): - pipeline_Q.producer_acquire(producer_state_Q_LSE) - load_Q(m_block, producer_state=producer_state_Q_LSE) - pipeline_Q.producer_commit(producer_state_Q_LSE) - pipeline_LSE.producer_acquire(producer_state_Q_LSE) - with cute.arch.elect_one(): - copy_stats( - gLSE[None, m_block], - sLSE[None, producer_state_Q_LSE.index], - mbar_ptr=pipeline_LSE.producer_get_barrier( - producer_state_Q_LSE - ), - ) - producer_state_Q_LSE.advance() - if const_expr(should_load_dO): - pipeline_dO.producer_acquire(producer_state_dO_dPsum) - load_dO(m_block, producer_state=producer_state_dO_dPsum) - pipeline_dO.producer_commit(producer_state_dO_dPsum) - pipeline_dPsum.producer_acquire(producer_state_dO_dPsum) - with cute.arch.elect_one(): - copy_stats( - gdPsum[None, m_block], - sdPsum[None, producer_state_dO_dPsum.index], - mbar_ptr=pipeline_dPsum.producer_get_barrier( - producer_state_dO_dPsum - ), - ) - producer_state_dO_dPsum.advance() - - if const_expr(should_load_Q): - pipeline_Q.producer_tail( - producer_state_Q_LSE.clone() - ) # will hang if we don't clone - pipeline_LSE.producer_tail(producer_state_Q_LSE) - if const_expr(should_load_dO): - pipeline_dO.producer_tail(producer_state_dO_dPsum.clone()) - pipeline_dPsum.producer_tail(producer_state_dO_dPsum) - - tile_scheduler.prefetch_next_work() - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def mma( - self, - tiled_mma_S: cute.TiledMma, - tiled_mma_dP: cute.TiledMma, - tiled_mma_dV: cute.TiledMma, - tiled_mma_dK: cute.TiledMma, - tiled_mma_dQ: cute.TiledMma, - sQ: cute.Tensor, - sQt: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - sdO: cute.Tensor, - sdOt: cute.Tensor, - sdSt: cute.Tensor, - sdS: cute.Tensor, - sKt: cute.Tensor, - tP: cute.Tensor, - tdS: cute.Tensor, - tStS: cute.Tensor, - tdPtdP: cute.Tensor, - tdVtdV: cute.Tensor, - tdKtdK: cute.Tensor, - tdQtdQ: cute.Tensor, - pipeline_Q_consumer: PipelineConsumer, - pipeline_dO: PipelineAsync, - pipeline_S_P: PipelineAsync, - pipeline_dS: PipelineAsync, - pipeline_dKV: PipelineAsync, - pipeline_dP: PipelineAsync, - pipeline_dQ: PipelineAsync, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - # [2025-10-21] For reasons I don't understand, putting these partitioning in the main - # kernel (before warp specialization) is a lot slower tha putting them here. - # Partition smem / tmem tensors - # S = K @ Q.T - tSrK = tiled_mma_S.make_fragment_A(sK) - tSrQ = tiled_mma_S.make_fragment_B(sQ) - # dP = V @ dO.T - tdPrV = tiled_mma_dP.make_fragment_A(sV) - tdPrdOt = tiled_mma_dP.make_fragment_B(sdOt) - # dK = dS.T @ Q - if const_expr(self.use_smem_dS_for_mma_dK): - tdKrdS = tiled_mma_dK.make_fragment_A(sdSt) - else: - tdKrdS = tiled_mma_dK.make_fragment_A(tdS) - tdKrQ = tiled_mma_dK.make_fragment_B(sQt) - # dQ = dS @ K - tdQrdS = tiled_mma_dQ.make_fragment_A(sdS) - tdQrK = tiled_mma_dQ.make_fragment_B(sKt) - # dV = P @ dO.T - tdVrdO = tiled_mma_dV.make_fragment_B(sdO) - tdVrP = tiled_mma_dV.make_fragment_A(tP) - - # mma_qk_fn = partial(gemm_w_idx, tiled_mma_S, tStS, tSrK, tSrQ, zero_init=True) - mma_qk_fn = partial( - gemm_ptx_w_idx, tiled_mma_S, tStS, tSrK, tSrQ, sA=sK, sB=sQ, zero_init=True - ) - # mma_dov_fn = partial(gemm_w_idx, tiled_mma_dP, tdPtdP, tdPrV, tdPrdOt, zero_init=True) - mma_dov_fn = partial( - gemm_ptx_w_idx, - tiled_mma_dP, - tdPtdP, - tdPrV, - tdPrdOt, - sA=sV, - sB=sdOt, - zero_init=True, - ) - # mma_pdo_fn = partial(gemm_w_idx, tiled_mma_dV, tdVtdV, tdVrP, tdVrdO) - mma_pdo_fn = partial( - gemm_ptx_w_idx, - tiled_mma_dV, - tdVtdV, - tdVrP, - tdVrdO, - sA=None, - sB=sdO, - tA_addr=self.tmem_P_offset, - ) - mma_dsk_fn = partial(gemm_w_idx, tiled_mma_dQ, tdQtdQ, tdQrdS, tdQrK, zero_init=True) - # mma_dsk_fn = partial( - # gemm_ptx_w_idx, tiled_mma_dQ, tdQtdQ, tdQrdS, tdQrK, sA=sdS, sB=sKt, zero_init=True - # ) - if const_expr(self.use_smem_dS_for_mma_dK): - mma_dsq_fn = partial(gemm_w_idx, tiled_mma_dK, tdKtdK, tdKrdS, tdKrQ) - else: - # Need to explicitly pass in tA_addr for correctness - mma_dsq_fn = partial( - gemm_ptx_w_idx, - tiled_mma_dK, - tdKtdK, - tdKrdS, - tdKrQ, - sA=None, - sB=sQt, - tA_addr=self.tmem_dS_offset, - ) - - consumer_state_dO = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.dO_stage - ) - producer_phase_acc = Int32(1) # For S & P, dP, dQ - consumer_state_dS = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, 1 - ) - # producer_state_dKV = cutlass.pipeline.make_pipeline_state( - # cutlass.pipeline.PipelineUserType.Producer, 2 - # ) - producer_phase_dKV = Int32(1) - cta_group = pipeline_S_P.cta_group - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) # must be seqlen_k - m_block_min, m_block_max = block_info.get_m_block_min_max( - seqlen, n_block // self.cluster_shape_mnk[0] - ) - - if const_expr(self.use_block_sparsity): - block_iter_count = get_total_q_block_count_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = block_iter_count > Int32(0) - else: - block_iter_count = m_block_max - m_block_min - process_tile = ( - const_expr(not self.is_local and not self.is_varlen_q) - or m_block_min < m_block_max - ) - - if process_tile: - accumulate_dK = False - # ----------------------------------------------------------- - ###### Prologue - # ----------------------------------------------------------- - # 1. S = Q0 @ K.T - # 2. dP = V @ dO.T - # 3. dV = P @ dO - # 1) S = Q0 @ K.T - handle_Q = pipeline_Q_consumer.wait_and_advance() - pipeline_S_P.sync_object_empty.wait(0, producer_phase_acc) - mma_qk_fn(B_idx=handle_Q.index) - # Don't release Q yet - pipeline_S_P.sync_object_full.arrive(0, pipeline_S_P.producer_mask, cta_group) - - # 2) dP = V @ dO.T - pipeline_dO.consumer_wait(consumer_state_dO) - pipeline_dP.sync_object_empty.wait(0, producer_phase_acc) - # dQ uses the same tmem as dP - pipeline_dQ.sync_object_empty.wait(0, producer_phase_acc) - mma_dov_fn(B_idx=consumer_state_dO.index) - # Don't release dO yet - pipeline_dP.sync_object_full.arrive(0, pipeline_dP.producer_mask, cta_group) - - producer_phase_acc ^= 1 - # 3) dV = P.T @ dO - # wait for P to be ready, which uses the same tmem as S - pipeline_S_P.sync_object_empty.wait(0, producer_phase_acc) - mma_pdo_fn(B_idx=consumer_state_dO.index, zero_init=True) - pipeline_dO.consumer_release(consumer_state_dO) - consumer_state_dO.advance() - # ----------------------------------------------------------- - ###### MAIN LOOP - # ----------------------------------------------------------- - # 1. S = K @ Q.T - # 2. dQ = dS @ K - # 3. dK = dS.T @ Q - # 4. dP = V @ dO.T - # 5. dV = P.T @ dO - - # For block sparsity, we use block_iter_count; for dense, use m_block range - # MMA doesn't need actual m_block indices, just the iteration count - main_loop_iters = ( - block_iter_count - 1 - if const_expr(self.use_block_sparsity) - else m_block_max - m_block_min - 1 - ) - for _ in cutlass.range(main_loop_iters, unroll=1): - # 1) S = K @ Q_i - handle_Q_next = pipeline_Q_consumer.wait_and_advance() - # Don't need to wait for S, as P must have been ready ealier, i.e., S is ready - mma_qk_fn(B_idx=handle_Q_next.index) - pipeline_S_P.sync_object_full.arrive(0, pipeline_S_P.producer_mask, cta_group) - - # 2-3) - # Do dK = dS.T @ Q, then dQ = dS @ K if dS in tmem for first mma - # Otherwise, reverse order - pipeline_dS.consumer_wait(consumer_state_dS) - - if const_expr(self.use_smem_dS_for_mma_dK): - mma_dsk_fn() - pipeline_dQ.sync_object_full.arrive(0, pipeline_dQ.producer_mask, cta_group) - mma_dsq_fn(B_idx=handle_Q.index, zero_init=not accumulate_dK) - accumulate_dK = True - handle_Q.release() - else: - mma_dsq_fn(B_idx=handle_Q.index, zero_init=not accumulate_dK) - accumulate_dK = True - handle_Q.release() - mma_dsk_fn() - pipeline_dQ.sync_object_full.arrive(0, pipeline_dQ.producer_mask, cta_group) - - # dP uses the same tmem as dQ - # However, if dS is ready, then dP must have been ready, - # so we don't need this wait before mma_dsk_fn() - # pipeline_dP.sync_object_empty.wait(0, producer_phase_acc) - - pipeline_dS.consumer_release(consumer_state_dS) - consumer_state_dS.advance() - - # 4) dP = V @ dO.T - pipeline_dO.consumer_wait(consumer_state_dO) - # dQ uses the same tmem as dP - pipeline_dQ.sync_object_empty.wait(0, producer_phase_acc) - mma_dov_fn(B_idx=consumer_state_dO.index) - pipeline_dP.sync_object_full.arrive(0, pipeline_dP.producer_mask, cta_group) - - producer_phase_acc ^= 1 - # 5) dV += P @ dO - # wait for P to be ready, which uses the same tmem as S - pipeline_S_P.sync_object_empty.wait(0, producer_phase_acc) - mma_pdo_fn(B_idx=consumer_state_dO.index, zero_init=False) - pipeline_dO.consumer_release(consumer_state_dO) - consumer_state_dO.advance() - - handle_Q = handle_Q_next - - pipeline_S_P.sync_object_full.arrive(0, pipeline_S_P.producer_mask, cta_group) - - # signal to the epilogue that dV is ready - # pipeline_dKV.producer_acquire(producer_state_dKV) - pipeline_dKV.sync_object_empty.wait(0, producer_phase_dKV) - # pipeline_dKV.producer_commit(producer_state_dKV) - pipeline_dKV.sync_object_full.arrive(0, pipeline_dKV.producer_mask, cta_group) - # producer_state_dKV.advance() - # pipeline_dKV.producer_acquire(producer_state_dKV) - pipeline_dKV.sync_object_empty.wait(1, producer_phase_dKV) - - # ----------------------------------------------------------- - ###### Remaining 2 - # ----------------------------------------------------------- - # 1) dK += dS.T @ Q - pipeline_dS.consumer_wait(consumer_state_dS) - mma_dsq_fn(B_idx=handle_Q.index, zero_init=not accumulate_dK) - # signal to the epilogue that dK is ready - # pipeline_dKV.producer_commit(producer_state_dKV) - pipeline_dKV.sync_object_full.arrive(1, pipeline_dKV.producer_mask, cta_group) - # producer_state_dKV.advance() - producer_phase_dKV ^= 1 - - # 2) dQ = dS @ K - # dS is done, so dP must have been ready, we don't need to wait - mma_dsk_fn() - pipeline_dQ.sync_object_full.arrive(0, pipeline_dQ.producer_mask, cta_group) - # Wait until dQ is done before releasing Q, since K and Q0 uses the same mbarrier - handle_Q.release() - pipeline_dS.consumer_release(consumer_state_dS) - consumer_state_dS.advance() - - producer_phase_acc ^= 1 - - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - # Currently it hangs if we have this S_P.producer_tail, will need to understand why - # pipeline_S_P.producer_tail(producer_state_S_P) - # pipeline_dP.producer_tail(producer_state_dP) - # pipeline_dKV.producer_tail(producer_state_dKV) - # pipeline_dQ.producer_tail(producer_state_dQ) - - @cute.jit - def split_wg( - self, - t: cute.Tensor, - wg_idx: cutlass.Int32, - num_wg: cutlass.Constexpr[int], - ): - reduced_shape = cute.product_each(t.shape) - rank = len(reduced_shape) - if const_expr(reduced_shape[1] > 1): - assert rank >= 2, "Need rank >= 2 for t in split_wg" - t = cute.logical_divide(t, (reduced_shape[0], reduced_shape[1] // num_wg)) - coord = (None, (None, wg_idx)) + (None,) * (rank - 2) - else: - assert rank >= 3, "Need rank >= 3 for t in split_wg" - if const_expr(rank == 3): - t = cute.logical_divide( - t, (reduced_shape[0], reduced_shape[1], reduced_shape[2] // num_wg) - ) - coord = ( - None, - None, - (None, wg_idx), - ) + (None,) * (rank - 3) - else: - t = cute.logical_divide( - t, - ( - reduced_shape[0], - reduced_shape[1], - reduced_shape[2], - reduced_shape[3] // num_wg, - ), - ) - coord = ( - None, - None, - None, - (None, wg_idx), - ) + (None,) * (rank - 4) - return t[coord] - - @cute.jit - def apply_score_mod( - self, - tSrS_t2r, - thr_copy_t2r, - thr_mma_S, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen_info, - aux_tensors=None, - fastdiv_mods=(None, None), - ): - """Apply forward score modification for SM100 backward pass.""" - # In bwd, S is computed as K @ Q.T so dimensions are (tile_n, tile_m) - cS = cute.make_identity_tensor((self.tile_n, self.tile_m)) - cS = cute.domain_offset((n_block * self.tile_n, m_block * self.tile_m), cS) - tScS = thr_mma_S.partition_C(cS) - tScS_idx = thr_copy_t2r.partition_D(tScS) - - apply_score_mod_inner( - tSrS_t2r, - tScS_idx, - self.score_mod, - batch_idx, - head_idx, - softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info, - constant_q_idx=None, - qhead_per_kvhead=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - transpose_indices=True, - ) - - @cute.jit - def apply_score_mod_bwd( - self, - grad_tensor, - score_tensor, - index_tensor, - batch_idx, - head_idx, - softmax_scale, - seqlen_info, - aux_tensors=None, - fastdiv_mods=(None, None), - ): - """Apply backward score modification (joint graph) for SM100.""" - apply_score_mod_bwd_inner( - grad_tensor, - score_tensor, - index_tensor, - self.score_mod_bwd, - batch_idx, - head_idx, - softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info, - constant_q_idx=None, - qhead_per_kvhead=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - transpose_indices=True, - ) - - @cute.jit - def compute_loop( - self, - thr_mma_S: cute.core.ThrMma, - thr_mma_dP: cute.core.ThrMma, - thr_mma_dV: cute.core.ThrMma, - thr_mma_dK: cute.core.ThrMma, - tStS: cute.Tensor, - sLSE: cute.Tensor, - sdPsum: cute.Tensor, - tdVtdV: cute.Tensor, - tdKtdK: cute.Tensor, - mdV: cute.Tensor, - mdK: cute.Tensor, - sdS: cute.Tensor, - tdPtdP: cute.Tensor, - pipeline_LSE: PipelineAsync, - pipeline_dPsum: PipelineAsync, - pipeline_S_P: PipelineAsync, - pipeline_dS: PipelineAsync, - pipeline_dKV: PipelineAsync, - pipeline_dP: PipelineAsync, - softmax_scale: cutlass.Float32, - softmax_scale_log2: cutlass.Float32, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - AttentionMaskCls: Callable, - TileSchedulerCls: Callable, - sdV: Optional[cute.Tensor], - sdK: Optional[cute.Tensor], - mdV_tma_tensor: Optional[cute.Tensor], - mdK_tma_tensor: Optional[cute.Tensor], - tma_atom_dV: Optional[cute.CopyAtom], - tma_atom_dK: Optional[cute.CopyAtom], - tiled_copy_r2s_dKV: Optional[cute.TiledCopy], - mdK_semaphore: Optional[cute.Tensor], - mdV_semaphore: Optional[cute.Tensor], - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - sLSE_2D = cute.make_tensor( - sLSE.iterator, - cute.make_layout( - (self.tile_m, self.tile_n, self.Q_stage), - stride=(1, 0, cute.round_up(self.tile_m, 64)), - ), - ) - sdPsum_2D = cute.make_tensor( - sdPsum.iterator, - cute.make_layout( - (self.tile_m, self.tile_n, self.dO_stage), - stride=(1, 0, cute.round_up(self.tile_m, 64)), - ), - ) - # if const_expr(self.SdP_swapAB): - if const_expr(True): - sLSE_2D = utils.transpose_view(sLSE_2D) - sdPsum_2D = utils.transpose_view(sdPsum_2D) - - # tix: [128...384] 8 warps - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) # 4-11 - tidx = cute.arch.thread_idx()[0] % (cute.arch.WARP_SIZE * len(self.compute_warp_ids)) - # tidx = cute.arch.thread_idx()[0] - (cute.arch.WARP_SIZE * self.compute_warp_ids[0]) - dp_idx = tidx % 128 - num_wg = len(self.compute_warp_ids) // 4 # 2 - # wg_idx: - # 0: [256...384] - # 1: [128...256] - - tileP_f32_like = self.mma_tiler_kq[0] // 32 * self.v_dtype.width # 64 for tile_n = 128 - # tStS has shape ((128, 128), 1, 1), tStP has shape ((128, 64), 1, 1) - # tP overlap with tS - tStP = cute.composition(tStS, (cute.make_layout((self.tile_n, tileP_f32_like)), 1, 1)) - tStP = cute.make_tensor(tStS.iterator, tStP.layout) # Otherwise the tmem address is wrong - tScS = thr_mma_S.partition_C(cute.make_identity_tensor(self.mma_tiler_kq[:2])) - tScP = cute.composition(tScS, (cute.make_layout((self.tile_n, tileP_f32_like)), 1, 1)) - # tdS overlap with tdP - tdPtdS = cute.composition(tdPtdP, (cute.make_layout((self.tile_n, tileP_f32_like)), 1, 1)) - tdPcdP = thr_mma_dP.partition_C(cute.make_identity_tensor(self.mma_tiler_vdo[:2])) - tdPcdS = cute.composition(tdPcdP, (cute.make_layout((self.tile_n, tileP_f32_like)), 1, 1)) - - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), Float32 - ) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16)), Float32 - ) - - # tmem -> rmem - thr_copy_t2r = copy_utils.make_tmem_copy(tmem_load_atom, num_wg).get_slice(tidx) - tStS_t2r = thr_copy_t2r.partition_S(tStS) # (((32, 32), 1), 2, 1, 1) - tdPtdP_t2r = thr_copy_t2r.partition_S(tdPtdP) - tScS_t2r = thr_copy_t2r.partition_D(tScS) # ((32, 1), 2, 1, 1) - t0ScS_t2r = thr_copy_t2r.get_slice(0).partition_D(tScS) # ((32, 1), 2, 1, 1) - # ((32, 1), 2, 1, 1, STAGE) - tSsLSE = thr_copy_t2r.partition_D(thr_mma_S.partition_C(sLSE_2D)) - tSsdPsum = thr_copy_t2r.partition_D(thr_mma_dP.partition_C(sdPsum_2D)) - # rmem -> tmem - thr_copy_r2t = copy_utils.make_tmem_copy(tmem_store_atom, num_wg).get_slice(tidx) - tScP_r2t = thr_copy_r2t.partition_S(tScP) - tStP_r2t = thr_copy_r2t.partition_D(tStP) - tdPcdS_r2t = thr_copy_r2t.partition_S(tdPcdS) - tdPtdS_r2t = thr_copy_r2t.partition_D(tdPtdS) - # rmem -> smem - # This part is a bit iffy, we might be making a lot of assumptions here - copy_atom_r2s = sm100_utils_basic.get_smem_store_op( - LayoutEnum.ROW_MAJOR, self.ds_dtype, Float32, thr_copy_t2r - ) - thr_copy_r2s = cute.make_tiled_copy_D(copy_atom_r2s, thr_copy_t2r).get_slice(tidx) - # We assume the swizzle (i.e. layout.inner) stays the same - sdS_layout = sm100_utils_basic.make_smem_layout_epi( - self.ds_dtype, LayoutEnum.ROW_MAJOR, (self.tile_n, self.tile_m), 1 - ).outer # ((8,16), (64,2), (1, 1)) - sdS_layout = cute.slice_(sdS_layout, (None, None, 0)) # ((8,16), (64,2)) - # Need to group into 1 mode to be compatible w thr_copy_r2s - sdS_layout = cute.make_layout((sdS_layout.shape,), stride=(sdS_layout.stride,)) - sdS_epi = cute.make_tensor(sdS.iterator, sdS_layout) - tRS_sdS = thr_copy_r2s.partition_D(sdS_epi) - - consumer_state_S_P_dP = pipeline.make_pipeline_state( # Our impl has shortcut for stage==1 - cutlass.pipeline.PipelineUserType.Consumer, 1 - ) - # consumer_phase_S_P_dP = Int32(0) - producer_state_dS = pipeline.make_pipeline_state( # Our impl has shortcut for stage==1 - cutlass.pipeline.PipelineUserType.Producer, 1 - ) - consumer_state_dKV = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, 2 - ) - consumer_state_LSE = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.Q_stage - ) - # consumer_state_dPsum = cutlass.pipeline.make_pipeline_state( - consumer_state_dPsum = pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.dO_stage - ) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - m_block_min, m_block_max = block_info.get_m_block_min_max( - seqlen, n_block // self.cluster_shape_mnk[0] - ) - mask = AttentionMaskCls(seqlen) - # TODO: condition mask_seqlen - mask_fn = partial( - mask.apply_mask_sm100_transposed, - tScS_t2r=tScS_t2r, - t0ScS_t2r=t0ScS_t2r, - n_block=n_block, - mask_seqlen=True, - mask_causal=self.is_causal, - mask_local=self.is_local, - mask_mod=self.mask_mod, - batch_idx=batch_idx, - head_idx=head_idx, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - # prefetch_LSE = not self.is_causal - prefetch_LSE = False - - # some tiles might be empty due to block sparsity - if const_expr(self.use_block_sparsity): - ( - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - loop_count, - ) = get_block_sparse_iteration_info_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = loop_count > Int32(0) - else: - process_tile = ( - const_expr(not self.is_local and not self.is_varlen_q) - or m_block_min < m_block_max - ) - loop_count = m_block_max - m_block_min - - # Mainloop - # Block sparsity: iterate over sparse m_block count and derive actual m_block - # from Q_IDX/FULL_Q_IDX tensors. Dense: iterate m_block_min..m_block_max directly. - for iter_idx in cutlass.range(loop_count, unroll=1): - if const_expr(self.use_block_sparsity): - m_block, is_full_block = get_m_block_from_iter_bwd( - iter_idx, - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - m_block_oob = m_block >= m_block_max - else: - m_block = m_block_min + iter_idx - m_block_oob = False - is_full_block = False - # Prefetch 1 stage of LSE - pipeline_LSE.consumer_wait(consumer_state_LSE) - tSrLSE_s2r = cute.make_fragment(tScS_t2r[None, 0, 0, 0].shape, Float32) - if const_expr(prefetch_LSE and not self.shuffle_LSE): - cute.autovec_copy(tSsLSE[None, 0, 0, 0, consumer_state_LSE.index], tSrLSE_s2r) - - pipeline_S_P.consumer_wait(consumer_state_S_P_dP) - # pipeline_S_P.sync_object_full.wait(0, consumer_phase_S_P_dP) - #### TMEM->RMEM (Load S from TMEM) - tSrS_t2r = cute.make_fragment(tScS_t2r.shape, Float32) - cute.copy(thr_copy_t2r, tStS_t2r, tSrS_t2r) - if const_expr(self.score_mod_bwd is not None): - tSrS_pre = cute.make_fragment_like(tSrS_t2r) - cute.autovec_copy(tSrS_t2r, tSrS_pre) - - if const_expr(self.score_mod is not None): - # Apply score_mod FIRST -> matches forward - self.apply_score_mod( - tSrS_t2r, - thr_copy_t2r, - thr_mma_S, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen, - aux_tensors, - fastdiv_mods, - ) - - #### APPLY MASK (after score_mod, matching forward pass order) - check_m_boundary = (m_block + 1) * self.tile_m > seqlen.seqlen_q - mask_fn( - tSrS_t2r, - m_block=m_block, - is_full_block=is_full_block, - check_m_boundary=check_m_boundary, - ) - - num_stages = cute.size(tScS_t2r, mode=[1]) - - # --------------------------------------------- - #### P = exp(S - LSE) - # --------------------------------------------- - lane_idx = cute.arch.lane_idx() - tSrP_r2t_f32 = cute.make_fragment(tScP_r2t.shape, Float32) # 64 - tSrP_r2t = cute.recast_tensor(tSrP_r2t_f32, self.q_dtype) - for stage in cutlass.range_constexpr(num_stages): - tSrS_cur = tSrS_t2r[None, stage, 0, 0] - tSsLSE_cur = tSsLSE[None, stage, 0, 0, consumer_state_LSE.index] - if const_expr(not self.shuffle_LSE): - if const_expr(stage > 0 or not prefetch_LSE): - cute.autovec_copy(tSsLSE_cur, tSrLSE_s2r) - tSrLSE = tSrLSE_s2r - else: - tSrLSE = tSsLSE_cur[lane_idx] - for v in cutlass.range_constexpr(cute.size(tSrS_t2r, mode=[0]) // 2): - if const_expr(not self.shuffle_LSE): - lse_pair = (tSrLSE[2 * v], tSrLSE[2 * v + 1]) - else: - lse_pair = ( - utils.shuffle_sync(tSrLSE, offset=2 * v), - utils.shuffle_sync(tSrLSE, offset=2 * v + 1), - ) - tSrS_cur[2 * v], tSrS_cur[2 * v + 1] = utils.fma_packed_f32x2( - ((tSrS_cur[2 * v], tSrS_cur[2 * v + 1])), - (softmax_scale_log2, softmax_scale_log2), - (-lse_pair[0], -lse_pair[1]), - ) - tSrS_cur[2 * v] = cute.math.exp2(tSrS_cur[2 * v], fastmath=True) - tSrS_cur[2 * v + 1] = cute.math.exp2(tSrS_cur[2 * v + 1], fastmath=True) - utils.cvt_f16(tSrS_cur, tSrP_r2t[None, stage, 0, 0]) - if const_expr(stage == 0): - cute.arch.fence_view_async_tmem_load() - # Without this barrier, we could have 1 warp writing to P in tmem while - # another warp is still reading S from tmem. - self.compute_sync_barrier.arrive_and_wait() - cute.copy( - thr_copy_r2t, - tSrP_r2t_f32[None, stage, None, None], - tStP_r2t[None, stage, None, None], - ) - - cute.arch.fence_view_async_tmem_store() - self.compute_sync_barrier.arrive_and_wait() - - with cute.arch.elect_one(): - pipeline_S_P.consumer_release(consumer_state_S_P_dP) - # pipeline_S_P.sync_object_empty.arrive(0, pipeline_S_P.consumer_mask) - pipeline_LSE.consumer_release(consumer_state_LSE) - # consumer_state_S_P_dP.advance() - consumer_state_LSE.advance() - - # --------------------------------------------- - # dS.T = P.T * (dP.T - D) - # --------------------------------------------- - pipeline_dPsum.consumer_wait(consumer_state_dPsum) - - pipeline_dP.consumer_wait(consumer_state_S_P_dP) - # pipeline_dP.sync_object_full.wait(0, consumer_phase_S_P_dP) - consumer_state_S_P_dP.advance() - # consumer_phase_S_P_dP ^= 1 - - ##### dS.T = P.T * (dP.T - Psum) - for stage in cutlass.range_constexpr(num_stages): - tdPrdP_t2r = cute.make_fragment(tScS_t2r[None, 0, None, None].shape, Float32) - cute.copy(thr_copy_t2r, tdPtdP_t2r[None, stage, None, None], tdPrdP_t2r) - cute.arch.fence_view_async_tmem_load() - self.compute_sync_barrier.arrive_and_wait() - tdPrdP_cur = tdPrdP_t2r[None, 0, 0] - tSrS_cur = tSrS_t2r[None, stage, 0, 0] - tSsdPsum_cur = tSsdPsum[None, stage, 0, 0, consumer_state_dPsum.index] - if const_expr(not self.shuffle_dPsum): - tSrdPsum = cute.make_fragment_like(tSsdPsum_cur, Float32) - cute.autovec_copy(tSsdPsum_cur, tSrdPsum) - else: - tSrdPsum = tSsdPsum_cur[lane_idx] - for v in cutlass.range_constexpr(cute.size(tdPrdP_t2r, mode=[0]) // 2): - if const_expr(not self.shuffle_dPsum): - dPsum_pair = (tSrdPsum[2 * v], tSrdPsum[2 * v + 1]) - else: - dPsum_pair = ( - utils.shuffle_sync(tSrdPsum, offset=2 * v), - utils.shuffle_sync(tSrdPsum, offset=2 * v + 1), - ) - tdPrdP_cur[2 * v], tdPrdP_cur[2 * v + 1] = utils.sub_packed_f32x2( - (tdPrdP_cur[2 * v], tdPrdP_cur[2 * v + 1]), dPsum_pair - ) - tdPrdP_cur[2 * v], tdPrdP_cur[2 * v + 1] = utils.mul_packed_f32x2( - (tSrS_cur[2 * v], tSrS_cur[2 * v + 1]), - (tdPrdP_cur[2 * v], tdPrdP_cur[2 * v + 1]), - ) - - if const_expr(self.score_mod_bwd is not None): - tSrS_pre_cur = tSrS_pre[None, stage, 0, 0] - cS_bwd = cute.make_identity_tensor((self.tile_n, self.tile_m)) - cS_bwd = cute.domain_offset( - (n_block * self.tile_n, m_block * self.tile_m), cS_bwd - ) - tScS_bwd = thr_mma_S.partition_C(cS_bwd) - tScS_idx_bwd = thr_copy_t2r.partition_D(tScS_bwd) - tScS_idx_cur = tScS_idx_bwd[None, stage, 0, 0] - self.apply_score_mod_bwd( - tdPrdP_cur, - tSrS_pre_cur, - tScS_idx_cur, - batch_idx, - head_idx, - softmax_scale, - seqlen, - aux_tensors, - fastdiv_mods, - ) - # Zero out OOB positions (kv_idx >= seqlen_k) after score_mod_bwd - for i in cutlass.range(cute.size(tdPrdP_cur), unroll_full=True): - kv_idx = tScS_idx_cur[i][0] - tdPrdP_cur[i] = 0.0 if kv_idx >= seqlen.seqlen_k else tdPrdP_cur[i] - - tdPrdS_cvt = cute.make_fragment_like(tdPrdP_cur, self.ds_dtype) - utils.cvt_f16(tdPrdP_cur, tdPrdS_cvt) - if const_expr(stage == 0): - pipeline_dS.producer_acquire(producer_state_dS) - cute.autovec_copy(tdPrdS_cvt, tRS_sdS[None, stage]) - if const_expr(not self.use_smem_dS_for_mma_dK): - tdPrdS_r2t_f32 = cute.recast_tensor(tdPrdS_cvt, Float32) - cute.copy(thr_copy_r2t, tdPrdS_r2t_f32, tdPtdS_r2t[None, stage, 0, 0]) - - if const_expr(not self.use_smem_dS_for_mma_dK): - cute.arch.fence_view_async_tmem_store() - cute.arch.fence_proxy("async.shared", space="cta") - self.compute_sync_barrier.arrive_and_wait() - - # with cute.arch.elect_one(): - # The mma warp no longer waits for dP (it waits for dS), so we don't have to arrive - # pipeline_dP.sync_object_empty.arrive(0, pipeline_dP.consumer_mask) - pipeline_dPsum.consumer_release(consumer_state_dPsum) - consumer_state_dPsum.advance() - with cute.arch.elect_one(): - pipeline_dS.producer_commit(producer_state_dS) - producer_state_dS.advance() - - # Epilogue - # Run epilogue if we processed any m_blocks for this n_block - if process_tile: - if const_expr(not self.use_tma_store): - consumer_state_dKV = self.epilogue_dKV( - dp_idx, - warp_idx, - batch_idx, - head_idx, - n_block, - seqlen, - thr_mma_dV, - thr_mma_dK, - tdVtdV, - tdKtdK, - mdV, - mdK, - pipeline_dKV, - consumer_state_dKV, - softmax_scale, - ) - else: - thr_copy_r2s_dKV = tiled_copy_r2s_dKV.get_slice(dp_idx) - #### STORE dV - consumer_state_dKV = self.epilogue_dK_or_dV_tma( - dp_idx, - batch_idx, - head_idx, - n_block, - seqlen, - thr_mma_dV, - tdVtdV, - mdV_tma_tensor, - sdV, - tma_atom_dV, - thr_copy_r2s_dKV, - pipeline_dKV, - consumer_state_dKV, - None, # Don't scale - int(NamedBarrierBwdSm100.EpilogueWG1), # barrier_id - mdV_semaphore, - ) - #### STORE dK - consumer_state_dKV = self.epilogue_dK_or_dV_tma( - dp_idx, - batch_idx, - head_idx, - n_block, - seqlen, - thr_mma_dK, - tdKtdK, - mdK_tma_tensor, - sdK, - tma_atom_dK, - thr_copy_r2s_dKV, - pipeline_dKV, - consumer_state_dKV, - softmax_scale if const_expr(not self.dKV_postprocess) else None, - int(NamedBarrierBwdSm100.EpilogueWG1), # barrier_id - mdK_semaphore, - ) - # Zero dK/dV for empty tiles (local attention or block sparsity) - # When total_m_block_cnt == 0 for block sparsity, no Q tiles contribute to this KV tile - if const_expr(not self.dKV_postprocess): - should_zero_dKV = False - if const_expr(self.is_local or self.is_varlen_q): - should_zero_dKV = m_block_min >= m_block_max - if const_expr(self.use_block_sparsity): - # For block sparsity, zero when no m_blocks contribute to this n_block - if not process_tile: - should_zero_dKV = True - - if should_zero_dKV: - # like other epis, currently assumes hdim == hdimv - gmem_tiled_copy_zero_dKV = copy_utils.tiled_copy_2d( - self.dk_dtype, - self.tile_hdim, - 128, # num_threads - ) - gmem_thr_copy_zero_dKV = gmem_tiled_copy_zero_dKV.get_slice(dp_idx) - mdV_cur = seqlen.offset_batch_K(mdV, batch_idx, dim=3)[None, None, head_idx] - mdK_cur = seqlen.offset_batch_K(mdK, batch_idx, dim=3)[None, None, head_idx] - gdK = cute.local_tile(mdK_cur, (self.tile_n, self.tile_hdim), (n_block, 0)) - gdV = cute.local_tile(mdV_cur, (self.tile_n, self.tile_hdimv), (n_block, 0)) - tdKgdK = gmem_thr_copy_zero_dKV.partition_D(gdK) - tdVgdV = gmem_thr_copy_zero_dKV.partition_D(gdV) - assert tdKgdK.shape[2] == 1 - assert tdVgdV.shape[2] == 1 - cdKV = cute.make_identity_tensor((self.tile_n, self.tile_hdim)) - tdKVcdKV = gmem_thr_copy_zero_dKV.partition_D(cdKV) - zero = cute.make_fragment_like(tdKgdK[None, 0, 0]) - zero.fill(0.0) - if tidx < 128: - for i in cutlass.range_constexpr(tdKgdK.shape[1]): - row_idx = tdKVcdKV[0, i, 0][0] - if row_idx < seqlen.seqlen_k - self.tile_n * n_block: - cute.copy(gmem_tiled_copy_zero_dKV, zero, tdKgdK[None, i, 0]) - else: - for i in cutlass.range_constexpr(tdVgdV.shape[1]): - row_idx = tdKVcdKV[0, i, 0][0] - if row_idx < seqlen.seqlen_k - self.tile_n * n_block: - cute.copy(gmem_tiled_copy_zero_dKV, zero, tdVgdV[None, i, 0]) - - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def dQacc_reduce( - self, - mdQaccum: cute.Tensor, - sdQaccum: cute.Tensor, - thr_mma_dQ: cute.core.ThrMma, - tdQtdQ: cute.Tensor, - pipeline_dQ: PipelineAsync, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - mdQ_semaphore: Optional[cute.Tensor], - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - num_reduce_threads = cute.arch.WARP_SIZE * len(self.reduce_warp_ids) - tidx = cute.arch.thread_idx()[0] % num_reduce_threads - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx() % len(self.reduce_warp_ids)) - is_tma_warp = warp_idx == 0 - # TMEM -> RMEM - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(self.dQ_reduce_ncol)), Float32 - ) - thr_copy_t2r = tcgen05.make_tmem_copy(tmem_load_atom, tdQtdQ).get_slice(tidx) - tdQtdQ_t2r = thr_copy_t2r.partition_S(tdQtdQ) - tdQcdQ = thr_mma_dQ.partition_C(cute.make_identity_tensor(self.mma_tiler_dsk[:2])) - tdQrdQ_t2r_shape = thr_copy_t2r.partition_D(tdQcdQ).shape - assert cute.size(tdQrdQ_t2r_shape, mode=[1]) == self.dQaccum_reduce_stage, ( - "dQaccum reduce stage mismatch" - ) - - thr_copy_dQaccum_r2s = copy_utils.tiled_copy_1d( - self.dqaccum_dtype, num_reduce_threads, num_copy_elems=128 // self.dqaccum_dtype.width - ).get_slice(tidx) - tdQsdQ = thr_copy_dQaccum_r2s.partition_D(sdQaccum) - - read_flag = const_expr(not self.deterministic) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - dQ_consumer_state = pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, 1 - ) - dQ_tma_store_producer_state = pipeline.make_pipeline_state( - pipeline.PipelineUserType.Producer, self.sdQaccum_stage - ) - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - m_block_min, m_block_max = block_info.get_m_block_min_max( - seqlen, n_block // self.cluster_shape_mnk[0] - ) - if const_expr(not seqlen.has_cu_seqlens_q): - mdQaccum_cur = mdQaccum[None, head_idx, batch_idx] - else: - mdQaccum_cur = cute.domain_offset( - (seqlen.padded_offset_q * self.tile_hdim,), mdQaccum[None, head_idx] - ) - gdQaccum_ = cute.local_tile(mdQaccum_cur, (self.tile_m * self.tile_hdim,), (None,)) - # (M * K / STAGE, STAGE, _) - gdQaccum = cute.flat_divide( - gdQaccum_, (self.tile_m * self.tile_hdim // self.dQaccum_reduce_stage,) - ) - - if const_expr(self.deterministic): - mdQ_semaphore_cur = mdQ_semaphore[None, None, head_idx, batch_idx] - - delay_semaphore_release = self.is_causal - n_block_global_max = cute.ceil_div(seqlen.seqlen_k, self.tile_n) - - # some tiles might be empty due to block sparsity - if const_expr(self.use_block_sparsity): - ( - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - loop_count, - ) = get_block_sparse_iteration_info_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = loop_count > Int32(0) - else: - process_tile = ( - const_expr(not self.is_local and not self.is_varlen_q) - or m_block_min < m_block_max - ) - loop_count = m_block_max - m_block_min - - # dQacc_reduce mainloop - # Block sparsity: iterate over sparse m_block count and derive actual m_block - # from Q_IDX/FULL_Q_IDX tensors. Dense: iterate m_block_min..m_block_max directly. - for iter_idx in cutlass.range(loop_count, unroll=1): - if const_expr(self.use_block_sparsity): - m_block, _ = get_m_block_from_iter_bwd( - iter_idx, - curr_q_cnt, - curr_q_idx, - curr_full_cnt, - curr_full_idx, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - if m_block_max > 0: - m_block = cutlass.min(m_block, m_block_max - 1) - else: - m_block = m_block_min + iter_idx - pipeline_dQ.consumer_wait(dQ_consumer_state) - # TMEM -> RMEM - tdQrdQ_t2r = cute.make_fragment(tdQrdQ_t2r_shape, Float32) - cute.copy(thr_copy_t2r, tdQtdQ_t2r, tdQrdQ_t2r) - cute.arch.fence_view_async_tmem_load() - cute.arch.sync_warp() - with cute.arch.elect_one(): - pipeline_dQ.consumer_release(dQ_consumer_state) - dQ_consumer_state.advance() - - gdQaccum_cur = gdQaccum[None, None, m_block] - - for stage in cutlass.range_constexpr(cute.size(tdQrdQ_t2r, mode=[1])): # 4 - smem_idx = dQ_tma_store_producer_state.index - tdQsdQ_r2s = tdQsdQ[None, None, smem_idx] - tdQrdQ_r2s = cute.make_tensor( - tdQrdQ_t2r[None, stage, None, None].iterator, tdQsdQ_r2s.shape - ) - cute.copy(thr_copy_dQaccum_r2s, tdQrdQ_r2s, tdQsdQ_r2s) - # Fence and barrier to make sure shared memory store is visible to TMA store - cute.arch.fence_proxy("async.shared", space="cta") - # semaphore acquire - if const_expr(self.deterministic and stage == 0): - if const_expr(self.spt): - if const_expr( - self.is_causal or block_info.window_size_right is not None - ): - n_idx_right = ( - (m_block + 1) * self.tile_m + seqlen.seqlen_k - seqlen.seqlen_q - ) - if const_expr(block_info.window_size_right is not None): - n_idx_right += block_info.window_size_right - n_block_max_for_m_block = min( - n_block_global_max, - cute.ceil_div(n_idx_right, self.tile_n), - ) - else: - n_block_max_for_m_block = n_block_global_max - lock_value = n_block_max_for_m_block - 1 - n_block - else: - lock_value = n_block - barrier.wait_eq( - mdQ_semaphore_cur[(m_block, None)].iterator, tidx, 0, lock_value - ) - self.reduce_sync_barrier.arrive_and_wait() - # Copy from shared memory to global memory - if is_tma_warp: - with cute.arch.elect_one(): - copy_utils.cpasync_reduce_bulk_add_f32( - sdQaccum[None, smem_idx].iterator, - gdQaccum_cur[None, stage].iterator, - self.tma_copy_bytes["dQ"] // 1, - ) - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(self.sdQaccum_stage - 1, read=read_flag) - self.reduce_sync_barrier.arrive_and_wait() - dQ_tma_store_producer_state.advance() - # Directly add to gmem, much slower - # tdQgdQ = thr_copy_dQaccum_r2s.partition_D(gdQaccum[None, stage, m_block]) - # assert cute.size(tdQrdQ_r2s) == cute.size(tdQgdQ) - # for i in cutlass.range(cute.size(tdQrdQ_r2s) // 4, unroll_full=True): - # copy_utils.atomic_add_fp32x4( - # tdQrdQ_r2s[4 * i], - # tdQrdQ_r2s[4 * i + 1], - # tdQrdQ_r2s[4 * i + 2], - # tdQrdQ_r2s[4 * i + 3], - # utils.elem_pointer(tdQgdQ, 4 * i), - # ) - # semaphore release for prior m_block - if const_expr(self.deterministic and stage == 0 and delay_semaphore_release): - if m_block > m_block_min: - barrier.arrive_inc( - mdQ_semaphore_cur[(m_block - 1, None)].iterator, tidx, 0, 1 - ) - - # semaphore release - # NOTE: arrive_inc calls red_release which issues membar - if const_expr(self.deterministic and not delay_semaphore_release): - if is_tma_warp: - cute.arch.cp_async_bulk_wait_group(0, read=read_flag) - self.reduce_sync_barrier.arrive_and_wait() - barrier.arrive_inc(mdQ_semaphore_cur[m_block, None].iterator, tidx, 0, 1) - - if const_expr(not self.is_local) or m_block_min < m_block_max: - if is_tma_warp: - cute.arch.cp_async_bulk_wait_group(0, read=read_flag) - self.reduce_sync_barrier.arrive_and_wait() - # final semaphore release - if const_expr(self.deterministic and delay_semaphore_release): - barrier.arrive_inc( - mdQ_semaphore_cur[(m_block_max - 1, None)].iterator, tidx, 0, 1 - ) - - if const_expr( - self.deterministic and not self.spt and block_info.window_size_left is not None - ): - m_block_global_max = cute.ceil_div(seqlen.seqlen_q, self.tile_m) - for m_block in cutlass.range(m_block_max, m_block_global_max, unroll=1): - barrier.arrive_inc(mdQ_semaphore_cur[(m_block, None)].iterator, tidx, 0, 1) - - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def epilogue_dKV( - self, - tidx: Int32, - warp_idx: Int32, - batch_idx: Int32, - head_idx: Int32, - n_block: Int32, - seqlen, - thr_mma_dV: cute.core.ThrMma, - thr_mma_dK: cute.core.ThrMma, - tdVtdV: cute.Tensor, - tdKtdK: cute.Tensor, - mdV: cute.Tensor, - mdK: cute.Tensor, - pipeline_dKV: PipelineAsync, - consumer_state_dKV: cutlass.pipeline.PipelineState, - softmax_scale: Float32, - ): - wg_idx = ( - cute.arch.thread_idx()[0] % (cute.arch.WARP_SIZE * len(self.compute_warp_ids)) - ) // 128 - num_wg = cute.arch.WARP_SIZE * len(self.compute_warp_ids) // 128 - - assert self.qhead_per_kvhead == 1, "This epilogue path is only for MHA" - mdV_cur = seqlen.offset_batch_K(mdV, batch_idx, dim=3)[None, None, head_idx] - mdK_cur = seqlen.offset_batch_K(mdK, batch_idx, dim=3)[None, None, head_idx] - - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(16)), Float32 - ) - - # dV - pipeline_dKV.consumer_wait(consumer_state_dKV) - - tiled_tmem_ld_dV = tcgen05.make_tmem_copy(tmem_load_atom, tdVtdV) - thr_tmem_ld_dV = tiled_tmem_ld_dV.get_slice(tidx) - - tdVtdV_t2r_p = thr_tmem_ld_dV.partition_S(tdVtdV) - tdVtdV_t2r = self.split_wg(tdVtdV_t2r_p, wg_idx, num_wg) - - cdV = cute.make_identity_tensor((self.mma_tiler_pdo[0], self.mma_tiler_pdo[1])) - tdVcdV = thr_mma_dV.partition_C(cdV) - tdVcdV_tensor = cute.make_tensor(tdVcdV.iterator, tdVcdV.layout) - - tdVcdV_t2r_p = thr_tmem_ld_dV.partition_D(tdVcdV_tensor) - tdVcdV_t2r = self.split_wg(tdVcdV_t2r_p, wg_idx, num_wg) - tdVrdV_t2r = cute.make_fragment(tdVcdV_t2r.shape, Float32) - - cute.copy(thr_tmem_ld_dV, tdVtdV_t2r, tdVrdV_t2r) - cute.arch.fence_view_async_tmem_load() - - universal_copy_bits = 128 - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dv_dtype, - num_bits_per_copy=universal_copy_bits, - ) - tiled_gmem_store_dV = cute.make_tiled_copy( - atom_universal_copy, - layout_tv=tiled_tmem_ld_dV.layout_dst_tv_tiled, - tiler_mn=tiled_tmem_ld_dV.tiler_mn, - ) - - tdVrdV_r2s = cute.make_fragment(tdVrdV_t2r.shape, self.dv_dtype) - for i in cutlass.range_constexpr(cute.size(tdVrdV_t2r, mode=[1])): - dV_vec = tdVrdV_t2r[(None, i, 0, 0)].load() - tdVrdV_r2s[(None, i, 0, 0)].store(dV_vec.to(self.dv_dtype)) - - gdV = cute.local_tile(mdV_cur, (self.tile_n, self.tile_hdimv), (None, 0)) - gdV_tile = gdV[None, None, n_block] - - tdVgdV = thr_mma_dV.partition_C(gdV_tile) - tdVgdV_r2g_p = thr_tmem_ld_dV.partition_D(tdVgdV) - tdVgdV_r2g = self.split_wg(tdVgdV_r2g_p, wg_idx, num_wg) - - if tidx < seqlen.seqlen_k - self.tile_n * n_block: - cute.copy(tiled_gmem_store_dV, tdVrdV_r2s, tdVgdV_r2g) - - cute.arch.sync_warp() - with cute.arch.elect_one(): - pipeline_dKV.consumer_release(consumer_state_dKV) - consumer_state_dKV.advance() - - # dK - pipeline_dKV.consumer_wait(consumer_state_dKV) - - tiled_tmem_ld_dK = tcgen05.make_tmem_copy(tmem_load_atom, tdKtdK) - thr_tmem_ld_dK = tiled_tmem_ld_dK.get_slice(tidx) - - tdKtdK_t2r_p = thr_tmem_ld_dK.partition_S(tdKtdK) - tdKtdK_t2r = self.split_wg(tdKtdK_t2r_p, wg_idx, num_wg) - - cdK = cute.make_identity_tensor((self.mma_tiler_dsq[0], self.mma_tiler_dsq[1])) - tdKcdK = thr_mma_dK.partition_C(cdK) - tdKcdK_tensor = cute.make_tensor(tdKcdK.iterator, tdKcdK.layout) - - tdKcdK_t2r_p = thr_tmem_ld_dK.partition_D(tdKcdK_tensor) - tdKcdK_t2r = self.split_wg(tdKcdK_t2r_p, wg_idx, num_wg) - tdKrdK_t2r = cute.make_fragment(tdKcdK_t2r.shape, Float32) - - cute.copy(tiled_tmem_ld_dK, tdKtdK_t2r, tdKrdK_t2r) - cute.arch.fence_view_async_tmem_load() - - universal_copy_bits = 128 - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dk_dtype, - num_bits_per_copy=universal_copy_bits, - ) - - tiled_gmem_store_dK = cute.make_tiled_copy( - atom_universal_copy, - layout_tv=tiled_tmem_ld_dK.layout_dst_tv_tiled, - tiler_mn=tiled_tmem_ld_dK.tiler_mn, - ) - - tdKrdK_r2s = cute.make_fragment(tdKrdK_t2r.shape, self.dk_dtype) - - for i in cutlass.range_constexpr(cute.size(tdKrdK_t2r, mode=[1])): - dK_vec = tdKrdK_t2r[(None, i, 0, 0)].load() * softmax_scale - tdKrdK_r2s[(None, i, 0, 0)].store(dK_vec.to(self.dk_dtype)) - - gdK = cute.local_tile(mdK_cur, (self.tile_n, self.tile_hdimv), (None, 0)) - gdK_tile = gdK[None, None, n_block] - - tdKgdK = thr_mma_dK.partition_C(gdK_tile) - tdKgdK_r2g_p = thr_tmem_ld_dK.partition_D(tdKgdK) - tdKgdK_r2g = self.split_wg(tdKgdK_r2g_p, wg_idx, num_wg) - - if tidx < seqlen.seqlen_k - self.tile_n * n_block: - cute.copy(tiled_gmem_store_dK, tdKrdK_r2s, tdKgdK_r2g) - - cute.arch.sync_warp() - with cute.arch.elect_one(): - pipeline_dKV.consumer_release(consumer_state_dKV) - consumer_state_dKV.advance() - return consumer_state_dKV - - @cute.jit - def epilogue_dK_or_dV_tma( - self, - tidx: Int32, - batch_idx: Int32, - head_idx: Int32, - n_block: Int32, - seqlen, - thr_mma: cute.core.ThrMma, - tdKVtdKV: cute.Tensor, - mdKV: cute.Tensor, - sdKV: cute.Tensor, - tma_atom_dKV: cute.CopyAtom, - thr_copy_r2s_dKV: cute.TiledCopy, - pipeline_dKV: PipelineAsync, - consumer_state_dKV: cutlass.pipeline.PipelineState, - scale: Optional[Float32], - barrier_id: Int32, - mdKV_semaphore: Optional[cute.Tensor], - ) -> cutlass.pipeline.PipelineState: - # assumes mma_tiler_pdo = mma_tiler_dsq = (tile_n, head_dim) - # head_dim = head_dim_v, dk_dtype = dv_dtype - num_compute_threads = cute.arch.WARP_SIZE * len(self.compute_warp_ids) - wg_idx = (cute.arch.thread_idx()[0] % num_compute_threads) // 128 - num_wg = num_compute_threads // 128 - leader_warp = (cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4) == 0 - - if const_expr(not self.dKV_postprocess): - sdKV = sdKV[None, None, wg_idx] # (tile_n, 64) for bf16 - else: - sdKV = sdKV[None, wg_idx] # (tile_n * 32) for fp32 - - # (8, tile_n / 128, 64 / 8) = (8, 1, 8) or (4, tile_n * 32 / (128 * 4)) = (4, 8) - tdKVsdKV_r2s = thr_copy_r2s_dKV.partition_D(sdKV) - - head_idx_kv = head_idx // self.qhead_per_kvhead - if const_expr(not self.dKV_postprocess): - assert not seqlen.has_cu_seqlens_k, "varlen uses non tma store path" - mdKV_cur = mdKV[None, None, head_idx_kv, batch_idx] # (seqlen, hdim) - gdKV_p = cute.local_tile( - mdKV_cur, (self.tile_n, self.tile_hdim), (n_block, 0) - ) # (tile_n, hdim) - gdKV = self.split_wg(gdKV_p, wg_idx, num_wg) # (tile_n, hdim / 2) - gdKV_epi = cute.local_tile( - gdKV, self.sdKV_epi_tile, (0, None) - ) # (tile_n, 64, epi_stage = (hdim / 2) / 64) - else: - if const_expr(not seqlen.has_cu_seqlens_k): - mdKV_cur = mdKV[None, head_idx_kv, batch_idx] # (seqlen * hdim) - else: - mdKV_cur = cute.domain_offset( - (seqlen.padded_offset_k * self.tile_hdim,), mdKV[None, head_idx_kv] - ) - gdKV_p = cute.local_tile( - mdKV_cur, (self.tile_n * self.tile_hdim,), (n_block,) - ) # (tile_n * hdim) - gdKV = cute.logical_divide(gdKV_p, (self.tile_n * self.tile_hdim // num_wg,))[ - ((None, wg_idx),) - ] # (tile_n * hdim / 2) - gdKV_epi = cute.flat_divide( - gdKV, (self.sdKV_flat_epi_tile,) - ) # (tile_n * hdim / 2 / epi_stage, epi_stage) - - deterministic_KV = self.deterministic and self.qhead_per_kvhead > 1 - if const_expr(deterministic_KV): - mdKV_semaphore_cur = mdKV_semaphore[n_block, None, head_idx_kv, batch_idx] - - if const_expr(not self.dKV_postprocess): - tdKVsdKV, tdKVgdKV = cpasync.tma_partition( - tma_atom_dKV, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sdKV, 0, 2), - cute.group_modes(gdKV_epi, 0, 2), - ) # (TMA) and (TMA, EPI_STAGE) - assert len(tdKVsdKV.shape) == 1, "Wrong rank for SMEM fragment tdKVsdKV" - assert len(tdKVgdKV.shape) == 2, "Wrong rank for GMEM fragment tdKVgdKV" - num_epi_stages = cute.size(tdKVgdKV.shape[1]) - assert num_epi_stages == self.num_epi_stages, "Epi stage calculation is wrong" - else: - num_epi_stages = self.num_epi_stages - - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), Float32 - ) - - read_flag = const_expr(not deterministic_KV) - - pipeline_dKV.consumer_wait(consumer_state_dKV) - - # semaphore acquire - if const_expr(deterministic_KV): - barrier.wait_eq( - mdKV_semaphore_cur.iterator, tidx, wg_idx, head_idx % self.qhead_per_kvhead - ) - cute.arch.barrier(barrier_id=barrier_id + wg_idx, number_of_threads=128) - - for epi_stage in cutlass.range_constexpr(num_epi_stages): - # TMEM -> RMEM -- setup - thr_copy_t2r = tcgen05.make_tmem_copy(tmem_load_atom, tdKVtdKV).get_slice(tidx) - tdKVtdKV_t2r_p = thr_copy_t2r.partition_S(tdKVtdKV) - tdKVtdKV_t2r = self.split_wg(tdKVtdKV_t2r_p, wg_idx, num_wg)[None, None, 0, 0] - if const_expr(num_epi_stages > 1): - tdKVtdKV_t2r = tdKVtdKV_t2r[None, epi_stage] - - cdKV = cute.make_identity_tensor((self.tile_n, self.tile_hdim)) - tdKVcdKV = thr_mma.partition_C(cdKV) - tdKVcdKV_t2r_p = thr_copy_t2r.partition_D(tdKVcdKV) - tdKVcdKV_t2r = self.split_wg(tdKVcdKV_t2r_p, wg_idx, num_wg)[None, None, 0, 0] - if const_expr(num_epi_stages > 1): - tdKVcdKV_t2r = tdKVcdKV_t2r[None, epi_stage] - - tdKVrdKV_t2r = cute.make_fragment(tdKVcdKV_t2r.shape, Float32) - - assert cute.size(tdKVrdKV_t2r) == cute.size(tdKVtdKV_t2r) // cute.arch.WARP_SIZE, ( - "RMEM<->TMEM fragment size mismatch" - ) - - # TMEM -> RMEM -- copy and fence - cute.copy(thr_copy_t2r, tdKVtdKV_t2r, tdKVrdKV_t2r) - cute.arch.fence_view_async_tmem_load() - - # RMEM -- scale and convert - if const_expr(scale is not None): - for i in cutlass.range(cute.size(tdKVrdKV_t2r.shape) // 2, unroll_full=True): - tdKVrdKV_t2r[2 * i], tdKVrdKV_t2r[2 * i + 1] = utils.mul_packed_f32x2( - (tdKVrdKV_t2r[2 * i], tdKVrdKV_t2r[2 * i + 1]), (scale, scale) - ) - tdKVrdKV = cute.make_fragment(tdKVrdKV_t2r.shape, self.dv_dtype) # (32 columns) - tdKVrdKV.store(tdKVrdKV_t2r.load().to(self.dv_dtype)) - - # RMEM -> SMEM -- copy, fence and barrier - tdKVrdKV_r2s = cute.make_tensor(tdKVrdKV.iterator, tdKVsdKV_r2s.shape) - cute.copy(thr_copy_r2s_dKV, tdKVrdKV_r2s, tdKVsdKV_r2s) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier(barrier_id=barrier_id + wg_idx, number_of_threads=128) - - # SMEM -> GMEM - if leader_warp: - if const_expr(not self.dKV_postprocess): - cute.copy(tma_atom_dKV, tdKVsdKV, tdKVgdKV[None, epi_stage]) - else: - with cute.arch.elect_one(): - copy_utils.cpasync_reduce_bulk_add_f32( - sdKV.iterator, - gdKV_epi[None, epi_stage].iterator, - self.tma_copy_bytes["dKacc"], - ) - if const_expr(epi_stage < num_epi_stages - 1): - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=read_flag) - cute.arch.barrier_arrive( - barrier_id=barrier_id + wg_idx, number_of_threads=128 + cute.arch.WARP_SIZE - ) - - # Barrier since all warps need to wait for SMEM to be freed - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=barrier_id + wg_idx, number_of_threads=128 + cute.arch.WARP_SIZE - ) - - # semaphore release - # NOTE: arrive_inc calls red_release which issues membar - if const_expr(deterministic_KV): - if leader_warp: - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=read_flag) - cute.arch.barrier(barrier_id=barrier_id + wg_idx, number_of_threads=128) - barrier.arrive_inc(mdKV_semaphore_cur.iterator, tidx, wg_idx, 1) - - cute.arch.sync_warp() - with cute.arch.elect_one(): - pipeline_dKV.consumer_release(consumer_state_dKV) - consumer_state_dKV.advance() - return consumer_state_dKV diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm90.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm90.py deleted file mode 100644 index b97d751ca639..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_bwd_sm90.py +++ /dev/null @@ -1,1701 +0,0 @@ -import math -from typing import Callable, Optional, Type -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -import cutlass.utils.hopper_helpers as sm90_utils_basic -from cutlass.cute.nvgpu import cpasync, warpgroup -from cutlass.cute import FastDivmodDivisor -from cutlass import Float32, Int32, Boolean, const_expr -from cutlass.utils import LayoutEnum - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.hopper_helpers as sm90_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -from .hopper_helpers import gemm_zero_init, gemm_w_idx -from .mask import AttentionMask -from .seqlen_info import SeqlenInfoQK -from .block_info import BlockInfo -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.pipeline as pipeline -from .tile_scheduler import TileSchedulerArguments, SingleTileScheduler, ParamsBase -from .named_barrier import NamedBarrierFwd, NamedBarrierBwd -from .softmax import apply_score_mod_inner, apply_score_mod_bwd_inner -from .block_sparsity import BlockSparseTensors -from .block_sparse_utils import ( - get_total_q_block_count_bwd, - produce_block_sparse_q_loads_bwd_sm90, - consume_block_sparse_mma_bwd_sm90, - dQaccum_store_block_sparse_bwd_sm90, -) - - -def mma_partition_fragment_AB( - thr_mma: cute.core.ThrMma, sA: Optional[cute.Tensor], sB: Optional[cute.Tensor], swap_AB: bool -): - if const_expr(not swap_AB): - return ( - thr_mma.make_fragment_A(thr_mma.partition_A(sA)) if sA is not None else None, - thr_mma.make_fragment_B(thr_mma.partition_B(sB)) if sB is not None else None, - ) - else: - return ( - thr_mma.make_fragment_B(thr_mma.partition_B(sA)) if sA is not None else None, - thr_mma.make_fragment_A(thr_mma.partition_A(sB)) if sB is not None else None, - ) - - -class FlashAttentionBackwardSm90: - arch = 90 - - def __init__( - self, - dtype: Type[cutlass.Numeric], - head_dim: int, - head_dim_v: Optional[int] = None, - qhead_per_kvhead: int = 1, - is_causal: bool = False, - tile_m: int = 64, - tile_n: int = 128, - Q_stage: int = 2, - dO_stage: int = 2, - PdS_stage: int = 2, - SdP_swapAB: bool = False, - dKV_swapAB: bool = False, - dQ_swapAB: bool = False, - AtomLayoutMSdP: int = 1, - AtomLayoutNdKV: int = 2, - AtomLayoutMdQ: int = 1, - num_threads: int = 384, - V_in_regs: bool = False, - score_mod: cutlass.Constexpr | None = None, - score_mod_bwd: cutlass.Constexpr | None = None, - mask_mod: cutlass.Constexpr | None = None, - has_aux_tensors: cutlass.Constexpr = False, - subtile_factor: cutlass.Constexpr[int] = 1, - ): - self.dtype = dtype - # padding head_dim to a multiple of 16 as k_block_size - hdim_multiple_of = 16 - self.tile_hdim = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - head_dim_v = head_dim_v if head_dim_v is not None else head_dim - self.same_hdim_kv = head_dim == head_dim_v - self.tile_hdimv = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of) - # Can save registers (and hence be faster) if we don't have to check hdim predication - self.check_hdim_oob = head_dim != self.tile_hdim - self.check_hdim_v_oob = head_dim_v != self.tile_hdimv - self.qhead_per_kvhead = qhead_per_kvhead - self.is_causal = is_causal - self.is_local = False - self.tile_m = tile_m - self.tile_n = tile_n - self.num_threads = num_threads - self.Q_stage = Q_stage - self.dO_stage = dO_stage - self.PdS_stage = PdS_stage - assert self.dO_stage in [1, self.Q_stage] - assert self.PdS_stage in [1, self.Q_stage] - self.SdP_swapAB = SdP_swapAB - self.dKV_swapAB = dKV_swapAB - self.dQ_swapAB = dQ_swapAB - self.AtomLayoutMSdP = AtomLayoutMSdP - self.AtomLayoutNdKV = AtomLayoutNdKV - self.AtomLayoutMdQ = AtomLayoutMdQ - self.num_mma_warp_groups = (self.num_threads // 128) - 1 - self.mma_dkv_is_rs = ( - AtomLayoutMSdP == 1 - and AtomLayoutNdKV == self.num_mma_warp_groups - and SdP_swapAB - and not dKV_swapAB - ) - self.V_in_regs = V_in_regs - if qhead_per_kvhead > 1: - assert self.same_hdim_kv, "GQA backward requires head_dim == head_dim_v" - assert self.num_mma_warp_groups == 2, "GQA backward assumes 2 warp groups" - # These are tuned for speed - # Do we keep the LSE and dPsum in each thread, or split them across 8 threads that share - # them and then shuffle to get the value whenever we need? This can reduce register - # pressure when SdP_swapAB, where each thread needs to keep statistics for (kBlockM / 4) - # rows. If !SdP_swapAB, each thread only needs to keep statistics for 2 rows. - # TODO: impl these for hdim 64 - self.shuffle_LSE = self.SdP_swapAB and self.tile_hdim <= 64 - self.shuffle_dPsum = self.SdP_swapAB and self.tile_hdim <= 64 - - self.score_mod = score_mod - self.score_mod_bwd = score_mod_bwd - self.mask_mod = mask_mod - self.has_aux_tensors = has_aux_tensors - self.subtile_factor = subtile_factor - if cutlass.const_expr(has_aux_tensors): - self.vec_size: cutlass.Constexpr = 1 - else: - self.vec_size: cutlass.Constexpr = 4 - self.qk_acc_dtype = Float32 - - @staticmethod - def can_implement( - dtype, - head_dim, - head_dim_v, - tile_m, - tile_n, - Q_stage, - num_threads, - V_in_regs=False, - ) -> bool: - if dtype not in [cutlass.Float16, cutlass.BFloat16]: - return False - if head_dim % 8 != 0: - return False - if head_dim_v % 8 != 0: - return False - if tile_n % 16 != 0: - return False - if num_threads % 32 != 0: - return False - if (tile_m * 2) % num_threads != 0: - return False - return True - - def _check_type( - self, - mQ_type: Type[cutlass.Numeric], - mK_type: Type[cutlass.Numeric], - mV_type: Type[cutlass.Numeric], - mdO_type: Type[cutlass.Numeric], - mLSE_type: Type[cutlass.Numeric], - mdPsum_type: Type[cutlass.Numeric], - mdQaccum_type: Type[cutlass.Numeric], - mdK_type: Type[cutlass.Numeric], - mdV_type: Type[cutlass.Numeric], - ): - # Get the data type and check if it is fp16 or bf16 - if const_expr(not (mQ_type == mK_type == mV_type == mdO_type)): - raise TypeError("All tensors must have the same data type") - if const_expr(mQ_type not in [cutlass.Float16, cutlass.BFloat16]): - raise TypeError("Only Float16 or BFloat16 is supported") - if const_expr(mLSE_type not in [Float32]): - raise TypeError("LSE tensor must be Float32") - if const_expr(mdPsum_type not in [Float32]): - raise TypeError("dPsum tensor must be Float32") - if const_expr(mdQaccum_type not in [Float32]): - raise TypeError("dQaccum tensor must be Float32") - if const_expr(self.qhead_per_kvhead == 1): - if const_expr(not (mdK_type == mdV_type == mQ_type)): - raise TypeError("mdK and mdV tensors must have the same data type as mQ") - else: - if const_expr(not (mdK_type == mdV_type == Float32)): - raise TypeError("mdKaccum and mdVaccum tensors must have the data type Float32") - assert mQ_type == self.dtype - - def _setup_attributes(self): - self.sQ_layout, self.sK_layout, self.sV_layout, self.sdO_layout, self.sPdS_layout = [ - sm90_utils.make_smem_layout(self.dtype, LayoutEnum.ROW_MAJOR, shape, stage) - for shape, stage in [ - ((self.tile_m, self.tile_hdim), self.Q_stage), - ((self.tile_n, self.tile_hdim), None), - ((self.tile_n, self.tile_hdimv), None), - ((self.tile_m, self.tile_hdimv), self.dO_stage), - ((self.tile_m, self.tile_n), self.PdS_stage), - ] - ] - self.sdQaccum_layout = cute.make_layout( - (self.tile_m * self.tile_hdim // self.num_mma_warp_groups, self.num_mma_warp_groups) - ) - # dQaccum R->S - self.r2s_tiled_copy_dQaccum = cute.make_tiled_copy_tv( - cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128), - # thr_layout - cute.make_layout((self.num_threads_per_warp_group, self.num_mma_warp_groups)), - cute.make_layout(128 // Float32.width), # val_layout - ) - # dKVaccum for GQA epilogue - reuses sV+sK memory recast as f32 - self.sdKVaccum_layout = cute.make_layout( - (self.tile_n * self.tile_hdim // self.num_mma_warp_groups, self.num_mma_warp_groups) - ) - # dKVaccum R->S (same pattern as dQaccum but sized for tile_n) - self.r2s_tiled_copy_dKVaccum = cute.make_tiled_copy_tv( - cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128), - cute.make_layout((self.num_threads_per_warp_group, self.num_mma_warp_groups)), - cute.make_layout(128 // Float32.width), - ) - - def _get_tiled_mma(self): - # S = Q @ K.T, dP = dO @ V.T - atom_layout_SdP = (self.AtomLayoutMSdP, self.num_mma_warp_groups // self.AtomLayoutMSdP) - tiler_mn_SdP = (self.tile_m // atom_layout_SdP[0], self.tile_n // atom_layout_SdP[1]) - tiled_mma_SdP = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K, - warpgroup.OperandMajorMode.K, - Float32, - atom_layout_mnk=(atom_layout_SdP if not self.SdP_swapAB else atom_layout_SdP[::-1]) - + (1,), - tiler_mn=tiler_mn_SdP if not self.SdP_swapAB else tiler_mn_SdP[::-1], - ) - # dV = P.T @ dO, dK = dS.T @ Q - atom_layout_dKV = (self.AtomLayoutNdKV, self.num_mma_warp_groups // self.AtomLayoutNdKV) - tiler_mn_dK = (self.tile_n // atom_layout_dKV[0], self.tile_hdim // atom_layout_dKV[1]) - tiler_mn_dV = (self.tile_n // atom_layout_dKV[0], self.tile_hdimv // atom_layout_dKV[1]) - tiled_mma_dK, tiled_mma_dV = [ - sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.MN - if not self.mma_dkv_is_rs - else warpgroup.OperandMajorMode.K, - warpgroup.OperandMajorMode.MN, - Float32, - atom_layout_mnk=(atom_layout_dKV if not self.dKV_swapAB else atom_layout_dKV[::-1]) - + (1,), - tiler_mn=tiler_mn_d if not self.dKV_swapAB else tiler_mn_d[::-1], - a_source=warpgroup.OperandSource.RMEM - if self.mma_dkv_is_rs - else warpgroup.OperandSource.SMEM, - ) - for tiler_mn_d in (tiler_mn_dK, tiler_mn_dV) - ] - # dQ = dS @ K - atom_layout_dQ = (self.AtomLayoutMdQ, self.num_mma_warp_groups // self.AtomLayoutMdQ) - tiler_mn_dQ = (self.tile_m // atom_layout_dQ[0], self.tile_hdim // atom_layout_dQ[1]) - tiled_mma_dQ = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K if not self.dQ_swapAB else warpgroup.OperandMajorMode.MN, - warpgroup.OperandMajorMode.MN if not self.dQ_swapAB else warpgroup.OperandMajorMode.K, - Float32, - atom_layout_mnk=(atom_layout_dQ if not self.dQ_swapAB else atom_layout_dQ[::-1]) + (1,), - tiler_mn=tiler_mn_dQ if not self.dQ_swapAB else tiler_mn_dQ[::-1], - ) - return tiled_mma_SdP, tiled_mma_dK, tiled_mma_dV, tiled_mma_dQ - - def _get_shared_storage_cls(self): - sQ_alignment = sK_alignment = sV_alighment = sdQaccum_alignment = sdO_alignment = 1024 - - sQ_struct, sK_struct, sV_struct, sdO_struct, sdQaccum_struct = [ - cute.struct.Align[cute.struct.MemRange[type, cute.cosize(layout)], alignment] - for (layout, type, alignment) in [ - (self.sQ_layout, self.dtype, sQ_alignment), - (self.sK_layout, self.dtype, sK_alignment), - (self.sV_layout, self.dtype, sV_alighment), - (self.sdO_layout, self.dtype, sdO_alignment), - (self.sdQaccum_layout, Float32, sdQaccum_alignment), - ] - ] - - cosize_sdS = cute.cosize(self.sPdS_layout) - cosize_sP = cute.cosize(self.sPdS_layout) if const_expr(not self.mma_dkv_is_rs) else 0 - sLSE_struct = cute.struct.Align[ - cute.struct.MemRange[Float32, cute.round_up(self.tile_m, 64) * self.Q_stage], 128 - ] - sdPsum_struct = cute.struct.Align[ - cute.struct.MemRange[Float32, cute.round_up(self.tile_m, 64) * self.dO_stage], 128 - ] - - @cute.struct - class SharedStorageQKV: - mbar_ptr_Q: cute.struct.MemRange[cutlass.Int64, self.Q_stage * 2] - mbar_ptr_dO: cute.struct.MemRange[cutlass.Int64, self.dO_stage * 2] - sLSE: sLSE_struct - sdPsum: sdPsum_struct - sQ: sQ_struct - sV: sV_struct - sK: sK_struct - sdO: sdO_struct - sP: cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sP], 1024] - sdS: cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sdS], 1024] - sdQaccum: sdQaccum_struct - - return SharedStorageQKV - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdQaccum: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - softmax_scale: Float32, - stream: cuda.CUstream, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - softcap: Float32 | float | None = None, - window_size_left: Int32 | int | None = None, - window_size_right: Int32 | int | None = None, - mdQ_semaphore: Optional[cute.Tensor] = None, - mdK_semaphore: Optional[cute.Tensor] = None, - mdV_semaphore: Optional[cute.Tensor] = None, - aux_tensors: Optional[list] = None, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - assert mdQ_semaphore is None and mdK_semaphore is None and mdV_semaphore is None, ( - "determinism not supported yet for Sm90" - ) - - self._check_type( - *( - t.element_type if t is not None else None - for t in (mQ, mK, mV, mdO, mLSE, mdPsum, mdQaccum, mdK, mdV) - ) - ) - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mQ, mK, mV, mdO, mLSE, mdPsum, mdQaccum, mdK, mdV = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - if t is not None - else None - for t in (mQ, mK, mV, mdO, mLSE, mdPsum, mdQaccum, mdK, mdV) - ] - - layout_transpose = [1, 3, 2, 0] # (b, s, n, h) --> (s, h, n, b) - mQ, mK, mV, mdO = [utils.select(t, layout_transpose) for t in (mQ, mK, mV, mdO)] - if const_expr(self.qhead_per_kvhead == 1): - mdK, mdV = [utils.select(t, layout_transpose) for t in (mdK, mdV)] - else: - accum_transpose = [2, 1, 0] # (b, n, s*h) -> (s*h, n, b) - mdK, mdV = [utils.select(t, accum_transpose) for t in (mdK, mdV)] - LSE_dPsum_dQaccum_transpose = [2, 1, 0] # (b, n, s) -> (s, n, b) - mLSE, mdPsum, mdQaccum = [ - utils.select(t, LSE_dPsum_dQaccum_transpose) for t in (mLSE, mdPsum, mdQaccum) - ] - - tiled_mma_SdP, tiled_mma_dK, tiled_mma_dV, tiled_mma_dQ = self._get_tiled_mma() - - self.num_mma_threads = tiled_mma_SdP.size - assert self.num_mma_threads + 128 == self.num_threads - - self.num_threads_per_warp_group = 128 - self.num_producer_threads = 32 - - self.num_mma_regs = 240 - self.num_producer_regs = 24 - # self.num_mma_regs = 232 - # self.num_producer_regs = 40 - - self._setup_attributes() - SharedStorage = self._get_shared_storage_cls() - - self.tma_copy_bytes = { - name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1])) - for name, mX, layout in [ - ("Q", mQ, self.sQ_layout), - ("K", mK, self.sK_layout), - ("V", mV, self.sV_layout), - ("dO", mdO, self.sdO_layout), - ] - } - self.tma_copy_bytes["LSE"] = self.tile_m * Float32.width // 8 - self.tma_copy_bytes["dPsum"] = self.tile_m * Float32.width // 8 - self.tma_copy_bytes["dQ"] = ( - self.tile_m * self.tile_hdim * Float32.width // 8 // self.num_mma_warp_groups - ) - self.tma_copy_bytes["dKacc"] = self.tile_n * self.tile_hdim * Float32.width // 8 - self.tma_copy_bytes["dVacc"] = self.tile_n * self.tile_hdimv * Float32.width // 8 - - tma_atom_Q, tma_tensor_Q = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileG2SOp(), - mQ, - cute.select(self.sQ_layout, mode=[0, 1]), - (self.tile_m, self.tile_hdim), - ) - tma_atom_K, tma_tensor_K = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileG2SOp(), - mK, - cute.select(self.sK_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdim), - ) - tma_atom_V, tma_tensor_V = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileG2SOp(), - mV, - cute.select(self.sV_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdimv), - ) - tma_atom_dO, tma_tensor_dO = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileG2SOp(), - mdO, - cute.select(self.sdO_layout, mode=[0, 1]), - (self.tile_m, self.tile_hdimv), - ) - if const_expr(self.qhead_per_kvhead == 1): - tma_atom_dK, tma_tensor_dK = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileS2GOp(), - mdK, - cute.select(self.sK_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdim), - ) - tma_atom_dV, tma_tensor_dV = cpasync.make_tiled_tma_atom( - cpasync.CopyBulkTensorTileS2GOp(), - mdV, - cute.select(self.sV_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdimv), - ) - else: - tma_atom_dK = tma_atom_dV = tma_tensor_dK = tma_tensor_dV = None - - TileScheduler = SingleTileScheduler - tile_sched_args = TileSchedulerArguments( - cute.ceil_div(cute.size(mK.shape[0]), self.tile_n), - cute.size(mQ.shape[2]), - cute.size(mQ.shape[3]), - 1, # num_splits - cute.size(mK.shape[0]), - mQ.shape[1], - mV.shape[1], - total_q=cute.size(mQ.shape[0]) * cute.size(mQ.shape[3]), - tile_shape_mn=(self.tile_m, self.tile_n), - mCuSeqlensQ=None, - mSeqUsedQ=None, - qhead_per_kvhead_packgqa=1, - element_size=self.dtype.width // 8, - is_persistent=False, - lpt=False, - ) - - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - - LOG2_E = math.log2(math.e) - if const_expr(self.score_mod is None): - softmax_scale_log2 = softmax_scale * LOG2_E - else: - softmax_scale_log2 = LOG2_E - - fastdiv_mods = None - if const_expr(aux_tensors is not None): - seqlen_q = cute.size(mQ.shape[0]) - seqlen_k = cute.size(mK.shape[0]) - seqlen_q_divmod = FastDivmodDivisor(seqlen_q) - seqlen_k_divmod = FastDivmodDivisor(seqlen_k) - fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod) - - qhead_per_kvhead_divmod = None - if const_expr(self.qhead_per_kvhead > 1): - qhead_per_kvhead_divmod = FastDivmodDivisor(self.qhead_per_kvhead) - - self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None) - - self.kernel( - tma_tensor_Q, - tma_tensor_K, - tma_tensor_V, - tma_tensor_dO, - tma_tensor_dK if const_expr(self.qhead_per_kvhead == 1) else mdK, - tma_tensor_dV if const_expr(self.qhead_per_kvhead == 1) else mdV, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_dO, - tma_atom_dK, - tma_atom_dV, - mLSE, - mdPsum, - mdQaccum, - self.sQ_layout, - self.sK_layout, - self.sV_layout, - self.sPdS_layout, - self.sdO_layout, - self.sdQaccum_layout, - self.sdKVaccum_layout, - self.r2s_tiled_copy_dQaccum, - self.r2s_tiled_copy_dKVaccum, - tiled_mma_SdP, - tiled_mma_dK, - tiled_mma_dV, - tiled_mma_dQ, - softmax_scale_log2, - softmax_scale, - tile_sched_params, - TileScheduler, - SharedStorage, - aux_tensors, - fastdiv_mods, - blocksparse_tensors, - qhead_per_kvhead_divmod, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - smem=SharedStorage.size_in_bytes(), - stream=stream, - min_blocks_per_mp=1, - ) - - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mdK: cute.Tensor, - mdV: cute.Tensor, - tma_atom_Q: cute.CopyAtom, - tma_atom_K: cute.CopyAtom, - tma_atom_V: cute.CopyAtom, - tma_atom_dO: cute.CopyAtom, - tma_atom_dK: cute.CopyAtom, - tma_atom_dV: cute.CopyAtom, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - mdQaccum: cute.Tensor, - sQ_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sPdS_layout: cute.ComposedLayout, - sdO_layout: cute.ComposedLayout, - sdQaccum_layout: cute.Layout, - sdKVaccum_layout: cute.Layout, - r2s_tiled_copy_dQaccum: cute.TiledCopy, - r2s_tiled_copy_dKVaccum: cute.TiledCopy, - tiled_mma_SdP: cute.TiledMma, - tiled_mma_dK: cute.TiledMma, - tiled_mma_dV: cute.TiledMma, - tiled_mma_dQ: cute.TiledMma, - softmax_scale_log2, - softmax_scale, - tile_sched_params: ParamsBase, - TileScheduler: cutlass.Constexpr[Callable], - SharedStorage: cutlass.Constexpr[Callable], - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - blocksparse_tensors: Optional[BlockSparseTensors] = None, - qhead_per_kvhead_divmod: Optional[FastDivmodDivisor] = None, - ): - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - - # prefetch TMA descriptors - if warp_idx == 0: - cpasync.prefetch_descriptor(tma_atom_Q) - cpasync.prefetch_descriptor(tma_atom_K) - cpasync.prefetch_descriptor(tma_atom_V) - cpasync.prefetch_descriptor(tma_atom_dO) - - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage) - - pipeline_producer_group = cutlass.pipeline.CooperativeGroup(cutlass.pipeline.Agent.Thread) - pipeline_consumer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, self.num_mma_threads // cute.arch.WARP_SIZE - ) - pipeline_Q = pipeline.PipelineTmaAsync.create( - barrier_storage=storage.mbar_ptr_Q.data_ptr(), - num_stages=self.Q_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group, - tx_count=self.tma_copy_bytes["Q"] + self.tma_copy_bytes["LSE"], - defer_sync=True, - ) - pipeline_dO = pipeline.PipelineTmaAsync.create( - barrier_storage=storage.mbar_ptr_dO.data_ptr(), - num_stages=self.dO_stage, - producer_group=pipeline_producer_group, - consumer_group=pipeline_consumer_group, - tx_count=self.tma_copy_bytes["dO"] + self.tma_copy_bytes["dPsum"], - defer_sync=False, - ) - - sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner) - sdO = storage.sdO.get_tensor(sdO_layout.outer, swizzle=sdO_layout.inner) - sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner) - sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner) - sP = None - if const_expr(not self.mma_dkv_is_rs): - sP = storage.sP.get_tensor(sPdS_layout.outer, swizzle=sPdS_layout.inner) - sdS = storage.sdS.get_tensor(sPdS_layout.outer, swizzle=sPdS_layout.inner) - sLSE = storage.sLSE.get_tensor( - cute.make_layout( - (self.tile_m, self.Q_stage), - stride=(1, cute.round_up(self.tile_m, 64)), - ) - ) - sdPsum = storage.sdPsum.get_tensor( - cute.make_layout( - (self.tile_m, self.dO_stage), - stride=(1, cute.round_up(self.tile_m, 64)), - ) - ) - sdQaccum = storage.sdQaccum.get_tensor(sdQaccum_layout) - - block_info = BlockInfo( - self.tile_m, - self.tile_n, - self.is_causal, - self.is_local, - False, # is_split_kv - None, - None, - qhead_per_kvhead_packgqa=1, - ) - SeqlenInfoCls = partial( - SeqlenInfoQK.create, - seqlen_q_static=mQ.shape[0], - seqlen_k_static=mK.shape[0], - mCuSeqlensQ=None, - mCuSeqlensK=None, - mSeqUsedQ=None, - mSeqUsedK=None, - ) - AttentionMaskCls = partial( - AttentionMask, - self.tile_m, - self.tile_n, - window_size_left=None, - window_size_right=None, - swap_AB=self.SdP_swapAB, - ) - TileSchedulerCls = partial(TileScheduler.create, tile_sched_params) - - if warp_idx < 4: - cute.arch.warpgroup_reg_dealloc(self.num_producer_regs) - if warp_idx == 0: - self.load( - mQ, - mK, - mV, - mdO, - mLSE, - mdPsum, - sQ, - sK, - sV, - sdO, - sLSE, - sdPsum, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_dO, - pipeline_Q, - pipeline_dO, - block_info, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - qhead_per_kvhead_divmod, - ) - if warp_idx == 1: - for warp_group_idx in cutlass.range(self.num_mma_warp_groups): - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierBwd.dQEmptyWG0) + warp_group_idx, - number_of_threads=self.num_threads_per_warp_group + cute.arch.WARP_SIZE, - ) - self.dQaccum_store( - mdQaccum, - sdQaccum, - block_info, - TileSchedulerCls, - SeqlenInfoCls, - blocksparse_tensors, - ) - else: - cute.arch.warpgroup_reg_alloc(self.num_mma_regs) - tidx, _, _ = cute.arch.thread_idx() - tidx = tidx - 128 - self.mma( - tiled_mma_SdP, - tiled_mma_dK, - tiled_mma_dV, - tiled_mma_dQ, - mdK, - mdV, - mdQaccum, - sQ, - sK, - sV, - sdO, - sP, - sdS, - sLSE, - sdPsum, - sdQaccum, - pipeline_Q, - pipeline_dO, - tidx, - tma_atom_dK, - tma_atom_dV, - r2s_tiled_copy_dQaccum, - r2s_tiled_copy_dKVaccum, - sdKVaccum_layout, - softmax_scale_log2, - softmax_scale, - block_info, - SeqlenInfoCls, - AttentionMaskCls, - TileSchedulerCls, - aux_tensors, - fastdiv_mods, - blocksparse_tensors, - qhead_per_kvhead_divmod, - ) - - @cute.jit - def load( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mdO: cute.Tensor, - mLSE: cute.Tensor, - mdPsum: cute.Tensor, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - sdO: cute.Tensor, - sLSE: cute.Tensor, - sdPsum: cute.Tensor, - tma_atom_Q: cute.CopyAtom, - tma_atom_K: cute.CopyAtom, - tma_atom_V: cute.CopyAtom, - tma_atom_dO: cute.CopyAtom, - pipeline_Q: cutlass.pipeline.PipelineAsync, - pipeline_dO: cutlass.pipeline.PipelineAsync, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - qhead_per_kvhead_divmod: Optional[FastDivmodDivisor] = None, - ): - warp_idx_in_wg = cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4 - - if warp_idx_in_wg == 0: - producer_state_Q = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.Q_stage - ) - producer_state_dO = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.dO_stage - ) - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - head_idx_kv = ( - head_idx - if const_expr(self.qhead_per_kvhead == 1) - else head_idx // qhead_per_kvhead_divmod - ) - mK_cur = mK[None, None, head_idx_kv, batch_idx] - gK = cute.local_tile(mK_cur, (self.tile_n, self.tile_hdim), (n_block, 0)) - mV_cur = mV[None, None, head_idx_kv, batch_idx] - gV = cute.local_tile(mV_cur, (self.tile_n, self.tile_hdimv), (n_block, 0)) - - mQ_cur = mQ[None, None, head_idx, batch_idx] - gQ = cute.local_tile(mQ_cur, (self.tile_m, self.tile_hdim), (None, 0)) - mdO_cur = mdO[None, None, head_idx, batch_idx] - gdO = cute.local_tile(mdO_cur, (self.tile_m, self.tile_hdimv), (None, 0)) - mLSE_cur = mLSE[None, head_idx, batch_idx] - gLSE = cute.local_tile(mLSE_cur, (self.tile_m,), (None,)) - mdPsum_cur = mdPsum[None, head_idx, batch_idx] - gdPsum = cute.local_tile(mdPsum_cur, (self.tile_m,), (None,)) - - load_K, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_K, 0, cute.make_layout(1), gK, sK, single_stage=True - ) - load_V, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_V, 0, cute.make_layout(1), gV, sV, single_stage=True - ) - load_Q, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_Q, 0, cute.make_layout(1), gQ, sQ - ) - load_Q = copy_utils.tma_producer_copy_fn(load_Q, pipeline_Q) - load_dO, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_dO, 0, cute.make_layout(1), gdO, sdO - ) - load_dO = copy_utils.tma_producer_copy_fn(load_dO, pipeline_dO) - load_LSE = copy_utils.cpasync_bulk_get_copy_fn(gLSE, sLSE) - load_LSE = copy_utils.tma_producer_copy_fn(load_LSE, pipeline_Q) - load_dPsum = copy_utils.cpasync_bulk_get_copy_fn(gdPsum, sdPsum) - load_dPsum = copy_utils.tma_producer_copy_fn(load_dPsum, pipeline_dO) - - m_block_min, m_block_max = block_info.get_m_block_min_max(seqlen, n_block) - - if const_expr(not self.use_block_sparsity): - total_m_block_cnt = m_block_max - m_block_min - process_tile = const_expr(not self.is_local) or m_block_min < m_block_max - else: - total_m_block_cnt = get_total_q_block_count_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = total_m_block_cnt > Int32(0) - - if process_tile: - if const_expr(not self.use_block_sparsity): - first_m_block = m_block_min - pipeline_Q.producer_acquire( - producer_state_Q, extra_tx_count=self.tma_copy_bytes["K"] - ) - load_K(tma_bar_ptr=pipeline_Q.producer_get_barrier(producer_state_Q)) - load_Q(first_m_block, producer_state=producer_state_Q) - with cute.arch.elect_one(): - load_LSE(first_m_block, producer_state=producer_state_Q) - producer_state_dO_cur = ( - producer_state_dO - if const_expr(self.Q_stage != self.dO_stage) - else producer_state_Q - ) - pipeline_dO.producer_acquire( - producer_state_dO_cur, extra_tx_count=self.tma_copy_bytes["V"] - ) - load_V(tma_bar_ptr=pipeline_dO.producer_get_barrier(producer_state_dO_cur)) - load_dO(first_m_block, producer_state=producer_state_dO_cur) - with cute.arch.elect_one(): - load_dPsum(first_m_block, producer_state=producer_state_dO_cur) - producer_state_Q.advance() - producer_state_dO.advance() - - for m_block in cutlass.range(m_block_min + 1, m_block_max, unroll=1): - pipeline_Q.producer_acquire(producer_state_Q) - load_Q(m_block, producer_state=producer_state_Q) - with cute.arch.elect_one(): - load_LSE(m_block, producer_state=producer_state_Q) - producer_state_dO_cur = ( - producer_state_dO - if const_expr(self.Q_stage != self.dO_stage) - else producer_state_Q - ) - pipeline_dO.producer_acquire(producer_state_dO_cur) - load_dO(m_block, producer_state=producer_state_dO_cur) - with cute.arch.elect_one(): - load_dPsum(m_block, producer_state=producer_state_dO_cur) - producer_state_Q.advance() - producer_state_dO.advance() - else: - producer_state_Q, producer_state_dO = produce_block_sparse_q_loads_bwd_sm90( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - producer_state_Q, - producer_state_dO, - pipeline_Q, - pipeline_dO, - load_K, - load_V, - load_Q, - load_dO, - load_LSE, - load_dPsum, - self.tma_copy_bytes["K"], - self.tma_copy_bytes["V"], - Q_stage_eq_dO_stage=(self.Q_stage == self.dO_stage), - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - - tile_scheduler.prefetch_next_work() - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def apply_score_mod( - self, - acc_S: cute.Tensor, - thr_mma_SdP: cute.core.ThrMma, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen_info: SeqlenInfoQK, - aux_tensors=None, - fastdiv_mods=(None, None), - ): - # [NOTE] SdP_swapAB: swapAB transposes the tile, so use (n, m) indexing - cS = cute.make_identity_tensor( - (self.tile_n, self.tile_m) if self.SdP_swapAB else (self.tile_m, self.tile_n) - ) - cS = cute.domain_offset( - (n_block * self.tile_n, m_block * self.tile_m) - if self.SdP_swapAB - else (m_block * self.tile_m, n_block * self.tile_n), - cS, - ) - tScS = thr_mma_SdP.partition_C(cS) - - apply_score_mod_inner( - acc_S, - tScS, - self.score_mod, - batch_idx, - head_idx, - softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info, - constant_q_idx=None, - qhead_per_kvhead=self.qhead_per_kvhead, - transpose_indices=self.SdP_swapAB, - ) - - @cute.jit - def apply_score_mod_bwd( - self, - grad_tensor: cute.Tensor, - score_tensor: cute.Tensor, - thr_mma_SdP: cute.core.ThrMma, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen_info: SeqlenInfoQK, - aux_tensors=None, - fastdiv_mods=(None, None), - ): - cS = cute.make_identity_tensor( - (self.tile_n, self.tile_m) if self.SdP_swapAB else (self.tile_m, self.tile_n) - ) - cS = cute.domain_offset( - (n_block * self.tile_n, m_block * self.tile_m) - if self.SdP_swapAB - else (m_block * self.tile_m, n_block * self.tile_n), - cS, - ) - tScS = thr_mma_SdP.partition_C(cS) - - apply_score_mod_bwd_inner( - grad_tensor, - score_tensor, - tScS, - self.score_mod_bwd, - batch_idx, - head_idx, - softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info, - constant_q_idx=None, - qhead_per_kvhead=self.qhead_per_kvhead, - transpose_indices=self.SdP_swapAB, - ) - - @cute.jit - def mma( - self, - tiled_mma_SdP: cute.TiledMma, - tiled_mma_dK: cute.TiledMma, - tiled_mma_dV: cute.TiledMma, - tiled_mma_dQ: cute.TiledMma, - mdK: cute.Tensor, - mdV: cute.Tensor, - mdQaccum: cute.Tensor, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - sdO: cute.Tensor, - sP: Optional[cute.Tensor], - sdS: cute.Tensor, - sLSE: cute.Tensor, - sdPsum: cute.Tensor, - sdQaccum: cute.Tensor, - pipeline_Q: cutlass.pipeline.PipelineAsync, - pipeline_dO: cutlass.pipeline.PipelineAsync, - tidx: Int32, - tma_atom_dK: cute.CopyAtom, - tma_atom_dV: cute.CopyAtom, - r2s_tiled_copy_dQaccum: cute.TiledCopy, - r2s_tiled_copy_dKVaccum: cute.TiledCopy, - sdKVaccum_layout: cute.Layout, - softmax_scale_log2: Float32, - softmax_scale: Float32, - block_info: BlockInfo, - SeqlenInfoCls: Callable, - AttentionMaskCls: Callable, - TileSchedulerCls: Callable, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - blocksparse_tensors: Optional[BlockSparseTensors] = None, - qhead_per_kvhead_divmod: Optional[FastDivmodDivisor] = None, - ): - warp_group_idx = cute.arch.make_warp_uniform(tidx // self.num_threads_per_warp_group) - warp_group_thread_layout = cute.make_layout( - self.num_mma_warp_groups, stride=self.num_threads_per_warp_group - ) - thr_mma_SdP = tiled_mma_SdP.get_slice(tidx) - wg_mma_SdP = tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx)) - wg_mma_dK = tiled_mma_dK.get_slice(warp_group_thread_layout(warp_group_idx)) - wg_mma_dV = tiled_mma_dV.get_slice(warp_group_thread_layout(warp_group_idx)) - wg_mma_dQ = tiled_mma_dQ.get_slice(warp_group_thread_layout(warp_group_idx)) - # S = Q @ K.T - tSrQ, tSrK = mma_partition_fragment_AB(wg_mma_SdP, sQ, sK, self.SdP_swapAB) - # dP = dO @ V.T - tdPrdO, tdPrV = mma_partition_fragment_AB(wg_mma_SdP, sdO, sV, self.SdP_swapAB) - # dV += P.T @ dO - sPt = utils.transpose_view(sP) if sP is not None else None - sdOt = utils.transpose_view(sdO) - tdVrPt, tdVrdOt = mma_partition_fragment_AB(wg_mma_dV, sPt, sdOt, self.dKV_swapAB) - # dK += dS.T @ Q - sdSt = utils.transpose_view(sdS) - sQt = utils.transpose_view(sQ) - tdKrdSt, tdKrQt = mma_partition_fragment_AB(wg_mma_dK, sdSt, sQt, self.dKV_swapAB) - # dQ = dS @ K - sKt = utils.transpose_view(sK) - tdQrdS, tdQrKt = mma_partition_fragment_AB(wg_mma_dQ, sdS, sKt, self.dQ_swapAB) - - # Smem copy atom tiling - smem_copy_atom_PdS = utils.get_smem_store_atom( - self.arch, self.dtype, transpose=self.SdP_swapAB - ) - smem_thr_copy_PdS = cute.make_tiled_copy_C(smem_copy_atom_PdS, tiled_mma_SdP).get_slice( - tidx - ) - tPsP = None - if const_expr(sP is not None): - tPsP = smem_thr_copy_PdS.partition_D(sP if const_expr(not self.SdP_swapAB) else sPt) - tdSsdS = smem_thr_copy_PdS.partition_D(sdS if const_expr(not self.SdP_swapAB) else sdSt) - - sLSE_mma = cute.make_tensor( - sLSE.iterator, - cute.make_layout( - (self.tile_m, self.tile_n, self.Q_stage), - stride=(1, 0, cute.round_up(self.tile_m, 64)), - ), - ) - sdPsum_mma = cute.make_tensor( - sdPsum.iterator, - cute.make_layout( - (self.tile_m, self.tile_n, self.dO_stage), - stride=(1, 0, cute.round_up(self.tile_m, 64)), - ), - ) - if const_expr(self.SdP_swapAB): - sLSE_mma = utils.transpose_view(sLSE_mma) - sdPsum_mma = utils.transpose_view(sdPsum_mma) - LSEslice = (None, 0, None) if const_expr(not self.SdP_swapAB) else (0, None, None) - tLSEsLSE = utils.make_acc_tensor_mn_view(thr_mma_SdP.partition_C(sLSE_mma))[LSEslice] - tLSEsdPsum = utils.make_acc_tensor_mn_view(thr_mma_SdP.partition_C(sdPsum_mma))[LSEslice] - - smem_thr_copy_dQaccum = r2s_tiled_copy_dQaccum.get_slice(tidx) - tdQsdQaccum = smem_thr_copy_dQaccum.partition_D(sdQaccum) - - dV_shape = (self.tile_n, self.tile_hdimv) - acc_dV = cute.make_fragment( - tiled_mma_dV.partition_shape_C(dV_shape if not self.dKV_swapAB else dV_shape[::-1]), - Float32, - ) - dK_shape = (self.tile_n, self.tile_hdim) - acc_dK = cute.make_fragment( - tiled_mma_dK.partition_shape_C(dK_shape if not self.dKV_swapAB else dK_shape[::-1]), - Float32, - ) - - mma_qk_fn = partial( - gemm_zero_init, - tiled_mma_SdP, - (self.tile_m, self.tile_n), - tSrQ, - tSrK, - swap_AB=self.SdP_swapAB, - ) - mma_dov_fn = partial( - gemm_zero_init, - tiled_mma_SdP, - (self.tile_m, self.tile_n), - tdPrdO, - tdPrV, - swap_AB=self.SdP_swapAB, - ) - if const_expr(not self.mma_dkv_is_rs): - mma_pdo_fn = partial( - gemm_w_idx, tiled_mma_dV, acc_dV, tdVrPt, tdVrdOt, swap_AB=self.dKV_swapAB - ) - mma_dsq_fn = partial( - gemm_w_idx, tiled_mma_dK, acc_dK, tdKrdSt, tdKrQt, swap_AB=self.dKV_swapAB - ) - else: - assert not self.dKV_swapAB - mma_pdo_fn = partial(gemm_w_idx, tiled_mma_dV, acc_dV, tCrB=tdVrdOt) - mma_dsq_fn = partial(gemm_w_idx, tiled_mma_dK, acc_dK, tCrB=tdKrQt) - mma_dsk_fn = partial( - gemm_zero_init, - tiled_mma_dQ, - (self.tile_m, self.tile_hdim), - tdQrdS, - tdQrKt, - swap_AB=self.dQ_swapAB, - ) - - mma_one_m_block_all = partial( - self.mma_one_m_block, - warp_group_idx=warp_group_idx, - mma_qk_fn=mma_qk_fn, - mma_dov_fn=mma_dov_fn, - mma_pdo_fn=mma_pdo_fn, - mma_dsq_fn=mma_dsq_fn, - mma_dsk_fn=mma_dsk_fn, - pipeline_Q=pipeline_Q, - pipeline_dO=pipeline_dO, - tLSEsLSE=tLSEsLSE, - tLSEsdPsum=tLSEsdPsum, - tPsP=tPsP, - tdSsdS=tdSsdS, - tdQsdQaccum=tdQsdQaccum, - smem_thr_copy_PdS=smem_thr_copy_PdS, - smem_thr_copy_dQaccum=smem_thr_copy_dQaccum, - softmax_scale_log2=softmax_scale_log2, - # acc_dV=acc_dV, - # acc_dK=acc_dK, - ) - - consumer_state_Q = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.Q_stage - ) - consumer_state_dO = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.dO_stage - ) - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - mask = AttentionMaskCls(seqlen) - m_block_min, m_block_max = block_info.get_m_block_min_max(seqlen, n_block) - - if const_expr(not self.use_block_sparsity): - process_tile = const_expr(not self.is_local) or m_block_min < m_block_max - else: - total_m_block_cnt = get_total_q_block_count_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = total_m_block_cnt > Int32(0) - - if process_tile: - if const_expr(not self.use_block_sparsity): - mask_fn = partial( - mask.apply_mask, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - thr_mma=thr_mma_SdP, - mask_seqlen=True, - mask_causal=self.is_causal, - mask_local=self.is_local, - mask_mod=self.mask_mod, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - dKV_accumulate = False - for m_block in cutlass.range(m_block_min, m_block_max, unroll=1): - consumer_state_Q, consumer_state_dO = mma_one_m_block_all( - m_block, - consumer_state_Q, - consumer_state_dO, - mask_fn=mask_fn, - dKV_accumulate=dKV_accumulate, - thr_mma_SdP=thr_mma_SdP, - batch_idx=batch_idx, - head_idx=head_idx, - n_block=n_block, - softmax_scale=softmax_scale, - seqlen=seqlen, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - dKV_accumulate = True - else: - consumer_state_Q, consumer_state_dO = consume_block_sparse_mma_bwd_sm90( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - consumer_state_Q, - consumer_state_dO, - mma_one_m_block_all, - mask, - self.mask_mod, - is_causal=self.is_causal, - is_local=self.is_local, - thr_mma_SdP=thr_mma_SdP, - softmax_scale=softmax_scale, - seqlen=seqlen, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - if const_expr(self.qhead_per_kvhead == 1): - acc_dK.store(acc_dK.load() * softmax_scale) - self.epilogue_dKV( - acc_dV, - mdV, - sV, - acc_dK, - mdK, - sK, - seqlen, - tma_atom_dK, - tma_atom_dV, - tiled_mma_dK, - tiled_mma_dV, - r2s_tiled_copy_dKVaccum, - sdKVaccum_layout, - tidx, - n_block, - head_idx, - batch_idx, - qhead_per_kvhead_divmod, - ) - else: - # Block sparsity: KV tile with zero Q blocks produces no dK/dV; write zeros. - if const_expr(self.use_block_sparsity): - acc_dK.fill(0.0) - acc_dV.fill(0.0) - self.epilogue_dKV( - acc_dV, - mdV, - sV, - acc_dK, - mdK, - sK, - seqlen, - tma_atom_dK, - tma_atom_dV, - tiled_mma_dK, - tiled_mma_dV, - r2s_tiled_copy_dKVaccum, - sdKVaccum_layout, - tidx, - n_block, - head_idx, - batch_idx, - qhead_per_kvhead_divmod, - ) - - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def mma_one_m_block( - self, - m_block: Int32, - consumer_state_Q: cutlass.pipeline.PipelineState | pipeline.PipelineStateSimple, - consumer_state_dO: cutlass.pipeline.PipelineState | pipeline.PipelineStateSimple, - warp_group_idx: Int32, - mma_qk_fn: Callable, - mma_dov_fn: Callable, - mma_pdo_fn: Callable, - mma_dsq_fn: Callable, - mma_dsk_fn: Callable, - pipeline_Q: cutlass.pipeline.PipelineAsync, - pipeline_dO: cutlass.pipeline.PipelineAsync, - tLSEsLSE: cute.Tensor, - tLSEsdPsum: cute.Tensor, - tPsP: Optional[cute.Tensor], - tdSsdS: Optional[cute.Tensor], - tdQsdQaccum: cute.Tensor, - smem_thr_copy_PdS: cute.TiledCopy, - smem_thr_copy_dQaccum: cute.TiledCopy, - softmax_scale_log2: Float32, - mask_fn: Optional[Callable] = None, - dKV_accumulate: Boolean = True, - thr_mma_SdP: Optional[cute.core.ThrMma] = None, - batch_idx: Int32 = 0, - head_idx: Int32 = 0, - n_block: Int32 = 0, - softmax_scale: Float32 = 1.0, - seqlen: Optional[SeqlenInfoQK] = None, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - ): - consumer_state_dO_cur = ( - consumer_state_dO if const_expr(self.Q_stage == self.dO_stage) else consumer_state_Q - ) - smem_idx_Q = consumer_state_Q.index - smem_idx_dO = consumer_state_dO_cur.index if const_expr(self.dO_stage > 1) else 0 - smem_idx_PdS = smem_idx_Q if const_expr(self.PdS_stage > 1) else 0 - # (1) [GEMM 1] S = Q @ K^T - pipeline_Q.consumer_wait(consumer_state_Q, pipeline_Q.consumer_try_wait(consumer_state_Q)) - acc_S = mma_qk_fn(A_idx=smem_idx_Q, wg_wait=-1) - tLSErLSE = copy_utils.load_s2r(tLSEsLSE[None, smem_idx_Q]) - # (2) [GEMM 2] dP = dO @ V.T - pipeline_dO.consumer_wait( - consumer_state_dO_cur, pipeline_dO.consumer_try_wait(consumer_state_dO_cur) - ) - acc_dP = mma_dov_fn(A_idx=smem_idx_Q, wg_wait=1) - - if const_expr(self.score_mod_bwd is not None): - acc_S_pre = cute.make_fragment_like(acc_S) - cute.autovec_copy(acc_S, acc_S_pre) - - if const_expr(self.score_mod is not None): - self.apply_score_mod( - acc_S, - thr_mma_SdP, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen, - aux_tensors, - fastdiv_mods, - ) - - # (3) [Pointwise 1] P = exp(S - LSE) - if cutlass.const_expr(mask_fn is not None): - mask_fn(acc_S, m_block=m_block) - acc_S_mn = utils.make_acc_tensor_mn_view(acc_S, transpose=self.SdP_swapAB) - for r in cutlass.range_constexpr(cute.size(acc_S_mn, mode=[0])): - for c in cutlass.range(cute.size(acc_S_mn, mode=[1]), unroll_full=True): - acc_S_mn[r, c] = cute.math.exp2( - acc_S_mn[r, c] * softmax_scale_log2 - tLSErLSE[r], fastmath=True - ) - tLSErdPsum = copy_utils.load_s2r(tLSEsdPsum[None, smem_idx_dO]) - - # Convert P from f32 -> f16 - tdVrP = utils.cvt_f16(utils.make_acc_tensor_frgA_view(acc_S), self.dtype) - # R2S for P - if const_expr(not self.mma_dkv_is_rs): - # sync to ensure P has already been used in the previous iteration before overwriting - if const_expr(self.PdS_stage == 1): - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.PdS), number_of_threads=self.num_mma_threads - ) - tPrP = smem_thr_copy_PdS.retile(tdVrP) - cute.copy(smem_thr_copy_PdS, tPrP, tPsP[None, None, None, smem_idx_PdS]) - - # (4) [Pointwise 2] dS = P*(dP-dPsum) - warpgroup.wait_group(0) - acc_dP_mn = utils.make_acc_tensor_mn_view(acc_dP, transpose=self.SdP_swapAB) - for r in cutlass.range_constexpr(cute.size(acc_dP_mn, mode=[0])): - for c in cutlass.range(cute.size(acc_dP_mn, mode=[1]), unroll_full=True): - acc_dP_mn[r, c] = acc_S_mn[r, c] * (acc_dP_mn[r, c] - tLSErdPsum[r]) - - if const_expr(self.score_mod_bwd is not None): - self.apply_score_mod_bwd( - acc_dP, - acc_S_pre, - thr_mma_SdP, - batch_idx, - head_idx, - m_block, - n_block, - softmax_scale, - seqlen, - aux_tensors, - fastdiv_mods, - ) - - # Convert dS from f32 -> f16 - tdKrdS = utils.cvt_f16(utils.make_acc_tensor_frgA_view(acc_dP), self.dtype) - - # If there's double buffering on dS, we don't need to sync here. - # Otherwise we might have WG1 writing to dS before WG2 is done reading from it during MmadQ. - # But because both WGs have to sync at the end of the loop and double buffering, - # this race condition is not possible. - # This sync is to ensure (1) P is written in case of !mma_dkv_is_rs and - # (2) dS is already read by the Mma in the previous iteration in case of mma_dkv_is_rs. - if const_expr(not self.mma_dkv_is_rs or (self.PdS_stage == 1 and self.mma_dkv_is_rs)): - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.PdS), number_of_threads=self.num_mma_threads - ) - - # R2S for dS - tdSrdS = smem_thr_copy_PdS.retile(tdKrdS) - cute.copy(smem_thr_copy_PdS, tdSrdS, tdSsdS[None, None, None, smem_idx_PdS]) - - # (5) [GEMM 3] dV += P.T @ dO - if const_expr(not self.mma_dkv_is_rs): - mma_pdo_fn( - A_idx=smem_idx_PdS, B_idx=smem_idx_dO, zero_init=not dKV_accumulate, wg_wait=-1 - ) - else: - mma_pdo_fn(tCrA=tdVrP, B_idx=smem_idx_dO, zero_init=not dKV_accumulate, wg_wait=-1) - - # smem fence to make sure sdS is written before it's read by WGMMA - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.PdS), number_of_threads=self.num_mma_threads - ) - # (6) [GEMM 4] dQ = dS @ K - acc_dQ = mma_dsk_fn(A_idx=smem_idx_PdS, wg_wait=1) - # if cute.arch.thread_idx()[0] == 128: cute.print_tensor(acc_dV) - pipeline_dO.consumer_release(consumer_state_dO_cur) # release dO as dV mma is done - - # (7) [GEMM 5] dK += dS.T @ Q - if const_expr(not self.mma_dkv_is_rs): - mma_dsq_fn( - A_idx=smem_idx_PdS, B_idx=smem_idx_Q, zero_init=not dKV_accumulate, wg_wait=1 - ) - else: - mma_dsq_fn(tCrA=tdKrdS, B_idx=smem_idx_Q, zero_init=not dKV_accumulate, wg_wait=1) - # if cute.arch.thread_idx()[0] == 128: cute.print_tensor(acc_dQ) - - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.dQEmptyWG0) + warp_group_idx, - number_of_threads=self.num_threads_per_warp_group + cute.arch.WARP_SIZE, - ) - tdQrdQaccum_flat = cute.make_tensor(acc_dQ.iterator, cute.make_layout(tdQsdQaccum.shape)) - cute.autovec_copy(tdQrdQaccum_flat, tdQsdQaccum) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierBwd.dQFullWG0) + warp_group_idx, - number_of_threads=self.num_threads_per_warp_group + cute.arch.WARP_SIZE, - ) - - warpgroup.wait_group(0) - # if cute.arch.thread_idx()[0] == 128: cute.print_tensor(acc_dK) - pipeline_Q.consumer_release(consumer_state_Q) - # if cute.arch.thread_idx()[0] % 32 == 0: cute.printf("tidx = {}, m_block = {}, after pipeline_Q consumer release", cute.arch.thread_idx()[0], m_block) - - consumer_state_Q.advance() - consumer_state_dO.advance() - return consumer_state_Q, consumer_state_dO - - @cute.jit - def epilogue_dKV( - self, - acc_dV: cute.Tensor, - mdV: cute.Tensor, - sV: cute.Tensor, - acc_dK: cute.Tensor, - mdK: cute.Tensor, - sK: cute.Tensor, - seqlen: SeqlenInfoQK, - tma_atom_dK: cute.CopyAtom, - tma_atom_dV: cute.CopyAtom, - tiled_mma_dK: cute.TiledMma, - tiled_mma_dV: cute.TiledMma, - r2s_tiled_copy_dKVaccum: cute.TiledCopy, - sdKVaccum_layout: cute.Layout, - tidx: Int32, - n_block: Int32, - head_idx: Int32, - batch_idx: Int32, - qhead_per_kvhead_divmod: Optional[FastDivmodDivisor] = None, - ): - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - - if const_expr(self.qhead_per_kvhead == 1): - rdV = cute.make_fragment_like(acc_dV, self.dtype) - rdV.store(acc_dV.load().to(self.dtype)) - rdK = utils.cvt_f16(acc_dK, self.dtype) - - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - - smem_copy_atom_dKV = cute.make_copy_atom( - cute.nvgpu.warp.StMatrix8x8x16bOp(transpose=self.dKV_swapAB, num_matrices=4), - self.dtype, - ) - smem_thr_copy_dK = cute.make_tiled_copy_C(smem_copy_atom_dKV, tiled_mma_dK).get_slice( - tidx - ) - smem_thr_copy_dV = cute.make_tiled_copy_C(smem_copy_atom_dKV, tiled_mma_dV).get_slice( - tidx - ) - mdV_cur = mdV[None, None, head_idx, batch_idx] - mdK_cur = mdK[None, None, head_idx, batch_idx] - gdK = cute.local_tile(mdK_cur, (self.tile_n, self.tile_hdim), (n_block, 0)) - gdV = cute.local_tile(mdV_cur, (self.tile_n, self.tile_hdimv), (n_block, 0)) - store_dK, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_dK, 0, cute.make_layout(1), sK, gdK, single_stage=True - ) - store_dV, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_dV, 0, cute.make_layout(1), sV, gdV, single_stage=True - ) - - taccdVrdV = smem_thr_copy_dV.retile(rdV) - sdV = sV if const_expr(not self.dKV_swapAB) else utils.transpose_view(sV) - taccdVsdV = smem_thr_copy_dV.partition_D(sdV) - cute.copy(smem_copy_atom_dKV, taccdVrdV, taccdVsdV) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - if warp_idx == 4: - store_dV() - taccdKrdK = smem_thr_copy_dK.retile(rdK) - sdK = sK if const_expr(not self.dKV_swapAB) else utils.transpose_view(sK) - taccdKsdK = smem_thr_copy_dK.partition_D(sdK) - cute.copy(smem_copy_atom_dKV, taccdKrdK, taccdKsdK) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - if warp_idx == 4: - store_dK() - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=True) - else: - head_idx_kv = head_idx // qhead_per_kvhead_divmod - - mdKaccum_cur = mdK[None, head_idx_kv, batch_idx] - gdKaccum_ = cute.local_tile(mdKaccum_cur, (self.tile_n * self.tile_hdim,), (n_block,)) - gdKaccum = cute.flat_divide( - gdKaccum_, (self.tile_n * self.tile_hdim // self.num_mma_warp_groups,) - ) - - mdVaccum_cur = mdV[None, head_idx_kv, batch_idx] - gdVaccum_ = cute.local_tile(mdVaccum_cur, (self.tile_n * self.tile_hdimv,), (n_block,)) - gdVaccum = cute.flat_divide( - gdVaccum_, (self.tile_n * self.tile_hdimv // self.num_mma_warp_groups,) - ) - - sdKVaccum = cute.make_tensor( - cute.recast_ptr(sV.iterator, dtype=Float32), - sdKVaccum_layout, - ) - - smem_thr_copy_dKVaccum = r2s_tiled_copy_dKVaccum.get_slice(tidx) - tdKsdKVaccum = smem_thr_copy_dKVaccum.partition_D(sdKVaccum) - - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - - tdKrdKaccum_flat = cute.make_tensor( - acc_dK.iterator, cute.make_layout(tdKsdKVaccum.shape) - ) - cute.autovec_copy(tdKrdKaccum_flat, tdKsdKVaccum) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - - if warp_idx == 4: - with cute.arch.elect_one(): - for wg_idx in cutlass.range_constexpr(self.num_mma_warp_groups): - copy_utils.cpasync_reduce_bulk_add_f32( - sdKVaccum[None, wg_idx].iterator, - gdKaccum[None, wg_idx].iterator, - self.tma_copy_bytes["dKacc"] // self.num_mma_warp_groups, - ) - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=True) - - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - - tdVrdVaccum_flat = cute.make_tensor( - acc_dV.iterator, cute.make_layout(tdKsdKVaccum.shape) - ) - cute.autovec_copy(tdVrdVaccum_flat, tdKsdKVaccum) - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_mma_threads - ) - - if warp_idx == 4: - with cute.arch.elect_one(): - for wg_idx in cutlass.range_constexpr(self.num_mma_warp_groups): - copy_utils.cpasync_reduce_bulk_add_f32( - sdKVaccum[None, wg_idx].iterator, - gdVaccum[None, wg_idx].iterator, - self.tma_copy_bytes["dVacc"] // self.num_mma_warp_groups, - ) - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=True) - - @cute.jit - def dQaccum_store( - self, - mdQaccum: cute.Tensor, - sdQaccum: cute.Tensor, - block_info: BlockInfo, - TileSchedulerCls: cutlass.Constexpr[Callable], - SeqlenInfoCls: cutlass.Constexpr[Callable], - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - n_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - mdQaccum_cur = mdQaccum[None, head_idx, batch_idx] - gdQaccum_ = cute.local_tile(mdQaccum_cur, (self.tile_m * self.tile_hdim,), (None,)) - # (M * K / WG, WG, _) - gdQaccum = cute.flat_divide( - gdQaccum_, (self.tile_m * self.tile_hdim // self.num_mma_warp_groups,) - ) - m_block_min, m_block_max = block_info.get_m_block_min_max(seqlen, n_block) - if const_expr(not self.use_block_sparsity): - process_tile = const_expr(not self.is_local) or m_block_min < m_block_max - loop_count = m_block_max - m_block_min - else: - total_block_cnt = get_total_q_block_count_bwd( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - ) - process_tile = total_block_cnt > Int32(0) - - if process_tile: - if const_expr(not self.use_block_sparsity): - for iter_idx in cutlass.range(loop_count, unroll=1): - m_block = m_block_min + iter_idx - m_block_safe = m_block - - for warp_group_idx in cutlass.range_constexpr(self.num_mma_warp_groups): - cute.arch.barrier( - barrier_id=int(NamedBarrierBwd.dQFullWG0) + warp_group_idx, - number_of_threads=self.num_threads_per_warp_group - + cute.arch.WARP_SIZE, - ) - with cute.arch.elect_one(): - copy_utils.cpasync_reduce_bulk_add_f32( - sdQaccum[None, warp_group_idx].iterator, - gdQaccum[None, warp_group_idx, m_block_safe].iterator, - self.tma_copy_bytes["dQ"], - ) - cute.arch.cp_async_bulk_commit_group() - for warp_group_idx in cutlass.range_constexpr(self.num_mma_warp_groups): - cute.arch.cp_async_bulk_wait_group( - self.num_mma_warp_groups - 1 - warp_group_idx, read=True - ) - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierBwd.dQEmptyWG0) + warp_group_idx, - number_of_threads=self.num_threads_per_warp_group - + cute.arch.WARP_SIZE, - ) - else: - dQaccum_store_block_sparse_bwd_sm90( - blocksparse_tensors, - batch_idx, - head_idx, - n_block, - sdQaccum, - gdQaccum, - subtile_factor=self.subtile_factor, - m_block_max=m_block_max, - num_mma_warp_groups=self.num_mma_warp_groups, - num_threads_per_warp_group=self.num_threads_per_warp_group, - tma_copy_bytes_dQ=self.tma_copy_bytes["dQ"], - ) - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd.py deleted file mode 100644 index edefbb03ab42..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd.py +++ /dev/null @@ -1,2468 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# A reimplementation of -# https://github.com/Dao-AILab/flash-attention/blob/main/hopper/flash_fwd_kernel_sm80.h -# and https://github.com/Dao-AILab/flash-attention/blob/main/hopper/flash_fwd_kernel_sm90.h -# from Cutlass C++ to Cute-DSL. -# Built on Cute-DSL example: https://github.com/NVIDIA/cutlass/blob/main/examples/python/CuTeDSL/ampere/flash_attention_v2.py - -import math -from types import SimpleNamespace -from typing import Type, Callable, Optional -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass import Float32, Int32, const_expr -from cutlass.cute.nvgpu import cpasync, warp, warpgroup -import cutlass.utils as utils_basic -from cutlass.utils import LayoutEnum -import cutlass.utils.hopper_helpers as sm90_utils_basic - - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.ampere_helpers as sm80_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.hopper_helpers as sm90_utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -from .mask import AttentionMask -from .softmax import Softmax, apply_score_mod_inner -from .seqlen_info import SeqlenInfoQK -from .block_info import BlockInfo -from .block_sparsity import BlockSparseTensors -from .block_sparse_utils import ( - produce_block_sparse_loads, - consume_block_sparse_loads, -) -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.pipeline as pipeline -from .pack_gqa import PackGQA -from .named_barrier import NamedBarrierFwd -from .tile_scheduler import ( - TileSchedulerArguments, - SingleTileScheduler, - SingleTileLPTScheduler, - SingleTileVarlenScheduler, - ParamsBase, -) -from cutlass.cute import FastDivmodDivisor - - -class FlashAttentionForwardBase: - arch: int = 80 - - def __init__( - self, - dtype: Type[cutlass.Numeric], - head_dim: int, - head_dim_v: Optional[int] = None, - qhead_per_kvhead: int = 1, - is_causal: bool = False, - is_local: bool = False, - pack_gqa: bool = True, - tile_m: int = 128, - tile_n: int = 128, - num_stages: int = 1, - num_threads: int = 128, - Q_in_regs: bool = False, - score_mod: Optional[cutlass.Constexpr] = None, - mask_mod: Optional[cutlass.Constexpr] = None, - has_aux_tensors: bool = False, - ): - """Initializes the configuration for a flash attention kernel. - - All contiguous dimensions must be at least 16 bytes aligned, which means that the head dimension - should be a multiple of 8. - - :param head_dim: head dimension - :type head_dim: int - :param tile_m: m block size - :type tile_m: int - :param tile_n: n block size - :type tile_n: int - :param num_threads: number of threads - :type num_threads: int - :param is_causal: is causal - :param score_mod: A callable that takes the attention scores and applies a modification. - Callable signature: ``score_mod(scores, batch_idx, head_idx, q_idx, kv_idx, aux_tensors) -> Any`` - :param mask_mod: A callable that takes the attention scores and returns a boolean representing whether that score should be masked. - Callable signature: ``mask_mod(batch_idx, head_idx, q_idx, kv_idx, aux_tensors) -> Boolean`` - """ - self.dtype = dtype - # padding head_dim to a multiple of 16 as k_block_size - hdim_multiple_of = 16 - self.tile_hdim = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - head_dim_v = head_dim_v if head_dim_v is not None else head_dim - self.same_hdim_kv = head_dim == head_dim_v - self.tile_hdimv = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of) - # Can save registers (and hence be faster) if we don't have to check hdim predication - self.check_hdim_oob = head_dim != self.tile_hdim - self.check_hdim_v_oob = head_dim_v != self.tile_hdimv - self.qhead_per_kvhead = qhead_per_kvhead - self.is_causal = is_causal - self.is_local = is_local - self.pack_gqa = pack_gqa - self.tile_m = tile_m - self.tile_n = tile_n - self.num_threads = num_threads - self.num_stages = num_stages - self.Q_in_regs = Q_in_regs - self.score_mod = score_mod - self.mask_mod = mask_mod - self.qk_acc_dtype = Float32 - if const_expr(has_aux_tensors): - self.vec_size: cutlass.Constexpr = 1 - else: - self.vec_size: cutlass.Constexpr = 2 - - @staticmethod - def can_implement( - dtype, - head_dim, - head_dim_v, - tile_m, - tile_n, - num_stages, - num_threads, - is_causal, - Q_in_regs=False, - ) -> bool: - """Check if the kernel can be implemented with the given parameters. - - :param dtype: data type - :type dtype: cutlass.Numeric - :param head_dim: head dimension - :type head_dim: int - :param tile_m: m block size - :type tile_m: int - :param tile_n: n block size - :type tile_n: int - :param num_threads: number of threads - :type num_threads: int - :param is_causal: is causal - :type is_causal: bool - - :return: True if the kernel can be implemented, False otherwise - :rtype: bool - """ - if dtype not in [cutlass.Float16, cutlass.BFloat16]: - return False - if head_dim % 8 != 0: - return False - if head_dim_v % 8 != 0: - return False - if tile_n % 16 != 0: - return False - if num_threads % 32 != 0: - return False - # Check if block size setting is out of shared memory capacity - # Shared memory usage: Q tile + (K tile + V tile) where K and V use the same tile size - smem_usage_Q = tile_m * head_dim * 2 - smem_usage_K = tile_n * head_dim * num_stages * 2 - smem_usage_V = tile_n * head_dim_v * num_stages * 2 - smem_usage_QV = ( - (smem_usage_Q + smem_usage_V) if not Q_in_regs else max(smem_usage_Q, smem_usage_V) - ) - smem_usage = smem_usage_QV + smem_usage_K - # TODO: sm86 and sm89 - smem_capacity = utils_basic.get_smem_capacity_in_bytes("sm_80") - if smem_usage > smem_capacity: - return False - # Check if twice the block size is divisible by the number of threads - if (tile_m * 2) % num_threads != 0: - return False - return True - - def _check_type( - self, - mQ_type: Type[cutlass.Numeric], - mK_type: Type[cutlass.Numeric], - mV_type: Type[cutlass.Numeric], - mO_type: Type[cutlass.Numeric], - mLSE_type: Type[cutlass.Numeric] | None, - mCuSeqlensQ_type: Type[cutlass.Numeric] | None, - mCuSeqlensK_type: Type[cutlass.Numeric] | None, - mSeqUsedQ_type: Type[cutlass.Numeric] | None, - mSeqUsedK_type: Type[cutlass.Numeric] | None, - ): - # Get the data type and check if it is fp16 or bf16 - if const_expr(not (mQ_type == mK_type == mV_type == mO_type)): - raise TypeError("All tensors must have the same data type") - if const_expr(mQ_type not in [cutlass.Float16, cutlass.BFloat16]): - raise TypeError("Only Float16 or BFloat16 is supported") - if const_expr(mLSE_type not in [None, Float32]): - raise TypeError("LSE tensor must be Float32") - if const_expr(mCuSeqlensQ_type not in [None, Int32]): - raise TypeError("cu_seqlens_q tensor must be Int32") - if const_expr(mCuSeqlensK_type not in [None, Int32]): - raise TypeError("cu_seqlens_k tensor must be Int32") - if const_expr(mSeqUsedQ_type not in [None, Int32]): - raise TypeError("seqused_q tensor must be Int32") - if const_expr(mSeqUsedK_type not in [None, Int32]): - raise TypeError("seqused_k tensor must be Int32") - assert mQ_type == self.dtype - - def _setup_attributes(self): - # /////////////////////////////////////////////////////////////////////////////// - # Shared memory layout: Q/K/V - # /////////////////////////////////////////////////////////////////////////////// - sQ_layout_atom, sK_layout_atom, sV_layout_atom, sO_layout_atom, sP_layout_atom = ( - self._get_smem_layout_atom() - ) - self.sQ_layout = cute.tile_to_shape( - sQ_layout_atom, - (self.tile_m, self.tile_hdim), - (0, 1), - ) - self.sK_layout = cute.tile_to_shape( - sK_layout_atom, - (self.tile_n, self.tile_hdim, self.num_stages), - (0, 1, 2), - ) - self.sV_layout = cute.tile_to_shape( - sV_layout_atom, - (self.tile_n, self.tile_hdimv, self.num_stages), - (0, 1, 2), - ) - self.sO_layout = cute.tile_to_shape( - sO_layout_atom, - (self.tile_m, self.tile_hdimv), - (0, 1), - ) - if const_expr(sP_layout_atom is not None): - self.sP_layout = cute.tile_to_shape( - sP_layout_atom, - (self.tile_m, self.tile_n), - (0, 1), - ) - else: - self.sP_layout = None - - # /////////////////////////////////////////////////////////////////////////////// - # GMEM Tiled copy: - # /////////////////////////////////////////////////////////////////////////////// - # Thread layouts for copies - universal_copy_bits = 128 - async_copy_elems = universal_copy_bits // self.dtype.width - # atom_async_copy: async copy atom for QKV load - atom_async_copy = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - self.dtype, - num_bits_per_copy=universal_copy_bits, - ) - # atom_universal_copy: universal copy atom for O store - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dtype, - num_bits_per_copy=universal_copy_bits, - ) - # tQ_layout and tK_layout: thread layout for QK load - tQK_shape_dim_1 = sQ_layout_atom.outer.shape[1] // async_copy_elems - assert self.num_Q_load_threads % tQK_shape_dim_1 == 0, ( - "num_threads must be divisible by tQK_shape_dim_1" - ) - assert self.num_producer_threads % tQK_shape_dim_1 == 0, ( - "num_threads must be divisible by tQK_shape_dim_1" - ) - tQ_layout = cute.make_ordered_layout( - (self.num_Q_load_threads // tQK_shape_dim_1, tQK_shape_dim_1), - order=(1, 0), - ) - tK_layout = cute.make_ordered_layout( - (self.num_producer_threads // tQK_shape_dim_1, tQK_shape_dim_1), - order=(1, 0), - ) - # So that we don't have to check if we overshoot kBlockM when we load Q - assert self.tile_m % tQ_layout.shape[0] == 0 - tV_shape_dim_1 = sV_layout_atom.outer.shape[1] // async_copy_elems - tV_layout = cute.make_ordered_layout( - (self.num_producer_threads // tV_shape_dim_1, tV_shape_dim_1), - order=(1, 0), - ) - # TODO: need a different layout for O if O dtype is not the same as V dtype - # tO_layout: thread layout for O store - tO_layout = cute.make_ordered_layout( - (self.num_epilogue_threads // tV_shape_dim_1, tV_shape_dim_1), - order=(1, 0), - ) - # So that we don't have to check if we overshoot kBlockM when we store O - assert self.tile_m % tO_layout.shape[0] == 0 - - # Value layouts for copies - vQKV_layout = cute.make_layout((1, async_copy_elems)) - vO_layout = vQKV_layout - - self.gmem_tiled_copy_Q = cute.make_tiled_copy_tv(atom_async_copy, tQ_layout, vQKV_layout) - self.gmem_tiled_copy_K = cute.make_tiled_copy_tv(atom_async_copy, tK_layout, vQKV_layout) - self.gmem_tiled_copy_V = cute.make_tiled_copy_tv(atom_async_copy, tV_layout, vQKV_layout) - # gmem_tiled_copy_O: tiled copy for O store - self.gmem_tiled_copy_O = cute.make_tiled_copy_tv(atom_universal_copy, tO_layout, vO_layout) - - def _get_smem_layout_atom(self): - raise NotImplementedError() - - def _get_tiled_mma(self): - raise NotImplementedError() - - def _get_shared_storage_cls(self): - raise NotImplementedError() - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - softmax_scale: Float32, - stream: cuda.CUstream, - ): - """Configures and launches the flash attention kernel. - - mQ/mK/mV/mO has same data types(supports fp16 and bf16) and same layout: - (batch_size, seqlen_q, num_head, head_dim):(_, _, _, 1) - """ - raise NotImplementedError() - - @cute.jit - def epilogue( - self, - acc_O: cute.Tensor, - lse: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - sO: cute.Tensor, - seqlen: SeqlenInfoQK, - gmem_tiled_copy_O: cute.TiledCopy, - tma_atom_O: Optional[cute.CopyAtom], - tiled_mma: cute.TiledMma, - tidx: Int32, - m_block: Int32, - head_idx: Int32, - batch_idx: Int32, - ): - # store acc_O - rO = cute.make_fragment_like(acc_O, self.dtype) - rO.store(acc_O.load().to(self.dtype)) - # Make sure all threads have finished reading V - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), number_of_threads=self.num_epilogue_threads - ) - smem_copy_atom_O = utils.get_smem_store_atom(self.arch, self.dtype) - smem_thr_copy_O = cute.make_tiled_copy_C(smem_copy_atom_O, tiled_mma).get_slice(tidx) - taccOrO = smem_thr_copy_O.retile(rO) - taccOsO = smem_thr_copy_O.partition_D(sO) - # taccOsO = quack_copy_utils.partition_D_position_independent(smem_thr_copy_O, sO) - # copy acc O from rmem to smem with the smem copy atom - cute.copy(smem_copy_atom_O, taccOrO, taccOsO) - - cO = cute.make_identity_tensor((self.tile_m, self.tile_hdimv)) - pack_gqa = PackGQA( - self.tile_m, self.tile_hdimv, self.check_hdim_v_oob, self.qhead_per_kvhead - ) - - # Write LSE from rmem -> gmem - if const_expr(mLSE is not None): - if const_expr(not seqlen.has_cu_seqlens_q): - mLSE_cur = mLSE[None, head_idx, batch_idx] - else: - offset = seqlen.offset_q if const_expr(not self.pack_gqa) else (0, seqlen.offset_q) - mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx]) - if const_expr(not self.pack_gqa): - gLSE = cute.local_tile(mLSE_cur, (self.tile_m,), (m_block,)) - gLSE_expanded_layout = cute.append( - gLSE.layout, cute.make_layout((self.tile_hdimv,), stride=(0,)) - ) - gLSE_expanded = cute.make_tensor(gLSE.iterator, gLSE_expanded_layout) - thr_mma = tiled_mma.get_slice(tidx) - taccOgLSE = utils.make_acc_tensor_mn_view(thr_mma.partition_C(gLSE_expanded)) - assert cute.size(taccOgLSE, mode=[0]) == cute.size(lse) - taccOcO = utils.make_acc_tensor_mn_view(thr_mma.partition_C(cO)) - t0accOcO = utils.make_acc_tensor_mn_view(thr_mma.get_slice(0).partition_C(cO)) - # Only the thread corresponding to column 0 writes out the lse to gmem - if taccOcO[0][1] == 0: - for m in cutlass.range_constexpr(cute.size(taccOgLSE.shape[1])): - if ( - t0accOcO[m, 0][0] - < seqlen.seqlen_q - m_block * self.tile_m - taccOcO[0][0] - ): - taccOgLSE[m, 0] = lse[m] - else: - pack_gqa.store_LSE(mLSE_cur, lse, tiled_mma, tidx, m_block, seqlen.seqlen_q) - - if const_expr(not seqlen.has_cu_seqlens_q): - mO_cur = mO[None, None, head_idx, batch_idx] - else: - offset = seqlen.offset_q if const_expr(not self.pack_gqa) else (0, seqlen.offset_q) - mO_cur = cute.domain_offset((offset, 0), mO[None, None, head_idx]) - # thr_mma = tiled_mma.get_slice(tidx) - # taccOgO = thr_mma.partition_C(gO) - # cute.autovec_copy(rO, taccOgO) - # sync to make sure all smem stores are done - if const_expr(self.use_tma_O): - # ensure smem writes are visible to TMA - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierFwd.Epilogue), - number_of_threads=self.num_epilogue_threads + cute.arch.WARP_SIZE, - ) - gO = cute.local_tile(mO_cur, (self.tile_m, self.tile_hdimv), (m_block, 0)) - store_O, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_O, 0, cute.make_layout(1), sO, gO, single_stage=True - ) - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - if warp_idx == 4: - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), - number_of_threads=self.num_epilogue_threads + cute.arch.WARP_SIZE, - ) - store_O() - cute.arch.cp_async_bulk_commit_group() - cute.arch.cp_async_bulk_wait_group(0, read=True) - else: - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), - number_of_threads=self.num_epilogue_threads, - ) - gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx) - tOsO = gmem_thr_copy_O.partition_S(sO) - tOrO = cute.make_fragment_like(tOsO, self.dtype) - # load acc O from smem to rmem for wider vectorization - cute.autovec_copy(tOsO, tOrO) - if const_expr(not self.pack_gqa): - gO = cute.local_tile(mO_cur, (self.tile_m, self.tile_hdimv), (m_block, 0)) - tOgO = gmem_thr_copy_O.partition_D(gO) - tOcO = gmem_thr_copy_O.partition_S(cO) - t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO) - tOpO = utils.predicate_k(tOcO, limit=mO.shape[1]) - # copy acc O from rmem to gmem - for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])): - if ( - t0OcO[0, rest_m, 0][0] - < seqlen.seqlen_q - m_block * self.tile_m - tOcO[0][0] - ): - cute.copy( - gmem_tiled_copy_O, - tOrO[None, rest_m, None], - tOgO[None, rest_m, None], - pred=tOpO[None, rest_m, None] - if const_expr(self.check_hdim_v_oob) - else None, - ) - else: - pack_gqa.store_O(mO_cur, tOrO, gmem_tiled_copy_O, tidx, m_block, seqlen.seqlen_q) - - @cute.jit - def advance_pipeline(self, pipeline_index): - return pipeline_index + 1 if pipeline_index < self.num_stages - 1 else 0 - - @cute.jit - def load_Q( - self, - gmem_thr_copy: cute.TiledCopy, - gQ: cute.Tensor, - sQ: cute.Tensor, - block: Int32, - seqlen: Int32, - headdim: Int32, - ): - tQsQ, tQgQ = gmem_thr_copy.partition_D(sQ), gmem_thr_copy.partition_S(gQ) - cQ = cute.make_identity_tensor((self.tile_m, self.tile_hdim)) - tQcQ = gmem_thr_copy.partition_S(cQ) - t0QcQ = gmem_thr_copy.get_slice(0).partition_S(cQ) - tQpQ = utils.predicate_k(tQcQ, limit=headdim) - for m in cutlass.range_constexpr(cute.size(tQsQ.shape[1])): - # Instead of using tQcQ, we using t0QcQ and subtract the offset from the limit - # (seqlen - block * kBlockM). This is because the entries of t0QcQ are known at compile time. - if t0QcQ[0, m, 0][0] < seqlen - block * self.tile_m - tQcQ[0][0]: - cute.copy( - gmem_thr_copy, - tQgQ[None, m, None], - tQsQ[None, m, None], - pred=tQpQ[None, m, None] if const_expr(self.check_hdim_oob) else None, - ) - # We don't need to clear the sQ smem tiles since we'll only write out the valid outputs - - @cute.jit - def load_K( - self, - gmem_tiled_copy: cute.TiledCopy, - tKgK: cute.Tensor, - tKsK: cute.Tensor, - tKcK: cute.Tensor, - t0KcK: cute.Tensor, - tKpK: cute.Tensor, - block: Int32, - smem_pipe_write: Int32, - seqlen: Int32, - need_predicates: cutlass.Constexpr, - ): - # Do we need to check if we overshoot kBlockN when we load K? - is_even_n_smem_k = self.tile_n % gmem_tiled_copy.tiler_mn[0].shape == 0 - if const_expr(need_predicates or not is_even_n_smem_k): - # Instead of using tKcK, we using t0KcK and subtract the offset from the limit - # (seqlen - block * kBlockN). This is because the entries of t0KcK are known at compile time. - if const_expr(is_even_n_smem_k): - seqlen_limit = seqlen - block * self.tile_n - else: - if const_expr(not need_predicates): - seqlen_limit = self.tile_n - else: - seqlen_limit = cutlass.min(seqlen - block * self.tile_n, self.tile_n) - seqlen_limit -= tKcK[0][0] - for n in cutlass.range_constexpr(cute.size(tKsK.shape[1])): - if t0KcK[0, n, 0][0] < seqlen_limit: - cute.copy( - gmem_tiled_copy, - tKgK[None, n, None, block], - tKsK[ - None, n, None, smem_pipe_write if const_expr(self.num_stages > 1) else 0 - ], - pred=tKpK[None, n, None] if const_expr(self.check_hdim_oob) else None, - ) - # We don't need to clear the sK smem tiles since we'll mask out the scores anyway. - else: - cute.copy( - gmem_tiled_copy, - tKgK[None, None, None, block], - tKsK[None, None, None, smem_pipe_write if const_expr(self.num_stages > 1) else 0], - pred=tKpK if const_expr(self.check_hdim_oob) else None, - ) - - @cute.jit - def load_V( - self, - gmem_tiled_copy: cute.TiledCopy, - tVgV: cute.Tensor, - tVsV: cute.Tensor, - tVcV: cute.Tensor, - t0VcV: cute.Tensor, - tVpV: cute.Tensor, - block: Int32, - smem_pipe_write: Int32, - seqlen: Int32, - need_predicates: cutlass.Constexpr, - ): - # Do we need to check if we overshoot kBlockN when we load V? - is_even_n_smem_v = self.tile_n % gmem_tiled_copy.tiler_mn[0].shape == 0 - if const_expr(need_predicates or not is_even_n_smem_v): - for n in cutlass.range_constexpr(cute.size(tVsV.shape[1])): - # If kBlockN doesn't evenly divide the tiled copy, only the last `n` needs to be checked - if ( - is_even_n_smem_v - or n < cute.size(tVsV.shape[1]) - 1 - or tVcV[0, n, 0][0] < self.tile_n - ): - predicate = tVpV[None, n, None] if const_expr(self.check_hdim_v_oob) else None - if const_expr(need_predicates): - seqlen_limit = seqlen - block * self.tile_n - tVcV[0][0] - predicate_n = t0VcV[0, n, 0][0] < seqlen_limit - predicate = cute.make_fragment_like(tVpV[None, 0, None]) - for k in cutlass.range_constexpr(cute.size(predicate.shape[1])): - for i in cutlass.range_constexpr(cute.size(predicate.shape[0])): - predicate[i, k] = ( - tVpV[i, n, k] if const_expr(self.check_hdim_v_oob) else True - ) and predicate_n - cute.copy( - gmem_tiled_copy, - tVgV[None, n, None, block], - tVsV[ - None, n, None, smem_pipe_write if const_expr(self.num_stages > 1) else 0 - ], - pred=predicate, - ) - else: - cute.copy( - gmem_tiled_copy, - tVgV[None, None, None, block], - tVsV[None, None, None, smem_pipe_write if const_expr(self.num_stages > 1) else 0], - pred=tVpV if const_expr(self.check_hdim_v_oob) else None, - ) - - -class FlashAttentionForwardSm80(FlashAttentionForwardBase): - def _get_smem_layout_atom(self): - sQ_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, self.tile_hdim) - sK_layout_atom = sQ_layout_atom - sV_layout_atom = sm80_utils.get_smem_layout_atom(self.dtype, self.tile_hdimv) - sO_layout_atom = sV_layout_atom - sP_layout_atom = None - return sQ_layout_atom, sK_layout_atom, sV_layout_atom, sO_layout_atom, sP_layout_atom - - def _get_tiled_mma(self): - tiled_mma_qk = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, Float32, (16, 8, 16)), - (self.num_threads // 32, 1, 1), - permutation_mnk=(self.num_threads // 32 * 16, 16, 16), - ) - tiled_mma_pv = cute.make_tiled_mma( - warp.MmaF16BF16Op(self.dtype, Float32, (16, 8, 16)), - (self.num_threads // 32, 1, 1), - permutation_mnk=(self.num_threads // 32 * 16, 16, 16), - ) - return tiled_mma_qk, tiled_mma_pv - - def _get_shared_storage_cls(self): - sQ_struct, sK_struct, sV_struct = [ - cute.struct.Align[cute.struct.MemRange[self.dtype, cute.cosize(layout)], 1024] - for layout in (self.sQ_layout, self.sK_layout, self.sV_layout) - ] - cosize_sQV = max(cute.cosize(self.sQ_layout), cute.cosize(self.sV_layout)) - sQV_struct = cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sQV], 1024] - - @cute.struct - class SharedStorageQKV: - sV: sV_struct - sQ: sQ_struct - sK: sK_struct - - @cute.struct - class SharedStorageSharedQV: - sQ: sQV_struct - sK: sK_struct - - return SharedStorageQKV if const_expr(not self.Q_in_regs) else SharedStorageSharedQV - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - stream: cuda.CUstream, - softmax_scale: Optional[Float32] = None, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - learnable_sink: Optional[cute.Tensor] = None, - aux_tensors=None, - ): - """Configures and launches the flash attention kernel. - - mQ/mK/mV/mO has same data types(supports fp16 and bf16) and same layout: - (batch_size, seqlen_q, num_head, head_dim):(_, _, _, 1) - """ - assert learnable_sink is None, "Learnable sink is not supported in this kernel" - self._check_type( - *(t.element_type if t is not None else None for t in (mQ, mK, mV, mO, mLSE)) - ) - tiled_mma_qk, tiled_mma_pv = self._get_tiled_mma() - self.num_mma_threads = tiled_mma_pv.size - self.num_producer_threads = self.num_threads - self.num_Q_load_threads = self.num_threads - self.num_epilogue_threads = self.num_threads - # self.use_tma_O = self.arch >= 90 and mCuSeqlensQ is None - self.use_tma_O = self.arch >= 90 - self._setup_attributes() - SharedStorage = self._get_shared_storage_cls() - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mQ, mK, mV, mO = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - for t in (mQ, mK, mV, mO) - ] - mQ, mK, mV, mO = [ - cute.make_tensor(t.iterator, cute.select(t.layout, mode=[1, 3, 2, 0])) - for t in (mQ, mK, mV, mO) - ] - mLSE = cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=[2, 1, 0])) - # grid_dim: (m_block, num_head, batch_size) - grid_dim = ( - cute.ceil_div(mQ.shape[0], self.tile_m), - cute.size(mQ.shape[2]), - cute.size(mQ.shape[3]), - ) - LOG2_E = math.log2(math.e) - if const_expr(self.score_mod is None): - softmax_scale_log2 = Float32(softmax_scale * LOG2_E) - softmax_scale = None - else: - # NB: If a user passes in a score mod, we want to apply the score-mod in the sm_scaled qk - # But in the original base 10. We hijack softmax_scale_log2 to just be the change of base - # and correctly apply the softmax_scale prior to score_mod in the softmax step - softmax_scale_log2 = Float32(LOG2_E) - softmax_scale = Float32(softmax_scale) - - fastdiv_mods = None - if const_expr(aux_tensors is not None): - seqlen_q = cute.size(mQ.shape[0]) // ( - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1 - ) - seqlen_k = cute.size(mK.shape[0]) - seqlen_q_divmod = FastDivmodDivisor(seqlen_q) - seqlen_k_divmod = FastDivmodDivisor(seqlen_k) - fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod) - - self.kernel( - mQ, - mK, - mV, - mO, - mLSE, - softmax_scale_log2, - softmax_scale, - window_size_left, - window_size_right, - self.sQ_layout, - self.sK_layout, - self.sV_layout, - self.sO_layout, - self.sP_layout, - self.gmem_tiled_copy_Q, - self.gmem_tiled_copy_K, - self.gmem_tiled_copy_V, - self.gmem_tiled_copy_O, - tiled_mma_qk, - tiled_mma_pv, - SharedStorage, - aux_tensors, - fastdiv_mods, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - smem=SharedStorage.size_in_bytes(), - stream=stream, - ) - - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - softmax_scale_log2: Float32, - softmax_scale: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - sQ_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sO_layout: cute.ComposedLayout, - sP_layout: cute.ComposedLayout | None, - gmem_tiled_copy_Q: cute.TiledCopy, - gmem_tiled_copy_K: cute.TiledCopy, - gmem_tiled_copy_V: cute.TiledCopy, - gmem_tiled_copy_O: cute.TiledCopy, - tiled_mma_qk: cute.TiledMma, - tiled_mma_pv: cute.TiledMma, - SharedStorage: cutlass.Constexpr, - aux_tensors=None, - fastdiv_mods=None, - ): - # Thread index, block index - tidx, _, _ = cute.arch.thread_idx() - m_block, num_head, batch_size = cute.arch.block_idx() - - block_info = BlockInfo( - self.tile_m, - self.tile_n, - self.is_causal, - self.is_local, - False, # is_split_kv - window_size_left, - window_size_right, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - seqlen = SeqlenInfoQK.create(seqlen_q_static=mQ.shape[0], seqlen_k_static=mK.shape[0]) - n_block_min, n_block_max = block_info.get_n_block_min_max(seqlen, m_block) - # TODO: return early if n_block_max == 0 - # if self.is_causal: - # if n_block_max <= 0: - # return - n_block = n_block_max - 1 - - # /////////////////////////////////////////////////////////////////////////////// - # Get the appropriate tiles for this thread block. - # /////////////////////////////////////////////////////////////////////////////// - blkQ_shape = (self.tile_m, self.tile_hdim) - blkK_shape = (self.tile_n, self.tile_hdim) - blkV_shape = (self.tile_n, self.tile_hdimv) - gQ = cute.local_tile(mQ[None, None, num_head, batch_size], blkQ_shape, (m_block, 0)) - num_head_kv = num_head // self.qhead_per_kvhead - gK = cute.local_tile(mK[None, None, num_head_kv, batch_size], blkK_shape, (None, 0)) - gV = cute.local_tile(mV[None, None, num_head_kv, batch_size], blkV_shape, (None, 0)) - - # /////////////////////////////////////////////////////////////////////////////// - # Get shared memory buffer - # /////////////////////////////////////////////////////////////////////////////// - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage) - sQ = storage.sQ.get_tensor(sQ_layout) - sK = storage.sK.get_tensor(sK_layout) - if const_expr(not self.Q_in_regs): - sV = storage.sV.get_tensor(sV_layout) - else: - sV = cute.make_tensor(cute.recast_ptr(sQ.iterator, dtype=self.dtype), sV_layout) - # Transpose view of V to tensor with layout (head_dim_v, tile_n) for tiled mma - sVt = utils.transpose_view(sV) - - gmem_thr_copy_K = gmem_tiled_copy_K.get_slice(tidx) - gmem_thr_copy_V = gmem_tiled_copy_V.get_slice(tidx) - # (CPY_Atom, CPY_N, CPY_K, n_block) - tKsK, tKgK = gmem_thr_copy_K.partition_D(sK), gmem_thr_copy_K.partition_S(gK) - # (CPY_Atom, CPY_N, CPY_K, n_block) - tVsV, tVgV = gmem_thr_copy_V.partition_D(sV), gmem_thr_copy_V.partition_S(gV) - - # /////////////////////////////////////////////////////////////////////////////// - # Tile MMA compute thread partitions and allocate accumulators - # /////////////////////////////////////////////////////////////////////////////// - thr_mma_qk = tiled_mma_qk.get_slice(tidx) - thr_mma_pv = tiled_mma_pv.get_slice(tidx) - tSrQ = thr_mma_qk.make_fragment_A(thr_mma_qk.partition_A(sQ)) - tSrK = thr_mma_qk.make_fragment_B(thr_mma_qk.partition_B(sK[None, None, 0])) - tOrVt = thr_mma_pv.make_fragment_B(thr_mma_pv.partition_B(sVt[None, None, 0])) - acc_shape_O = thr_mma_pv.partition_shape_C((self.tile_m, self.tile_hdimv)) - acc_O = cute.make_fragment(acc_shape_O, Float32) - acc_O.fill(0.0) - - # /////////////////////////////////////////////////////////////////////////////// - # Smem copy atom tiling - # /////////////////////////////////////////////////////////////////////////////// - smem_copy_atom_QK = cute.make_copy_atom( - warp.LdMatrix8x8x16bOp(transpose=False, num_matrices=4), - self.dtype, - ) - smem_copy_atom_V = cute.make_copy_atom( - warp.LdMatrix8x8x16bOp(transpose=True, num_matrices=4), - self.dtype, - ) - smem_thr_copy_Q = utils.make_tiled_copy_A(smem_copy_atom_QK, tiled_mma_qk).get_slice(tidx) - smem_thr_copy_K = utils.make_tiled_copy_B(smem_copy_atom_QK, tiled_mma_qk).get_slice(tidx) - smem_thr_copy_V = utils.make_tiled_copy_B(smem_copy_atom_V, tiled_mma_pv).get_slice(tidx) - - tSsQ = smem_thr_copy_Q.partition_S(sQ) - tSsK = smem_thr_copy_K.partition_S(sK) - tOsVt = smem_thr_copy_V.partition_S(sVt) - - # /////////////////////////////////////////////////////////////////////////////// - # Predicate: Mark indices that need to copy when problem_shape isn't a multiple - # of tile_shape - # /////////////////////////////////////////////////////////////////////////////// - # Construct identity layout for KV - cK = cute.make_identity_tensor((self.tile_n, self.tile_hdim)) - tKcK = gmem_thr_copy_K.partition_S(cK) - t0KcK = gmem_thr_copy_K.get_slice(0).partition_S(cK) - if const_expr(self.tile_hdim == self.tile_hdimv): - tVcV = tKcK - t0VcV = t0KcK - else: - cV = cute.make_identity_tensor((self.tile_n, self.tile_hdimv)) - tVcV = gmem_thr_copy_V.partition_S(cV) - t0VcV = gmem_thr_copy_V.get_slice(0).partition_S(cV) - # Allocate predicate tensors for m and n, here we only allocate the tile of k, and - # use "if" on the mn dimension. - # This is to reduce register pressure and gets 2-3% performance gain. - tKpK = utils.predicate_k(tKcK, limit=mK.shape[1]) - if const_expr(self.same_hdim_kv): - tVpV = tKpK - else: - tVpV = utils.predicate_k(tVcV, limit=mV.shape[1]) - - # shape: (atom_v_m * rest_m) - softmax = Softmax.create( - softmax_scale_log2, - num_rows=acc_O.shape[0][0] * acc_O.shape[1], - softmax_scale=softmax_scale, - ) - softmax.reset() - - # group parameters for compute_one_n_block - mma_params = SimpleNamespace( - thr_mma_qk=thr_mma_qk, - thr_mma_pv=thr_mma_pv, - tSrQ=tSrQ, - tSrK=tSrK, - tOrVt=tOrVt, - acc_O=acc_O, - ) - smem_copy_params = SimpleNamespace( - smem_thr_copy_Q=smem_thr_copy_Q, - smem_thr_copy_K=smem_thr_copy_K, - smem_thr_copy_V=smem_thr_copy_V, - tSsQ=tSsQ, - tSsK=tSsK, - tOsVt=tOsVt, - ) - load_K = partial( - self.load_K, gmem_tiled_copy_K, tKgK, tKsK, tKcK, t0KcK, tKpK, seqlen=seqlen.seqlen_k - ) - load_V = partial( - self.load_V, gmem_tiled_copy_V, tVgV, tVsV, tVcV, t0VcV, tVpV, seqlen=seqlen.seqlen_k - ) - - compute_one_n_block = partial( - self.compute_one_n_block, - mma_params=mma_params, - smem_copy_params=smem_copy_params, - softmax=softmax, - load_K=load_K, - load_V=load_V, - score_mod=self.score_mod, - batch_idx=batch_size, - head_idx=num_head, - m_block=m_block, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Prologue - # /////////////////////////////////////////////////////////////////////////////// - # Start async loads of the last mn-tile, where we take care of the mn residue - gmem_thr_copy_Q = gmem_tiled_copy_Q.get_slice(tidx) - self.load_Q(gmem_thr_copy_Q, gQ, sQ, m_block, seqlen=seqlen.seqlen_q, headdim=mQ.shape[1]) - cute.arch.cp_async_commit_group() - - def preprocess_Q(): - cute.arch.cp_async_wait_group(self.num_stages * 2 - 1) - if const_expr(self.Q_in_regs): - cute.arch.barrier() - tSrQ_copy_view = smem_thr_copy_Q.retile(tSrQ) - cute.copy(smem_thr_copy_Q, tSsQ, tSrQ_copy_view) - - # If Q_in_regs, we load Q, then load 1 stage of K, then (optionally) rotate Q and - # read from smem_q to registers, then load V. - # If !Q_in_regs, we load Q, load all stages of K & V, then (optionally) rotate Q. - if const_expr(self.Q_in_regs): - load_K(n_block, smem_pipe_write=0, need_predicates=True) - cute.arch.cp_async_commit_group() - preprocess_Q() - cute.arch.barrier() # Make sure all threads have read smem_q before loading V - - for stage in cutlass.range_constexpr(self.num_stages): - if const_expr(not self.Q_in_regs or stage > 0): - if stage == 0 or n_block - stage >= 0: - load_K(n_block - stage, smem_pipe_write=stage, need_predicates=stage == 0) - cute.arch.cp_async_commit_group() - if const_expr(stage < self.num_stages - 1): - if stage == 0 or n_block - stage >= 0: - load_V(n_block - stage, smem_pipe_write=stage, need_predicates=stage == 0) - cute.arch.cp_async_commit_group() - if const_expr(not self.Q_in_regs): - preprocess_Q() - - # /////////////////////////////////////////////////////////////////////////////// - # Mainloop - # /////////////////////////////////////////////////////////////////////////////// - # Start processing of the first n-block. - # For performance reason, we separate out two kinds of iterations: - # those that need masking on S, and those that don't. - # We need masking on S for the very last block when K and V has length not multiple of tile_n. - # We also need masking on S if it's causal, for the last several blocks. - mask = AttentionMask( - self.tile_m, - self.tile_n, - seqlen.seqlen_q, - seqlen.seqlen_k, - window_size_left, - window_size_right, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - mask_fn = partial( - mask.apply_mask, - m_block=m_block, - thr_mma=thr_mma_qk, - mask_causal=self.is_causal, - mask_local=self.is_local, - fastdiv_mods=fastdiv_mods if const_expr(self.mask_mod is not None) else None, - ) - - # First iteration with seqlen masking - smem_pipe_read = Int32(0) - smem_pipe_write = Int32(self.num_stages - 1) - compute_one_n_block( - n_block, - smem_pipe_read, - smem_pipe_write, - is_first_n_block=True, - check_inf=True, - mask_fn=partial(mask_fn, mask_seqlen=True), - ) - smem_pipe_read = self.advance_pipeline(smem_pipe_read) - smem_pipe_write = self.advance_pipeline(smem_pipe_write) - # Next couple of iterations with causal masking - if const_expr(self.is_causal or self.is_local): - n_block_min_causal_local_mask = block_info.get_n_block_min_causal_local_mask( - seqlen, m_block, n_block_min - ) - for n_tile in cutlass.range(n_block_max - 1 - n_block_min_causal_local_mask, unroll=1): - n_block = n_block_max - 2 - n_tile - compute_one_n_block( - n_block, - smem_pipe_read, - smem_pipe_write, - check_inf=True, - mask_fn=partial(mask_fn, mask_seqlen=False), - ) - smem_pipe_read = self.advance_pipeline(smem_pipe_read) - smem_pipe_write = self.advance_pipeline(smem_pipe_write) - # The remaining iterations have no masking - for n_tile in cutlass.range(n_block, unroll=1): - compute_one_n_block( - n_block - n_tile - 1, smem_pipe_read, smem_pipe_write, check_inf=True - ) - smem_pipe_read = self.advance_pipeline(smem_pipe_read) - smem_pipe_write = self.advance_pipeline(smem_pipe_write) - # TODO: local - - # normalize acc_O by row_sum and calculate the lse - row_scale = softmax.finalize() - softmax.rescale_O(acc_O, row_scale) - - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue - # /////////////////////////////////////////////////////////////////////////////// - # reuse sQ's data iterator - sO = cute.make_tensor(sQ.iterator, sO_layout) - self.epilogue( - acc_O, - softmax.row_sum, - mO, - mLSE, - sO, - seqlen, - gmem_tiled_copy_O, - None, - tiled_mma_pv, - tidx, - m_block, - num_head, - batch_size, - ) - - @cute.jit - def compute_one_n_block( - self, - n_block: Int32, - smem_pipe_read: Int32, - smem_pipe_write: Int32, - mma_params: SimpleNamespace, - smem_copy_params: SimpleNamespace, - softmax: Softmax, - load_K: Callable, - load_V: Callable, - score_mod: Callable | None, - batch_idx: cutlass.Int32, - head_idx: cutlass.Int32, - m_block: cutlass.Int32, - seqlen: SeqlenInfoQK, - aux_tensors=None, - fastdiv_mods=None, - mask_fn: Optional[Callable] = None, - is_first_n_block: cutlass.Constexpr = False, - check_inf: cutlass.Constexpr = True, - ): - """Compute one n_block of S/O. - - This function provides different variants for processing the first n block versus - subsequent blocks. - """ - - def sync(): - cute.arch.cp_async_wait_group(self.num_stages * 2 - 2) - cute.arch.barrier() - - acc_shape_S = mma_params.thr_mma_qk.partition_shape_C((self.tile_m, self.tile_n)) - acc_S = cute.make_fragment(acc_shape_S, Float32) - acc_S.fill(0.0) - # wait for smem tile QK before mma calculation for S - sync() - - # need predicates for the first tile - def load_V_next(): - if self.num_stages == 1 or n_block - self.num_stages + 1 >= 0: - load_V( - n_block - self.num_stages + 1, - smem_pipe_write, - need_predicates=is_first_n_block and self.num_stages == 1, - ) - cute.arch.cp_async_commit_group() - - load_V_next() - sm80_utils.gemm( - mma_params.thr_mma_qk, - acc_S, - mma_params.tSrQ, - mma_params.tSrK, - smem_copy_params.tSsQ, - smem_copy_params.tSsK[ - None, None, None, smem_pipe_read if const_expr(self.num_stages > 1) else 0 - ], - smem_copy_params.smem_thr_copy_Q, - smem_copy_params.smem_thr_copy_K, - # hook_fn=load_V_next, - A_in_regs=self.Q_in_regs, - ) - if const_expr(score_mod is not None): - self.apply_score_mod( - mma_params.thr_mma_qk, - batch_idx, - head_idx, - m_block, - acc_S, - n_block, - seqlen, - softmax_scale=softmax.softmax_scale, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - smem_pipe_write = self.advance_pipeline(smem_pipe_write) - - def load_K_next(): - if n_block - self.num_stages >= 0: - load_K(n_block - self.num_stages, smem_pipe_write, need_predicates=False) - cute.arch.cp_async_commit_group() - - # wait for smem tile V for O - if const_expr(self.num_stages == 1): - sync() - load_K_next() - if const_expr(mask_fn is not None): - mask_fn(acc_S, n_block=n_block) - row_scale = softmax.online_softmax(acc_S, is_first=is_first_n_block, check_inf=check_inf) - softmax.rescale_O(mma_params.acc_O, row_scale) - rP = cute.make_fragment_like(acc_S, self.dtype) - rP.store(acc_S.load().to(self.dtype)) - tOrP = cute.make_tensor(rP.iterator, utils.convert_layout_acc_frgA(rP.layout)) - if const_expr(self.num_stages > 1): - sync() - load_K_next() - sm80_utils.gemm_rs( - mma_params.thr_mma_pv, - mma_params.acc_O, - tOrP, - mma_params.tOrVt, - smem_copy_params.tOsVt[ - None, None, None, smem_pipe_read if const_expr(self.num_stages > 1) else 0 - ], - smem_copy_params.smem_thr_copy_V, - # hook_fn=load_K_next, - ) - # if const_expr(self.num_stages > 1): - # load_K_next() - - -class FlashAttentionForwardSm90(FlashAttentionForwardBase): - arch = 90 - - def __init__( - self, - *args, - intra_wg_overlap: bool = True, - mma_pv_is_rs: bool = True, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.intra_wg_overlap = intra_wg_overlap - self.mma_pv_is_rs = mma_pv_is_rs - self.buffer_align_bytes = 1024 - - def _get_smem_layout_atom(self): - sQ_layout_atom = warpgroup.make_smem_layout_atom( - sm90_utils_basic.get_smem_layout_atom(LayoutEnum.ROW_MAJOR, self.dtype, self.tile_hdim), - self.dtype, - ) - sK_layout_atom = sQ_layout_atom - sV_layout_atom = warpgroup.make_smem_layout_atom( - sm90_utils_basic.get_smem_layout_atom( - LayoutEnum.ROW_MAJOR, self.dtype, self.tile_hdimv - ), - self.dtype, - ) - sO_layout_atom = sV_layout_atom - if not self.mma_pv_is_rs: - sP_layout_atom = warpgroup.make_smem_layout_atom( - sm90_utils_basic.get_smem_layout_atom( - LayoutEnum.ROW_MAJOR, self.dtype, self.tile_n - ), - self.dtype, - ) - else: - sP_layout_atom = None - return sQ_layout_atom, sK_layout_atom, sV_layout_atom, sO_layout_atom, sP_layout_atom - - def _get_tiled_mma(self): - tiled_mma_qk = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K, - warpgroup.OperandMajorMode.K, - Float32, - atom_layout_mnk=(self.tile_m // 64, 1, 1), # Might need (1, 2, 1) for hdim 512 - tiler_mn=(64, self.tile_n), - ) - tiled_mma_pv = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K, - warpgroup.OperandMajorMode.MN, - Float32, - atom_layout_mnk=(self.tile_m // 64, 1, 1), # Might need (1, 2, 1) for hdim 512 - tiler_mn=(64, self.tile_hdimv), - a_source=warpgroup.OperandSource.RMEM - if self.mma_pv_is_rs - else warpgroup.OperandSource.SMEM, - ) - tiled_mma_pv_rs = sm90_utils_basic.make_trivial_tiled_mma( - self.dtype, - self.dtype, - warpgroup.OperandMajorMode.K, - warpgroup.OperandMajorMode.MN, - Float32, - atom_layout_mnk=(self.tile_m // 64, 1, 1), # Might need (1, 2, 1) for hdim 512 - tiler_mn=(64, self.tile_hdimv), - a_source=warpgroup.OperandSource.RMEM, - ) - return tiled_mma_qk, tiled_mma_pv, tiled_mma_pv_rs - - def _get_shared_storage_cls(self): - # If we use cp.async to load Q, we want sQ to align to 1024 bytes - sQ_struct, sK_struct, sV_struct = [ - cute.struct.Align[ - cute.struct.MemRange[self.dtype, cute.cosize(layout)], self.buffer_align_bytes - ] - for layout in (self.sQ_layout, self.sK_layout, self.sV_layout) - ] - cosize_sQV = max(cute.cosize(self.sQ_layout), cute.cosize(self.sV_layout)) - sQV_struct = cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sQV], 1024] - cosize_sP = cute.cosize(self.sP_layout) if const_expr(self.sP_layout is not None) else 0 - sP_struct = cute.struct.Align[cute.struct.MemRange[self.dtype, cosize_sP], 1024] - # 1 for Q, 1 for O, self.num_stages*2 for K, self.num_stages*2 for V, - mbar_ptr_QO_struct = cute.struct.MemRange[cutlass.Int64, 2] - mbar_ptr_K_struct = cute.struct.MemRange[cutlass.Int64, self.num_stages * 2] - mbar_ptr_V_struct = cute.struct.MemRange[cutlass.Int64, self.num_stages * 2] - - @cute.struct - class SharedStorageQKV: - mbar_ptr: mbar_ptr_QO_struct - mbar_ptr_K: mbar_ptr_K_struct - mbar_ptr_V: mbar_ptr_V_struct - sV: sV_struct - sQ: sQ_struct - sK: sK_struct - sP: sP_struct - - @cute.struct - class SharedStorageSharedQV: - mbar_ptr: mbar_ptr_QO_struct - mbar_ptr_K: mbar_ptr_K_struct - mbar_ptr_V: mbar_ptr_V_struct - sQ: sQV_struct - sK: sK_struct - sP: sP_struct - - return SharedStorageQKV if const_expr(not self.Q_in_regs) else SharedStorageSharedQV - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, # (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q - mK: cute.Tensor, # (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, h_k, d) if there is page_table - mV: cute.Tensor, # (b_k, s_k, h_k, dv) or (total_k, h_k, dv) if there is cu_seqlens_k or (num_pages, page_size, h_k, dv) if there is page_table - mO: cute.Tensor, # (b, s_q, h, dv) or (total_q, h, dv) if there is cu_seqlens_q - mLSE: Optional[cute.Tensor], - softmax_scale: Float32, - stream: cuda.CUstream, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - mPageTable: Optional[cute.Tensor] = None, # (b_k, max_num_pages_per_seq) - window_size_left: Int32 | int | None = None, - window_size_right: Int32 | int | None = None, - learnable_sink: Optional[cute.Tensor] = None, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - aux_tensors: Optional[list] = None, - ): - """Configures and launches the flash attention kernel. - - mQ/mK/mV/mO has same data types(supports fp16 and bf16) and same layout: - (batch_size, seqlen_q, num_head, head_dim):(_, _, _, 1) - """ - - self._check_type( - *( - t.element_type if t is not None else None - for t in (mQ, mK, mV, mO, mLSE, mCuSeqlensQ, mCuSeqlensK, mSeqUsedQ, mSeqUsedK) - ) - ) - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - - mQ, mK, mV, mO = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - for t in (mQ, mK, mV, mO) - ] - QO_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1] - mQ, mO = [utils.select(t, QO_layout_transpose) for t in (mQ, mO)] - KV_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensK is None) else [0, 2, 1] - mK, mV = [utils.select(t, KV_layout_transpose) for t in (mK, mV)] - LSE_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0] - mLSE = utils.select(mLSE, LSE_layout_transpose) if const_expr(mLSE is not None) else None - - tiled_mma_qk, tiled_mma_pv, tiled_mma_pv_rs = self._get_tiled_mma() - self.num_mma_threads = tiled_mma_qk.size - self.num_threads_per_warp_group = 128 - self.num_mma_warp_groups = self.num_mma_threads // self.num_threads_per_warp_group - self.num_threads = self.num_threads_per_warp_group * (self.num_mma_warp_groups + 1) - self.num_producer_threads = 32 - self.num_Q_load_threads = self.num_mma_threads # If not TMA_Q, MMA threads load Q - self.num_epilogue_threads = self.num_mma_threads - self.num_mma_regs = ( - 256 - if self.num_mma_warp_groups == 1 - else (240 if self.num_mma_warp_groups == 2 else 160) - ) - self.num_producer_regs = ( - 56 if self.num_mma_warp_groups == 1 else (24 if self.num_mma_warp_groups == 2 else 32) - ) - # self.num_mma_regs = 232 - # self.num_producer_regs = 40 - self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None) - - self.use_scheduler_barrier = ( - (self.num_mma_warp_groups >= 2 and self.tile_hdim <= 128) - if const_expr(self.intra_wg_overlap) - else (self.num_mma_warp_groups == 2) - ) - self.use_tma_Q = self.arch >= 90 and not ( - self.pack_gqa and self.tile_m % self.qhead_per_kvhead != 0 - ) - self.use_tma_O = ( - self.arch >= 90 and mCuSeqlensQ is None and mSeqUsedQ is None and not self.pack_gqa - ) - # TODO: rescale_O_before_gemm - self._setup_attributes() - # TODO: we prob don't need most of what's in _setup_attributes - self.sQ_layout, self.sK_layout, self.sV_layout, self.sO_layout = [ - sm90_utils.make_smem_layout(mX.element_type, LayoutEnum.ROW_MAJOR, shape, stage) - for mX, shape, stage in [ - (mQ, (self.tile_m, self.tile_hdim), None), - (mK, (self.tile_n, self.tile_hdim), self.num_stages), - (mV, (self.tile_n, self.tile_hdimv), self.num_stages), - (mO, (self.tile_m, self.tile_hdimv), None), - ] - ] - self.sP_layout = None - if const_expr(not self.mma_pv_is_rs): - self.sP_layout = sm90_utils.make_smem_layout( - mV.dtype, LayoutEnum.ROW_MAJOR, (self.tile_m, self.tile_n) - ) - - SharedStorage = self._get_shared_storage_cls() - - if const_expr(self.pack_gqa): - shape_Q_packed = ( - (self.qhead_per_kvhead, mQ.shape[0]), - mQ.shape[1], - mK.shape[2], - *mQ.shape[3:], - ) - stride_Q_packed = ( - (mQ.stride[2], mQ.stride[0]), - mQ.stride[1], - mQ.stride[2] * self.qhead_per_kvhead, - *mQ.stride[3:], - ) - mQ = cute.make_tensor( - mQ.iterator, cute.make_layout(shape_Q_packed, stride=stride_Q_packed) - ) - shape_O_packed = ( - (self.qhead_per_kvhead, mO.shape[0]), - mK.shape[1], - mK.shape[2], - *mO.shape[3:], - ) - stride_O_packed = ( - (mO.stride[2], mO.stride[0]), - mO.stride[1], - mO.stride[2] * self.qhead_per_kvhead, - *mO.stride[3:], - ) - mO = cute.make_tensor( - mO.iterator, cute.make_layout(shape_O_packed, stride=stride_O_packed) - ) - if const_expr(mLSE is not None): - shape_LSE_packed = ( - (self.qhead_per_kvhead, mLSE.shape[0]), - mK.shape[2], - *mLSE.shape[2:], - ) - stride_LSE_packed = ( - (mLSE.stride[1], mLSE.stride[0]), - mLSE.stride[1] * self.qhead_per_kvhead, - *mLSE.stride[2:], - ) - mLSE = cute.make_tensor( - mLSE.iterator, cute.make_layout(shape_LSE_packed, stride=stride_LSE_packed) - ) - - # TMA - gmem_tiled_copy_Q = cpasync.CopyBulkTensorTileG2SOp() - gmem_tiled_copy_KV = cpasync.CopyBulkTensorTileG2SOp() # Might multicast - gmem_tiled_copy_O = cpasync.CopyBulkTensorTileS2GOp() - self.tma_copy_bytes = { - name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1])) - for name, mX, layout in [ - ("Q", mQ, self.sQ_layout), - ("K", mK, self.sK_layout), - ("V", mV, self.sV_layout), - ] - } - tma_atom_Q, tma_tensor_Q = None, None - if const_expr(self.use_tma_Q): - tma_atom_Q, tma_tensor_Q = cpasync.make_tiled_tma_atom( - gmem_tiled_copy_Q, - mQ, - self.sQ_layout, - (self.tile_m, self.tile_hdim), # No mcast - ) - tma_atom_K, tma_tensor_K = cpasync.make_tiled_tma_atom( - gmem_tiled_copy_KV, - mK, - cute.select(self.sK_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdim), - 1, # No mcast for now - ) - tma_atom_V, tma_tensor_V = cpasync.make_tiled_tma_atom( - gmem_tiled_copy_KV, - mV, - cute.select(self.sV_layout, mode=[0, 1]), - (self.tile_n, self.tile_hdimv), - 1, # No mcast for now - ) - tma_atom_O, tma_tensor_O = None, None - if const_expr(self.use_tma_O): - tma_atom_O, tma_tensor_O = cpasync.make_tiled_tma_atom( - gmem_tiled_copy_O, - mO, - self.sO_layout, - (self.tile_m, self.tile_hdimv), # No mcast - ) - if const_expr(mCuSeqlensQ is not None or mSeqUsedQ is not None): - TileScheduler = SingleTileVarlenScheduler - else: - TileScheduler = ( - SingleTileScheduler - if const_expr(not self.is_causal or self.is_local) - else SingleTileLPTScheduler - ) - tile_sched_args = TileSchedulerArguments( - cute.ceil_div(cute.size(mQ.shape[0]), self.tile_m), - cute.size(mQ.shape[2]), - cute.size(mQ.shape[3]) - if const_expr(mCuSeqlensQ is None) - else cute.size(mCuSeqlensQ.shape[0] - 1), - 1, # num_splits - cute.size(mK.shape[0]), - mQ.shape[1], - mV.shape[1], - total_q=cute.size(mQ.shape[0]) - if const_expr(mCuSeqlensQ is not None) - else cute.size(mQ.shape[0]) * cute.size(mQ.shape[3]), - tile_shape_mn=(self.tile_m, self.tile_n), - mCuSeqlensQ=mCuSeqlensQ, - mSeqUsedQ=mSeqUsedQ, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - element_size=self.dtype.width // 8, - is_persistent=False, - lpt=self.is_causal or self.is_local, - ) - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - LOG2_E = math.log2(math.e) - if const_expr(self.score_mod is None): - softmax_scale_log2 = softmax_scale * LOG2_E - softmax_scale = None - else: - # NB: If a user passes in a score mod, we want to apply the score-mod in the sm_scaled qk - # But in the original base 10. We hijack softmax_scale_log2 to just be the change of base - # and correctly apply the softmax_scale prior to score_mod in the softmax step - softmax_scale_log2 = LOG2_E - softmax_scale = softmax_scale - if const_expr(window_size_left is not None): - window_size_left = Int32(window_size_left) - if const_expr(window_size_right is not None): - window_size_right = Int32(window_size_right) - - fastdiv_mods = None - if const_expr(aux_tensors is not None): - seqlen_q = cute.size(mQ.shape[0]) // ( - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1 - ) - seqlen_k = ( - cute.size(mK.shape[0]) - if const_expr(mPageTable is None) - else mK.shape[0] * mPageTable.shape[1] - ) - seqlen_q_divmod = FastDivmodDivisor(seqlen_q) - seqlen_k_divmod = FastDivmodDivisor(seqlen_k) - fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod) - - self.kernel( - tma_tensor_Q if const_expr(self.use_tma_Q) else mQ, - tma_tensor_K, - tma_tensor_V, - tma_tensor_O if const_expr(self.use_tma_O) else mO, - mLSE, - mCuSeqlensQ, - mCuSeqlensK, - mSeqUsedQ, - mSeqUsedK, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_O, - softmax_scale_log2, - softmax_scale, - window_size_left, - window_size_right, - learnable_sink, - blocksparse_tensors, - self.sQ_layout, - self.sK_layout, - self.sV_layout, - self.sO_layout, - self.sP_layout, - self.gmem_tiled_copy_Q, - self.gmem_tiled_copy_K, - self.gmem_tiled_copy_V, - self.gmem_tiled_copy_O, - tiled_mma_qk, - tiled_mma_pv, - tiled_mma_pv_rs, - tile_sched_params, - TileScheduler, - SharedStorage, - aux_tensors, - fastdiv_mods, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - stream=stream, - min_blocks_per_mp=1, - ) - - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - mCuSeqlensQ: Optional[cute.Tensor], - mCuSeqlensK: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - mSeqUsedK: Optional[cute.Tensor], - tma_atom_Q: Optional[cute.CopyAtom], - tma_atom_K: Optional[cute.CopyAtom], - tma_atom_V: Optional[cute.CopyAtom], - tma_atom_O: Optional[cute.CopyAtom], - softmax_scale_log2: Float32, - softmax_scale: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - learnable_sink: Optional[cute.Tensor], - blocksparse_tensors: Optional[BlockSparseTensors], - sQ_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sO_layout: cute.ComposedLayout, - sP_layout: cute.ComposedLayout | None, - gmem_tiled_copy_Q: cute.TiledCopy, - gmem_tiled_copy_K: cute.TiledCopy, - gmem_tiled_copy_V: cute.TiledCopy, - gmem_tiled_copy_O: cute.TiledCopy, - tiled_mma_qk: cute.TiledMma, - tiled_mma_pv: cute.TiledMma, - tiled_mma_pv_rs: cute.TiledMma, - tile_sched_params: ParamsBase, - TileScheduler: cutlass.Constexpr[Callable], - SharedStorage: cutlass.Constexpr[Callable], - aux_tensors=Optional[list[cute.Tensor]], - fastdiv_mods=None, - ): - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - # Prefetch tma descriptor - if warp_idx == 0: - for tma_atom in (tma_atom_Q, tma_atom_K, tma_atom_V, tma_atom_O): - if const_expr(tma_atom is not None): - cpasync.prefetch_descriptor(tma_atom) - - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage) - - # Mbarrier init - mbar_ptr_Q = storage.mbar_ptr.data_ptr() - if warp_idx == 1: - # if tidx < 2: - # # barrierO num threads should be self.num_mma_threads - # cute.arch.mbarrier_init(mbar_ptr_Q + tidx, 1 if tidx == 0 else self.num_mma_threads) - if const_expr(not self.use_tma_Q): - cute.arch.mbarrier_init(mbar_ptr_Q, self.num_Q_load_threads) - # cute.arch.mbarrier_init(mbar_ptr_Q + 1, self.num_mma_threads) - # We rely on pipeline_k and pipeline_v to initialize the mbarrier fence and sync - pipeline_kv_producer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread - ) - pipeline_kv_consumer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, self.num_mma_threads // cute.arch.WARP_SIZE - ) - pipeline_k = pipeline.PipelineTmaAsync.create( - barrier_storage=storage.mbar_ptr_K.data_ptr(), - num_stages=self.num_stages, - producer_group=pipeline_kv_producer_group, - consumer_group=pipeline_kv_consumer_group, - tx_count=self.tma_copy_bytes["K"], - defer_sync=True, - ) - pipeline_v = pipeline.PipelineTmaAsync.create( - barrier_storage=storage.mbar_ptr_V.data_ptr(), - num_stages=self.num_stages, - producer_group=pipeline_kv_producer_group, - consumer_group=pipeline_kv_consumer_group, - tx_count=self.tma_copy_bytes["V"], - defer_sync=False, - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Get shared memory buffer - # /////////////////////////////////////////////////////////////////////////////// - sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner) - sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner) - if const_expr(not self.Q_in_regs): - sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner) - else: - sV = storage.sQ.get_tensor( - sV_layout.outer, swizzle=sV_layout.inner, dtype=mV.element_type - ) - # Transpose view of V to tensor with layout (head_dim_v, tile_n) for tiled mma - sVt = utils.transpose_view(sV) - sP = None - if const_expr(sP_layout is not None): - sP = storage.sP.get_tensor(sP_layout.outer, swizzle=sP_layout.inner) - # reuse sQ's data iterator - sO = storage.sQ.get_tensor(sO_layout.outer, swizzle=sO_layout.inner, dtype=self.dtype) - - block_info = BlockInfo( - self.tile_m, - self.tile_n, - self.is_causal, - self.is_local, - False, # is_split_kv - window_size_left, - window_size_right, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - SeqlenInfoCls = partial( - SeqlenInfoQK.create, - seqlen_q_static=mQ.shape[0] if const_expr(not self.pack_gqa) else mQ.shape[0][1], - seqlen_k_static=mK.shape[0], - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=mCuSeqlensK, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=mSeqUsedK, - ) - AttentionMaskCls = partial( - AttentionMask, - self.tile_m, - self.tile_n, - window_size_left=window_size_left, - window_size_right=window_size_right, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - TileSchedulerCls = partial(TileScheduler.create, tile_sched_params) - - if warp_idx < 4: # Producer - cute.arch.warpgroup_reg_dealloc(self.num_producer_regs) - self.load( - mQ, - mK, - mV, - sQ, - sK, - sV, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - pipeline_k, - pipeline_v, - mbar_ptr_Q, - blocksparse_tensors, - block_info, - SeqlenInfoCls, - TileSchedulerCls, - ) - - else: # Consumer - cute.arch.warpgroup_reg_alloc(self.num_mma_regs) - # /////////////////////////////////////////////////////////////////////////////// - # Tile MMA compute thread partitions and allocate accumulators - # /////////////////////////////////////////////////////////////////////////////// - tidx, _, _ = cute.arch.thread_idx() - tidx = tidx - 128 - self.mma( - tiled_mma_qk, - tiled_mma_pv, - tiled_mma_pv_rs, - mQ, - mO, - mLSE, - sQ, - sK, - sVt, - sP, - sO, - learnable_sink, - pipeline_k, - pipeline_v, - mbar_ptr_Q, - gmem_tiled_copy_Q, - gmem_tiled_copy_O, - tma_atom_O, - tidx, - softmax_scale_log2, - softmax_scale, - block_info, - SeqlenInfoCls, - AttentionMaskCls, - TileSchedulerCls, - blocksparse_tensors, - aux_tensors, - fastdiv_mods, - ) - - @cute.jit - def load( - self, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - tma_atom_Q: cute.CopyAtom, - tma_atom_K: cute.CopyAtom, - tma_atom_V: cute.CopyAtom, - pipeline_k: cutlass.pipeline.PipelineAsync, - pipeline_v: cutlass.pipeline.PipelineAsync, - mbar_ptr_Q: cutlass.Pointer, - blocksparse_tensors: Optional[BlockSparseTensors], - block_info: BlockInfo, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - ): - warp_idx_in_wg = cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4 - if warp_idx_in_wg == 0: - q_producer_phase = Int32(1) - kv_producer_state = pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.num_stages - ) - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - # if work_tile.is_valid_tile: - m_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - mQ_cur = seqlen.offset_batch_Q(mQ, batch_idx, dim=3)[None, None, head_idx] - head_idx_kv = ( - head_idx // self.qhead_per_kvhead if const_expr(not self.pack_gqa) else head_idx - ) - mK_cur = seqlen.offset_batch_K(mK, batch_idx, dim=3)[None, None, head_idx_kv] - mV_cur = seqlen.offset_batch_K(mV, batch_idx, dim=3)[None, None, head_idx_kv] - gK = cute.local_tile(mK_cur, (self.tile_n, self.tile_hdim), (None, 0)) - gV = cute.local_tile(mV_cur, (self.tile_n, self.tile_hdimv), (None, 0)) - if const_expr(self.use_tma_Q): - gQ = cute.local_tile(mQ_cur, (self.tile_m, self.tile_hdim), (m_block, 0)) - load_Q, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_Q, 0, cute.make_layout(1), gQ, sQ, single_stage=True - ) - # TODO: mcast - # TODO check warp_idx if we have 128 producer threads - load_K, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_K, 0, cute.make_layout(1), gK, sK - ) - load_K = copy_utils.tma_producer_copy_fn(load_K, pipeline_k) - load_V, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_V, 0, cute.make_layout(1), gV, sV - ) - load_V = copy_utils.tma_producer_copy_fn(load_V, pipeline_v) - - if const_expr(not self.use_block_sparsity): - n_block_min, n_block_max = block_info.get_n_block_min_max(seqlen, m_block) - # if cute.arch.thread_idx()[0] == 0: - # cute.printf("m_block = %d, n_block_min: %d, n_block_max: %d", m_block, n_block_min, n_block_max) - # First iteration: load both Q & K with the same mbarrier - n_block = n_block_max - 1 - pipeline_k.producer_acquire( - kv_producer_state, - extra_tx_count=self.tma_copy_bytes["Q"] - if const_expr(self.use_tma_Q) - else 0, - ) - if const_expr(self.use_tma_Q): - load_Q(tma_bar_ptr=pipeline_k.producer_get_barrier(kv_producer_state)) - load_K(src_idx=n_block, producer_state=kv_producer_state) - - if const_expr(not self.intra_wg_overlap): - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block, producer_state=kv_producer_state) - kv_producer_state.advance() - for i in cutlass.range(n_block_max - 1 - n_block_min, unroll=1): - n_block = n_block_max - 1 - i - 1 - pipeline_k.producer_acquire(kv_producer_state) - load_K(src_idx=n_block, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block, producer_state=kv_producer_state) - kv_producer_state.advance() - else: - for i in cutlass.range(n_block_max - 1 - n_block_min, unroll=1): - n_block_prev = n_block_max - i - 1 - n_block = n_block_prev - 1 - kv_producer_state_prev = kv_producer_state.clone() - kv_producer_state.advance() - pipeline_k.producer_acquire(kv_producer_state) - load_K(src_idx=n_block, producer_state=kv_producer_state) - pipeline_v.producer_acquire(kv_producer_state_prev) - load_V(src_idx=n_block_prev, producer_state=kv_producer_state_prev) - n_block = n_block_min - pipeline_v.producer_acquire(kv_producer_state) - load_V(src_idx=n_block, producer_state=kv_producer_state) - kv_producer_state.advance() - else: - kv_producer_state = produce_block_sparse_loads( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_k, - pipeline_v, - self.use_tma_Q, - self.tma_copy_bytes["Q"], - self.intra_wg_overlap, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - - tile_scheduler.prefetch_next_work() - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def mma( - self, - tiled_mma_qk: cute.TiledMma, - tiled_mma_pv: cute.TiledMma, - tiled_mma_pv_rs: cute.TiledMma, - # softmax: Softmax, - # acc_O: cute.Tensor, - mQ: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - sQ: cute.Tensor, - sK: cute.Tensor, - sVt: cute.Tensor, - sP: Optional[cute.Tensor], - sO: cute.Tensor, - learnable_sink: Optional[cute.Tensor], - pipeline_k: cutlass.pipeline.PipelineAsync, - pipeline_v: cutlass.pipeline.PipelineAsync, - mbar_ptr_Q: cutlass.Pointer, - gmem_tiled_copy_Q: cute.TiledCopy, - gmem_tiled_copy_O: cute.TiledCopy, - tma_atom_O: Optional[cute.CopyAtom], - tidx: Int32, - softmax_scale_log2: Float32, - softmax_scale: Optional[Float32], - block_info: BlockInfo, - SeqlenInfoCls: Callable, - AttentionMaskCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors], - aux_tensors: Optional[list], - fastdiv_mods=None, - ): - warp_group_idx = cute.arch.make_warp_uniform(tidx // self.num_threads_per_warp_group) - warp_group_thread_layout = cute.make_layout( - self.num_mma_warp_groups, stride=self.num_threads_per_warp_group - ) - thr_mma_qk = tiled_mma_qk.get_slice(tidx) - wg_mma_qk = tiled_mma_qk.get_slice(warp_group_thread_layout(warp_group_idx)) - wg_mma_pv = tiled_mma_pv.get_slice(warp_group_thread_layout(warp_group_idx)) - tSrQ = tiled_mma_qk.make_fragment_A(wg_mma_qk.partition_A(sQ)) - tSrK = tiled_mma_qk.make_fragment_B(wg_mma_qk.partition_B(sK)) - if const_expr(self.mma_pv_is_rs): - acc_S_shape = tiled_mma_qk.partition_shape_C((self.tile_m, self.tile_n)) - tOrP = cute.make_fragment( - utils.convert_layout_acc_frgA(cute.make_layout(acc_S_shape)), self.dtype - ) - else: - tOrP = tiled_mma_pv.make_fragment_A(wg_mma_pv.partition_A(sP)) - tOrVt = tiled_mma_pv.make_fragment_B(wg_mma_pv.partition_B(sVt)) - - # /////////////////////////////////////////////////////////////////////////////// - # Smem copy atom tiling - # /////////////////////////////////////////////////////////////////////////////// - smem_copy_atom_P = utils.get_smem_store_atom(self.arch, self.dtype) - smem_thr_copy_P = cute.make_tiled_copy_C(smem_copy_atom_P, tiled_mma_qk).get_slice(tidx) - # tPsP = smem_thr_copy_P.partition_D(sP_pi) if const_expr(sP_pi is not None) else None - tPsP = smem_thr_copy_P.partition_D(sP) if const_expr(sP is not None) else None - # if cute.arch.thread_idx()[0] == 0: - # cute.printf(sP_pi.layout, sP_pi.iterator) - # cute.printf(sP.layout, sP.iterator) - # cute.printf(tPsP.layout, tPsP.iterator) - - self.mma_init() - - acc_shape_O = tiled_mma_pv.partition_shape_C((self.tile_m, self.tile_hdimv)) - acc_O = cute.make_fragment(acc_shape_O, Float32) - smem_copy_params = SimpleNamespace(smem_thr_copy_P=smem_thr_copy_P, tPsP=tPsP) - - mma_qk_fn = partial( - sm90_utils.gemm_zero_init, tiled_mma_qk, (self.tile_m, self.tile_n), tSrQ, tSrK - ) - mma_pv_fn = partial(sm90_utils.gemm_w_idx, tiled_mma_pv, acc_O, tOrP, tOrVt) - - mma_one_n_block_all = partial( - self.mma_one_n_block_intrawg_overlap - if const_expr(self.intra_wg_overlap) - else self.mma_one_n_block, - mma_qk_fn=mma_qk_fn, - tiled_mma_pv_rs=tiled_mma_pv_rs, - pipeline_k=pipeline_k, - pipeline_v=pipeline_v, - acc_O=acc_O, - tOrP=tOrP, - smem_copy_params=smem_copy_params, - check_inf=True, - ) - - q_consumer_phase = Int32(0) - kv_consumer_state = pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.num_stages - ) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - softmax = Softmax.create( - softmax_scale_log2, - num_rows=acc_O.shape[0][0] * acc_O.shape[1], - softmax_scale=softmax_scale, - ) - - process_first_half_block = partial( - self.first_half_block_overlap, - mma_qk_fn=mma_qk_fn, - pipeline_k=pipeline_k, - tOrP=tOrP, - smem_copy_params=smem_copy_params, - softmax=softmax, - ) - process_last_half_block = partial( - self.last_half_block_overlap, - pipeline_v=pipeline_v, - mma_pv_fn=mma_pv_fn, - ) - while work_tile.is_valid_tile: - # if work_tile.is_valid_tile: - - # shape: (atom_v_m * rest_m) - m_block, head_idx, batch_idx, _ = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - - # Recompute fastdiv_mods if necessary for varlen with aux_tensors - recompute_fastdiv_mods_q = cutlass.const_expr( - aux_tensors is not None and (seqlen.has_cu_seqlens_q or seqlen.has_seqused_q) - ) - recompute_fastdiv_mods_k = cutlass.const_expr( - aux_tensors is not None and (seqlen.has_cu_seqlens_k or seqlen.has_seqused_k) - ) - if cutlass.const_expr(fastdiv_mods is not None): - seqlen_q_divmod, seqlen_k_divmod = fastdiv_mods - fastdiv_mods = ( - seqlen_q_divmod - if not recompute_fastdiv_mods_q - else FastDivmodDivisor(seqlen.seqlen_q), - seqlen_k_divmod - if not recompute_fastdiv_mods_k - else FastDivmodDivisor(seqlen.seqlen_k), - ) - - mask = AttentionMaskCls(seqlen) - mask_fn = partial( - mask.apply_mask, - batch_idx=batch_idx, - head_idx=head_idx, - m_block=m_block, - thr_mma=thr_mma_qk, - mask_causal=self.is_causal, - mask_local=self.is_local, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - score_mod_fn = None - if const_expr(self.score_mod is not None): - score_mod_fn = partial( - self.apply_score_mod, - thr_mma_qk, - batch_idx, - head_idx, - m_block, - softmax_scale=softmax_scale, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - mma_one_n_block = partial( - mma_one_n_block_all, - seqlen=seqlen, - softmax=softmax, - score_mod_fn=score_mod_fn, - ) - # Load Q if not TMA_Q - if const_expr(not self.use_tma_Q): - pack_gqa = PackGQA( - self.tile_m, self.tile_hdim, self.check_hdim_oob, self.qhead_per_kvhead - ) - mQ_cur = seqlen.offset_batch_Q(mQ, batch_idx, dim=3)[None, None, head_idx] - # gmem_thr_copy_Q = gmem_tiled_copy_Q.get_slice(tidx) - # gQ = cute.local_tile(mQ_cur, (self.tile_m, self.tile_hdim), (m_block, 0)) - # self.load_Q(gmem_thr_copy_Q, gQ, sQ, m_block, seqlen=seqlen.seqlen_q, - # headdim=mQ.shape[1]) - pack_gqa.load_Q(mQ_cur, sQ, gmem_tiled_copy_Q, tidx, m_block, seqlen.seqlen_q) - cute.arch.cp_async_mbarrier_arrive_noinc(mbar_ptr_Q) - - n_block_min, n_block_max = block_info.get_n_block_min_max(seqlen, m_block) - if const_expr(not self.use_tma_Q): - cute.arch.mbarrier_wait(mbar_ptr_Q, phase=q_consumer_phase) - q_consumer_phase ^= 1 - # For performance reason, we separate out two kinds of iterations: - # those that need masking on S, and those that don't. - # We need masking on S for the very last block when K and V has length not multiple of tile_n. - # We also need masking on S if it's causal, for the last several blocks. - # softmax.reset() # Don't need reset as we explicitly call softmax w is_first=True - O_should_accumulate = False - - # ========================================== - # MAINLOOP - # ========================================== - if const_expr(not self.use_block_sparsity): - # ========================================== - # No block-sparsity (original path) - # ========================================== - # First iteration with seqlen masking - if const_expr(self.intra_wg_overlap): - kv_consumer_state = process_first_half_block( - n_block=n_block_max - 1, - seqlen=seqlen, - kv_consumer_state=kv_consumer_state, - mask_fn=partial(mask_fn, mask_mod=self.mask_mod), - score_mod_fn=score_mod_fn, - is_first_block=True, - ) - # Need to initialize tOrO in the case of RescaleOBeforeGemm where we will scale tOrO even in the 1st iter - # acc_O.fill(0.0) - else: - self.warp_scheduler_barrier_sync() - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=n_block_max - 1, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=True), - is_first_n_block=True, - mask_fn=partial(mask_fn, mask_mod=self.mask_mod, mask_seqlen=True), - ) - O_should_accumulate = True - # if cute.arch.thread_idx()[0] == 128: cute.printf("m_block = {}, n_block_max = {}, n_block_min = {}", m_block, n_block_max, n_block_min) - n_block_max -= 1 - # Next couple of iterations with causal masking - if const_expr(self.is_causal or self.is_local): - n_block_min_causal_local_mask = block_info.get_n_block_min_causal_local_mask( - seqlen, m_block, n_block_min - ) - # if cute.arch.thread_idx()[0] == 128: cute.printf("n_block_min_causal_local_mask = {}", n_block_min_causal_local_mask) - for n_tile in cutlass.range( - n_block_max - n_block_min_causal_local_mask, unroll=1 - ): - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=n_block_max - 1 - n_tile, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=self.mask_mod, mask_seqlen=False), - ) - O_should_accumulate = True - n_block_max = cutlass.min(n_block_max, n_block_min_causal_local_mask) - # The remaining iterations have no masking - n_block_min_before_local_mask = block_info.get_n_block_min_before_local_mask( - seqlen, m_block, n_block_min - ) - # if cute.arch.thread_idx()[0] == 128: cute.printf("n_block_min_before_local_mask = {}, n_block_min = {}", n_block_min_before_local_mask, n_block_min) - for n_tile in cutlass.range(n_block_max - n_block_min_before_local_mask, unroll=1): - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=n_block_max - 1 - n_tile, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=self.mask_mod, mask_seqlen=False), - ) - O_should_accumulate = True - # Separate iterations with local masking on the left - if const_expr(self.is_local and block_info.window_size_left is not None): - n_block_max = cutlass.min(n_block_max, n_block_min_before_local_mask) - for n_tile in cutlass.range(n_block_max - n_block_min, unroll=1): - kv_consumer_state = mma_one_n_block( - kv_consumer_state, - n_block=n_block_max - 1 - n_tile, - seqlen=seqlen, - mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate), - mask_fn=partial(mask_fn, mask_mod=self.mask_mod, mask_seqlen=False), - ) - O_should_accumulate = True - # Last "half" iteration - if const_expr(self.intra_wg_overlap): - kv_consumer_state = process_last_half_block( - kv_consumer_state=kv_consumer_state, - zero_init=not O_should_accumulate, - ) - O_should_accumulate = True - else: - self.warp_scheduler_barrier_arrive() - - else: - # ========================================== - # Block sparsity - # ========================================== - kv_consumer_state, O_should_accumulate, processed_any = consume_block_sparse_loads( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - seqlen, - kv_consumer_state, - mma_pv_fn, - mma_one_n_block, - process_first_half_block, - process_last_half_block, - mask_fn, - score_mod_fn, - O_should_accumulate, - self.mask_mod, - fastdiv_mods, - self.intra_wg_overlap, - self.warp_scheduler_barrier_sync, - self.warp_scheduler_barrier_arrive, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - - # Handle empty case (when no blocks to process) - if not processed_any: - softmax.reset() - acc_O.fill(0.0) - - sink_val = None - if const_expr(learnable_sink is not None): - if const_expr(not self.pack_gqa): - sink_val = Float32(learnable_sink[head_idx]) - else: # Each thread might have a different sink value due to different q_head - sink_val = cute.make_fragment_like(softmax.row_max, Float32) - cS = cute.make_identity_tensor((self.tile_m, self.tile_n)) - tScS_mn = utils.make_acc_tensor_mn_view(thr_mma_qk.partition_C(cS)) - for r in cutlass.range(cute.size(sink_val), unroll_full=True): - row = m_block * self.tile_m + tScS_mn[r][0] - q_head_idx = row % self.qhead_per_kvhead + head_idx * self.qhead_per_kvhead - sink_val[r] = Float32(learnable_sink[q_head_idx]) - - # normalize acc_O by row_sum and calculate the lse - row_scale = softmax.finalize(sink_val=sink_val) - softmax.rescale_O(acc_O, row_scale) - - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue - # /////////////////////////////////////////////////////////////////////////////// - self.epilogue( - acc_O, - softmax.row_sum, - mO, - mLSE, - sO, - seqlen, - gmem_tiled_copy_O, - tma_atom_O, - tiled_mma_pv, - tidx, - m_block, - head_idx, - batch_idx, - ) - - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - @cute.jit - def first_half_block_overlap( - self, - n_block: Int32, - mma_qk_fn: Callable, - kv_consumer_state, - pipeline_k, - tOrP: cute.Tensor, - smem_copy_params: SimpleNamespace, - softmax: Softmax, - seqlen: SeqlenInfoQK, - mask_fn: Callable = None, - score_mod_fn: Optional[Callable] = None, - is_first_block: bool = False, - ): - """Processes the first half block when using intra-warpgroup-overlap""" - - pipeline_k.consumer_wait(kv_consumer_state, pipeline_k.consumer_try_wait(kv_consumer_state)) - acc_S = mma_qk_fn(B_idx=kv_consumer_state.index, wg_wait=0) - pipeline_k.consumer_release(kv_consumer_state) - - # Apply score modification if present - if const_expr(score_mod_fn is not None): - score_mod_fn(acc_S, n_block=n_block, seqlen=seqlen) - - # Apply mask; mask_seqlen always True for first block - # Caveat: if full block further right than mask block, seqlen masking is redundant; - # however, masking is being applied anyway, so essentially no perf hit - mask_fn(acc_S, n_block=n_block, mask_seqlen=True) - - softmax.online_softmax(acc_S, is_first=is_first_block) - - tOrP_acc = cute.make_tensor(acc_S.iterator, utils.convert_layout_acc_frgA(acc_S.layout)) - tOrP_cur = ( - tOrP if const_expr(self.mma_pv_is_rs) else cute.make_fragment_like(tOrP_acc, self.dtype) - ) - tOrP_cur.store(tOrP_acc.load().to(self.dtype)) - - # if pv gemm not rs - if const_expr(not self.mma_pv_is_rs): - tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) - cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) - # Fence and barrier to make smem store visible to WGMMA - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.sync_warp() - - return kv_consumer_state - - @cute.jit - def last_half_block_overlap( - self, - kv_consumer_state, - pipeline_v, - mma_pv_fn: Callable, - zero_init: bool, - ): - """Processes the final PV GEMM when using intra-warpgroup-overlap""" - - pipeline_v.consumer_wait(kv_consumer_state, pipeline_v.consumer_try_wait(kv_consumer_state)) - mma_pv_fn(B_idx=kv_consumer_state.index, zero_init=zero_init, wg_wait=0) - pipeline_v.consumer_release(kv_consumer_state) - kv_consumer_state.advance() - return kv_consumer_state - - @cute.jit - def mma_one_n_block( - self, - smem_pipe_read: cutlass.pipeline.PipelineState | pipeline.PipelineStateSimple, - n_block: Int32, - mma_qk_fn: Callable, - mma_pv_fn: Callable, - tiled_mma_pv_rs: cute.TiledMma, - pipeline_k: cutlass.pipeline.PipelineAsync, - pipeline_v: cutlass.pipeline.PipelineAsync, - acc_O: cute.Tensor, - tOrP: cute.Tensor, - smem_copy_params: SimpleNamespace, - softmax: Softmax, - seqlen: SeqlenInfoQK, - score_mod_fn: Optional[Callable] = None, - mask_fn: Optional[Callable] = None, - is_first_n_block: cutlass.Constexpr = False, - check_inf: cutlass.Constexpr = True, - ): - pipeline_k.consumer_wait(smem_pipe_read, pipeline_k.consumer_try_wait(smem_pipe_read)) - # S = Q @ K.T - acc_S = mma_qk_fn(B_idx=smem_pipe_read.index, wg_wait=-1) - self.warp_scheduler_barrier_arrive() - warpgroup.wait_group(0) - pipeline_k.consumer_release(smem_pipe_read) - - # handle score mods and masking - if const_expr(score_mod_fn is not None): - score_mod_fn(acc_S, n_block=n_block, seqlen=seqlen) - if const_expr(mask_fn is not None): - mask_fn(acc_S=acc_S, n_block=n_block) - - row_scale = softmax.online_softmax(acc_S, is_first=is_first_n_block, check_inf=check_inf) - # if cute.arch.thread_idx()[0] == 0: cute.print_tensor(utils.make_acc_tensor_mn_view(acc_S)) - tOrP_acc = cute.make_tensor(acc_S.iterator, utils.convert_layout_acc_frgA(acc_S.layout)) - tOrP_cur = ( - tOrP if const_expr(self.mma_pv_is_rs) else cute.make_fragment_like(tOrP_acc, self.dtype) - ) - # tOrP.store(tOrP_acc.load().to(self.dtype)) - # the "to(self.dtype)" conversion fails to vectorize for block sizes other - # than 128 x 128, i.e. it calls convert on 1 fp32 element at a time instead of - # 2 elements. So we just call ptx directly. - utils.cvt_f16(tOrP_acc, tOrP_cur) - if const_expr(not self.mma_pv_is_rs): - tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) - cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) - softmax.rescale_O(acc_O, row_scale) - if const_expr(not self.mma_pv_is_rs): - # Fence and barrier to make sure smem store is visible to WGMMA - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV - pipeline_v.consumer_wait(smem_pipe_read, pipeline_v.consumer_try_wait(smem_pipe_read)) - self.warp_scheduler_barrier_sync() - # O += P @ V - mma_pv_fn(B_idx=smem_pipe_read.index, wg_wait=0) - pipeline_v.consumer_release(smem_pipe_read) - smem_pipe_read.advance() - return smem_pipe_read - - @cute.jit - def mma_one_n_block_intrawg_overlap( - self, - smem_pipe_read: cutlass.pipeline.PipelineState | pipeline.PipelineStateSimple, - n_block: Int32, - mma_qk_fn: Callable, - mma_pv_fn: Callable, - tiled_mma_pv_rs: cute.TiledMma, - pipeline_k: cutlass.pipeline.PipelineAsync, - pipeline_v: cutlass.pipeline.PipelineAsync, - acc_O: cute.Tensor, - tOrP: cute.Tensor, - smem_copy_params: SimpleNamespace, - softmax: Softmax, - seqlen: SeqlenInfoQK, - score_mod_fn: Optional[Callable] = None, - mask_fn: Optional[Callable] = None, - check_inf: cutlass.Constexpr = True, - ): - smem_pipe_read_v = smem_pipe_read.clone() - smem_pipe_read.advance() - pipeline_k.consumer_wait(smem_pipe_read, pipeline_k.consumer_try_wait(smem_pipe_read)) - self.warp_scheduler_barrier_sync() - # S = Q @ K.T - acc_S = mma_qk_fn(B_idx=smem_pipe_read.index, wg_wait=-1) - pipeline_v.consumer_wait(smem_pipe_read_v, pipeline_v.consumer_try_wait(smem_pipe_read_v)) - # O += P @ V - mma_pv_fn(B_idx=smem_pipe_read_v.index, wg_wait=-1) - self.warp_scheduler_barrier_arrive() - warpgroup.wait_group(1) - pipeline_k.consumer_release(smem_pipe_read) - - # handle score mods and masking - if const_expr(score_mod_fn is not None): - score_mod_fn(acc_S, n_block=n_block, seqlen=seqlen) - if const_expr(mask_fn is not None): - mask_fn(acc_S=acc_S, n_block=n_block) - # if cute.arch.thread_idx()[0] == 128: cute.print_tensor(utils.make_acc_tensor_mn_view(acc_S)) - - row_scale = softmax.online_softmax(acc_S, check_inf=check_inf) - warpgroup.wait_group(0) - pipeline_v.consumer_release(smem_pipe_read_v) - tOrP_acc = cute.make_tensor(acc_S.iterator, utils.convert_layout_acc_frgA(acc_S.layout)) - tOrP_cur = ( - tOrP if const_expr(self.mma_pv_is_rs) else cute.make_fragment_like(tOrP_acc, self.dtype) - ) - # tOrP_cur.store(tOrP_acc.load().to(self.dtype)) - # the "to(self.dtype)" conversion fails to vectorize for block sizes other - # than 128 x 128, i.e. it calls convert on 1 fp32 element at a time instead of - # 2 elements. So we just call ptx directly. - utils.cvt_f16(tOrP_acc, tOrP_cur) - if const_expr(not self.mma_pv_is_rs): - tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) - cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) - softmax.rescale_O(acc_O, row_scale) - if const_expr(not self.mma_pv_is_rs): - # Fence and barrier to make sure smem store is visible to WGMMA - cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV - return smem_pipe_read - - @cute.jit - def mma_init(self): - warp_group_idx = utils.canonical_warp_group_idx(sync=False) - if const_expr(self.use_scheduler_barrier): - if warp_group_idx == 1: - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1), - number_of_threads=2 * self.num_threads_per_warp_group, - ) - - @cute.jit - def apply_score_mod( - self, - thr_mma_qk, - batch_idx, - head_idx, - m_block, - acc_S, - n_block, - softmax_scale, - seqlen, - aux_tensors: Optional[list] = None, - fastdiv_mods=None, - ): - # Prepare index tensor - cS = cute.make_identity_tensor((self.tile_m, self.tile_n)) - cS = cute.domain_offset((m_block * self.tile_m, n_block * self.tile_n), cS) - tScS = thr_mma_qk.partition_C(cS) - - apply_score_mod_inner( - acc_S, - tScS, - self.score_mod, - batch_idx, - head_idx, - softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info=seqlen, - constant_q_idx=None, - qhead_per_kvhead=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - - def warp_scheduler_barrier_sync(self): - if const_expr(self.use_scheduler_barrier): - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) - - 1 - + utils.canonical_warp_group_idx(sync=False), - number_of_threads=2 * self.num_threads_per_warp_group, - ) - - def warp_scheduler_barrier_arrive(self): - if const_expr(self.use_scheduler_barrier): - assert self.num_mma_warp_groups in [2, 3] - cur_wg = utils.canonical_warp_group_idx(sync=False) - 1 - if const_expr(self.num_mma_warp_groups == 2): - next_wg = 1 - cur_wg - else: - t = cur_wg + 1 - next_wg = t % self.num_mma_warp_groups - cute.arch.barrier_arrive( - barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + next_wg, - number_of_threads=2 * self.num_threads_per_warp_group, - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_combine.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_combine.py deleted file mode 100644 index 2a5cb933b2cc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_combine.py +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# A reimplementation of https://github.com/Dao-AILab/flash-attention/blob/main/hopper/flash_fwd_combine_kernel.h -# from Cutlass C++ to Cute-DSL. -import math -import operator -from typing import Type, Optional -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass.cute.nvgpu import cpasync -from cutlass import Float32, Int32, const_expr - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .seqlen_info import SeqlenInfo -from cutlass.cute import FastDivmodDivisor - - -class FlashAttentionForwardCombine: - def __init__( - self, - dtype: Type[cutlass.Numeric], - dtype_partial: Type[cutlass.Numeric], - head_dim: int, - m_block_size: int = 8, - k_block_size: int = 64, - log_max_splits: int = 4, - num_threads: int = 256, - stages: int = 4, - ): - """ - Forward combine kernel for split attention computation. - - :param dtype: output data type - :param dtype_partial: partial accumulation data type - :param head_dim: head dimension - :param m_block_size: m block size - :param k_block_size: k block size - :param log_max_splits: log2 of maximum splits - :param num_threads: number of threads - :param varlen: whether using variable length sequences - :param stages: number of pipeline stages - """ - self.dtype = dtype - self.dtype_partial = dtype_partial - self.head_dim = head_dim - self.m_block_size = m_block_size - self.k_block_size = k_block_size - self.max_splits = 1 << log_max_splits - self.num_threads = num_threads - self.is_even_k = head_dim % k_block_size == 0 - self.stages = stages - - @staticmethod - def can_implement( - dtype, - dtype_partial, - head_dim, - m_block_size, - k_block_size, - log_max_splits, - num_threads, - ) -> bool: - """Check if the kernel can be implemented with the given parameters.""" - if dtype not in [cutlass.Float16, cutlass.BFloat16, cutlass.Float32]: - return False - if dtype_partial not in [cutlass.Float16, cutlass.BFloat16, Float32]: - return False - if head_dim % 8 != 0: - return False - if num_threads % 32 != 0: - return False - if m_block_size % 8 != 0: - return False - max_splits = 1 << log_max_splits - if max_splits > 256: - return False - if (m_block_size * max_splits) % num_threads != 0: - return False - return True - - def _setup_attributes(self): - # GMEM copy setup for O partial - universal_copy_bits = 128 - async_copy_elems = universal_copy_bits // self.dtype_partial.width - assert self.k_block_size % async_copy_elems == 0 - - k_block_gmem = ( - 128 if self.k_block_size % 128 == 0 else (64 if self.k_block_size % 64 == 0 else 32) - ) - gmem_threads_per_row = k_block_gmem // async_copy_elems - assert self.num_threads % gmem_threads_per_row == 0 - - # Async copy atom for O partial load - atom_async_copy_partial = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - self.dtype_partial, - num_bits_per_copy=universal_copy_bits, - ) - tOpartial_layout = cute.make_ordered_layout( - (self.num_threads // gmem_threads_per_row, gmem_threads_per_row), - order=(1, 0), - ) - vOpartial_layout = cute.make_layout((1, async_copy_elems)) # 4 vals per load - self.gmem_tiled_copy_O_partial = cute.make_tiled_copy_tv( - atom_async_copy_partial, tOpartial_layout, vOpartial_layout - ) - - # GMEM copy setup for final O (use universal copy for store) - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dtype, - num_bits_per_copy=async_copy_elems * self.dtype.width, - ) - self.gmem_tiled_copy_O = cute.make_tiled_copy_tv( - atom_universal_copy, - tOpartial_layout, - vOpartial_layout, # 4 vals per store - ) - - # LSE copy setup with async copy (alignment = 1) - lse_copy_bits = Float32.width # 1 element per copy, width is in bits - m_block_smem = ( - 128 - if self.m_block_size % 128 == 0 - else ( - 64 - if self.m_block_size % 64 == 0 - else ( - 32 - if self.m_block_size % 32 == 0 - else (16 if self.m_block_size % 16 == 0 else 8) - ) - ) - ) - gmem_threads_per_row_lse = m_block_smem - assert self.num_threads % gmem_threads_per_row_lse == 0 - - # Async copy atom for LSE load - atom_async_copy_lse = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.ALWAYS), - Float32, - num_bits_per_copy=lse_copy_bits, - ) - tLSE_layout = cute.make_ordered_layout( - (self.num_threads // gmem_threads_per_row_lse, gmem_threads_per_row_lse), - order=(1, 0), - ) - vLSE_layout = cute.make_layout(1) - self.gmem_tiled_copy_LSE = cute.make_tiled_copy_tv( - atom_async_copy_lse, tLSE_layout, vLSE_layout - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Shared memory - # /////////////////////////////////////////////////////////////////////////////// - - # Shared memory to register copy for LSE - self.smem_threads_per_col_lse = self.num_threads // m_block_smem - assert 32 % self.smem_threads_per_col_lse == 0 # Must divide warp size - - s2r_layout_atom_lse = cute.make_ordered_layout( - (self.smem_threads_per_col_lse, self.num_threads // self.smem_threads_per_col_lse), - order=(0, 1), - ) - self.s2r_tiled_copy_LSE = cute.make_tiled_copy_tv( - cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Float32), - s2r_layout_atom_lse, - cute.make_layout(1), - ) - - # LSE shared memory layout with swizzling to avoid bank conflicts - # This works for kBlockMSmem = 8, 16, 32, 64, 128, no bank conflicts - if const_expr(m_block_smem == 8): - smem_lse_swizzle = cute.make_swizzle(5, 0, 5) - elif const_expr(m_block_smem == 16): - smem_lse_swizzle = cute.make_swizzle(4, 0, 4) - else: - smem_lse_swizzle = cute.make_swizzle(3, 2, 3) - smem_layout_atom_lse = cute.make_composed_layout( - smem_lse_swizzle, 0, cute.make_ordered_layout((8, m_block_smem), order=(1, 0)) - ) - self.smem_layout_lse = cute.tile_to_shape( - smem_layout_atom_lse, (self.max_splits, self.m_block_size), (0, 1) - ) - - # O partial shared memory layout (simple layout for pipeline stages) - self.smem_layout_o = cute.make_ordered_layout( - (self.m_block_size, self.k_block_size, self.stages), order=(1, 0, 2) - ) - - @cute.jit - def __call__( - self, - mO_partial: cute.Tensor, - mLSE_partial: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor] = None, - cu_seqlens: Optional[cute.Tensor] = None, - seqused: Optional[cute.Tensor] = None, - num_splits_dynamic_ptr: Optional[cute.Tensor] = None, - semaphore_to_reset: Optional[cute.Tensor] = None, - stream: cuda.CUstream = None, - ): - # Type checking - if const_expr(not (mO_partial.element_type == self.dtype_partial)): - raise TypeError("O partial tensor must match dtype_partial") - if const_expr(not (mO.element_type == self.dtype)): - raise TypeError("O tensor must match dtype") - if const_expr(mLSE_partial.element_type not in [Float32]): - raise TypeError("LSE partial tensor must be Float32") - if const_expr(mLSE is not None and mLSE.element_type not in [Float32]): - raise TypeError("LSE tensor must be Float32") - - # Shape validation - input tensors are in user format, need to be converted to kernel format - if const_expr(len(mO_partial.shape) not in [4, 5]): - raise ValueError( - "O partial tensor must have 4 or 5 dimensions: (num_splits, batch, seqlen, nheads, headdim) or (num_splits, total_q, nheads, headdim)" - ) - if const_expr(len(mLSE_partial.shape) not in [3, 4]): - raise ValueError( - "LSE partial tensor must have 3 or 4 dimensions: (num_splits, batch, seqlen, nheads) or (num_splits, total_q, nheads)" - ) - if const_expr(len(mO.shape) not in [3, 4]): - raise ValueError( - "O tensor must have 3 or 4 dimensions: (batch, seqlen, nheads, headdim) or (total_q, nheads, headdim)" - ) - if const_expr(mLSE is not None and len(mLSE.shape) not in [2, 3]): - raise ValueError( - "LSE tensor must have 2 or 3 dimensions: (batch, seqlen, nheads) or (total_q, nheads)" - ) - - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mO_partial, mO = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - for t in (mO_partial, mO) - ] - # (num_splits, b, seqlen, h, d) -> (seqlen, d, num_splits, h, b) - # or (num_splits, total_q, h, d) -> (total_q, d, num_splits, h) - O_partial_layout_transpose = ( - [2, 4, 0, 3, 1] if const_expr(cu_seqlens is None) else [1, 3, 0, 2] - ) - # (b, seqlen, h, d) -> (seqlen, d, h, b) or (total_q, h, d) -> (total_q, d, h) - mO_partial = cute.make_tensor( - mO_partial.iterator, cute.select(mO_partial.layout, mode=O_partial_layout_transpose) - ) - O_layout_transpose = [1, 3, 2, 0] if const_expr(cu_seqlens is None) else [0, 2, 1] - mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=O_layout_transpose)) - # (num_splits, b, seqlen, h) -> (seqlen, num_splits, h, b) - # or (num_splits, total_q, h) -> (total_q, num_splits, h) - LSE_partial_layout_transpose = [2, 0, 3, 1] if const_expr(cu_seqlens is None) else [1, 0, 2] - mLSE_partial = cute.make_tensor( - mLSE_partial.iterator, - cute.select(mLSE_partial.layout, mode=LSE_partial_layout_transpose), - ) - # (b, seqlen, h) -> (seqlen, h, b) or (total_q, h) -> (total_q, h) - LSE_layout_transpose = [1, 2, 0] if const_expr(cu_seqlens is None) else [0, 1] - mLSE = ( - cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=LSE_layout_transpose)) - if mLSE is not None - else None - ) - - # Determine if we have variable length sequences - varlen = const_expr(cu_seqlens is not None or seqused is not None) - - self._setup_attributes() - - @cute.struct - class SharedStorage: - sLSE: cute.struct.Align[ - cute.struct.MemRange[Float32, cute.cosize(self.smem_layout_lse)], 128 - ] - sMaxValidSplit: cute.struct.Align[cute.struct.MemRange[Int32, self.m_block_size], 128] - sO: cute.struct.Align[ - cute.struct.MemRange[self.dtype_partial, cute.cosize(self.smem_layout_o)], 128 - ] - - smem_size = SharedStorage.size_in_bytes() - - # Grid dimensions: (ceil_div(seqlen, m_block), ceil_div(head_dim, k_block), num_head * batch) - seqlen = mO_partial.shape[0] - num_head = mO_partial.shape[3] - batch_size = ( - mO_partial.shape[4] - if const_expr(cu_seqlens is None) - else Int32(cu_seqlens.shape[0] - 1) - ) - - # Create FastDivmodDivisor objects for efficient division - seqlen_divmod = FastDivmodDivisor(seqlen) - head_divmod = FastDivmodDivisor(num_head) - - grid_dim = ( - cute.ceil_div(seqlen * num_head, self.m_block_size), - cute.ceil_div(self.head_dim, self.k_block_size), - batch_size, - ) - - self.kernel( - mO_partial, - mLSE_partial, - mO, - mLSE, - cu_seqlens, - seqused, - num_splits_dynamic_ptr, - semaphore_to_reset, - SharedStorage, - self.smem_layout_lse, - self.smem_layout_o, - self.gmem_tiled_copy_O_partial, - self.gmem_tiled_copy_O, - self.gmem_tiled_copy_LSE, - self.s2r_tiled_copy_LSE, - seqlen_divmod, - head_divmod, - varlen, - ).launch( - grid=grid_dim, - block=[self.num_threads, 1, 1], - smem=smem_size, - stream=stream, - ) - - @cute.kernel - def kernel( - self, - mO_partial: cute.Tensor, - mLSE_partial: cute.Tensor, - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - cu_seqlens: Optional[cute.Tensor], - seqused: Optional[cute.Tensor], - num_splits_dynamic_ptr: Optional[cute.Tensor], - semaphore_to_reset: Optional[cute.Tensor], - SharedStorage: cutlass.Constexpr, - smem_layout_lse: cute.Layout | cute.ComposedLayout, - smem_layout_o: cute.Layout, - gmem_tiled_copy_O_partial: cute.TiledCopy, - gmem_tiled_copy_O: cute.TiledCopy, - gmem_tiled_copy_LSE: cute.TiledCopy, - s2r_tiled_copy_LSE: cute.TiledCopy, - seqlen_divmod: FastDivmodDivisor, - head_divmod: FastDivmodDivisor, - varlen: cutlass.Constexpr[bool], - ): - # Thread and block indices - tidx, _, _ = cute.arch.thread_idx() - m_block, k_block, batch_idx = cute.arch.block_idx() - - # /////////////////////////////////////////////////////////////////////////////// - # Get shared memory buffer - # /////////////////////////////////////////////////////////////////////////////// - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(SharedStorage) - sLSE = storage.sLSE.get_tensor(smem_layout_lse) - sMaxValidSplit = storage.sMaxValidSplit.get_tensor((self.m_block_size,)) - sO = storage.sO.get_tensor(smem_layout_o) - - # Handle semaphore reset - if const_expr(semaphore_to_reset is not None): - if ( - tidx == 0 - and m_block == cute.arch.grid_dim()[0] - 1 - and k_block == cute.arch.grid_dim()[1] - 1 - and batch_idx == cute.arch.grid_dim()[2] - 1 - ): - semaphore_to_reset[0] = 0 - - # Get number of splits - num_splits = ( - num_splits_dynamic_ptr[batch_idx] - if const_expr(num_splits_dynamic_ptr is not None) - else mLSE_partial.shape[1] - ) - # Handle variable length sequences using SeqlenInfo - seqlen_info = SeqlenInfo.create( - batch_idx=batch_idx, - seqlen_static=mO_partial.shape[0], - cu_seqlens=cu_seqlens, - seqused=seqused, - ) - seqlen, offset = seqlen_info.seqlen, seqlen_info.offset - - # Extract number of heads (head index will be determined dynamically) - num_head = mO_partial.shape[3] - max_idx = seqlen * num_head - - # Early exit for single split if dynamic - if (const_expr(num_splits_dynamic_ptr is None) or num_splits > 1) and ( - const_expr(not varlen) or m_block * self.m_block_size < max_idx - ): - # =============================== - # Step 1: Load LSE_partial from gmem to shared memory - # =============================== - - if const_expr(cu_seqlens is None): - # mLSE_partial_cur = mLSE_partial[None, None, None, batch_idx] - mLSE_partial_cur = utils.coord_offset_i64(mLSE_partial, batch_idx, dim=3) - else: - # mLSE_partial_cur = cute.domain_offset((offset, 0, 0), mLSE_partial) - mLSE_partial_cur = utils.domain_offset_i64((offset, 0, 0), mLSE_partial) - mLSE_partial_copy = cute.tiled_divide(mLSE_partial_cur, (1,)) - - gmem_thr_copy_LSE = gmem_tiled_copy_LSE.get_slice(tidx) - tLSEsLSE = gmem_thr_copy_LSE.partition_D(sLSE) - - # Create identity tensor for coordinate tracking - cLSE = cute.make_identity_tensor((self.max_splits, self.m_block_size)) - tLSEcLSE = gmem_thr_copy_LSE.partition_S(cLSE) - - # Load LSE partial values - for m in cutlass.range(cute.size(tLSEcLSE, mode=[2]), unroll_full=True): - mi = tLSEcLSE[0, 0, m][1] # Get m coordinate - idx = m_block * self.m_block_size + mi - if idx < max_idx: - # Calculate actual sequence position and head using FastDivmodDivisor - if const_expr(not varlen): - head_idx, m_idx = divmod(idx, seqlen_divmod) - else: - head_idx = idx // seqlen - m_idx = idx - head_idx * seqlen - mLSE_partial_cur_copy = mLSE_partial_copy[None, m_idx, None, head_idx] - for s in cutlass.range(cute.size(tLSEcLSE, mode=[1]), unroll_full=True): - si = tLSEcLSE[0, s, 0][0] # Get split coordinate - if si < num_splits: - cute.copy( - gmem_thr_copy_LSE, - mLSE_partial_cur_copy[None, si], - tLSEsLSE[None, s, m], - ) - else: - tLSEsLSE[None, s, m].fill(-Float32.inf) - # Don't need to zero out the rest of the LSEs, as we will not write the output to gmem - cute.arch.cp_async_commit_group() - - # =============================== - # Step 2: Load O_partial for pipeline stages - # =============================== - - gmem_thr_copy_O_partial = gmem_tiled_copy_O_partial.get_slice(tidx) - cO = cute.make_identity_tensor((self.m_block_size, self.k_block_size)) - tOcO = gmem_thr_copy_O_partial.partition_D(cO) - tOsO_partial = gmem_thr_copy_O_partial.partition_D(sO) - if const_expr(cu_seqlens is None): - # mO_partial_cur = mO_partial[None, None, None, None, batch_idx] - mO_partial_cur = utils.coord_offset_i64(mO_partial, batch_idx, dim=4) - else: - # mO_partial_cur = cute.domain_offset((offset, 0, 0, 0), mO_partial) - mO_partial_cur = utils.domain_offset_i64((offset, 0, 0, 0), mO_partial) - - # Precompute these values to avoid recomputing them in the loop - num_rows = const_expr(cute.size(tOcO, mode=[1])) - tOmidx = cute.make_fragment(num_rows, cutlass.Int32) - tOhidx = cute.make_fragment(num_rows, cutlass.Int32) - tOrOptr = cute.make_fragment(num_rows, cutlass.Int64) - for m in cutlass.range(num_rows, unroll_full=True): - mi = tOcO[0, m, 0][0] # m coordinate - idx = m_block * self.m_block_size + mi - if const_expr(not varlen): - tOhidx[m], tOmidx[m] = divmod(idx, seqlen_divmod) - else: - tOhidx[m] = idx // seqlen - tOmidx[m] = idx - tOhidx[m] * seqlen - tOrOptr[m] = utils.elem_pointer_i64( - mO_partial_cur, (tOmidx[m], k_block * self.k_block_size, 0, tOhidx[m]) - ).toint() - if idx >= max_idx: - tOhidx[m] = -1 - - tOpO = cute.make_fragment(cute.size(tOcO, [2]), cutlass.Boolean) - if const_expr(not self.is_even_k): - for k in cutlass.range(cute.size(tOpO), unroll_full=True): - tOpO[k] = tOcO[0, 0, k][1] < mO_partial.shape[1] - k_block * self.k_block_size - # if cute.arch.thread_idx()[0] == 0 and k_block == 1: cute.print_tensor(tOpO) - - load_O_partial = partial( - self.load_O_partial, - gmem_tiled_copy_O_partial, - tOrOptr, - tOsO_partial, - tOhidx, - tOpO, - tOcO, - mO_partial_cur.layout, - ) - - # Load first few stages of O_partial - for stage in cutlass.range(self.stages - 1, unroll_full=True): - if stage < num_splits: - load_O_partial(stage, stage) - cute.arch.cp_async_commit_group() - - # =============================== - # Step 3: Load and transpose LSE from smem to registers - # =============================== - - # Wait for LSE and initial O partial stages to complete - cute.arch.cp_async_wait_group(self.stages - 1) - cute.arch.sync_threads() - # if cute.arch.thread_idx()[0] == 0: - # # cute.print_tensor(sLSE) - # for i in range(64): - # cute.printf("sLSE[%d, 0] = %f", i, sLSE[i, 0]) - # cute.arch.sync_threads() - - s2r_thr_copy_LSE = s2r_tiled_copy_LSE.get_slice(tidx) - ts2rsLSE = s2r_thr_copy_LSE.partition_S(sLSE) - ts2rrLSE = cute.make_fragment_like(ts2rsLSE) - cute.copy(s2r_tiled_copy_LSE, ts2rsLSE, ts2rrLSE) - - # =============================== - # Step 4: Compute final LSE along split dimension - # =============================== - - lse_sum = cute.make_fragment(cute.size(ts2rrLSE, mode=[2]), Float32) - ts2rcLSE = s2r_thr_copy_LSE.partition_D(cLSE) - # We compute the max valid split for each row to short-circuit the computation later - max_valid_split = cute.make_fragment(cute.size(ts2rrLSE, mode=[2]), Int32) - assert cute.size(ts2rrLSE, mode=[0]) == 1 - # Compute max, scales, and final LSE for each row - for m in cutlass.range(cute.size(ts2rrLSE, mode=[2]), unroll_full=True): - # Find max LSE value across splits - threads_per_col = const_expr(self.smem_threads_per_col_lse) - lse_max = utils.warp_reduce( - ts2rrLSE[None, None, m] - .load() - .reduce(cute.ReductionOp.MAX, init_val=-Float32.inf, reduction_profile=0), - op=cute.arch.fmax, - width=threads_per_col, - ) - # if cute.arch.thread_idx()[0] == 0: cute.printf(lse_max) - # Find max valid split index - max_valid_idx = -1 - for s in cutlass.range(cute.size(ts2rrLSE, mode=[1]), unroll_full=True): - if ts2rrLSE[0, s, m] != -Float32.inf: - max_valid_idx = ts2rcLSE[0, s, 0][0] # Get split coordinate - # if cute.arch.thread_idx()[0] < 32: cute.printf(max_valid_idx) - max_valid_split[m] = utils.warp_reduce(max_valid_idx, max, width=threads_per_col) - # Compute exp scales and sum - lse_max_cur = ( - 0.0 if lse_max == -Float32.inf else lse_max - ) # In case all local LSEs are -inf - LOG2_E = math.log2(math.e) - lse_sum_cur = 0.0 - for s in cutlass.range(cute.size(ts2rrLSE, mode=[1]), unroll_full=True): - scale = utils.exp2f(ts2rrLSE[0, s, m] * LOG2_E - (lse_max_cur * LOG2_E)) - lse_sum_cur += scale - ts2rrLSE[0, s, m] = scale # Store scale for later use - lse_sum_cur = utils.warp_reduce(lse_sum_cur, operator.add, width=threads_per_col) - lse_sum[m] = utils.logf(lse_sum_cur) + lse_max - # Normalize scales - inv_sum = ( - 0.0 if (lse_sum_cur == 0.0 or lse_sum_cur != lse_sum_cur) else 1.0 / lse_sum_cur - ) - ts2rrLSE[None, None, m].store(ts2rrLSE[None, None, m].load() * inv_sum) - # Store the scales exp(lse - lse_logsum) back to smem - cute.copy(s2r_tiled_copy_LSE, ts2rrLSE, ts2rsLSE) - - # Store max valid split to smem - for m in cutlass.range(cute.size(ts2rrLSE, mode=[2]), unroll_full=True): - if ts2rcLSE[0, 0, m][0] == 0: # Only thread responsible for s=0 writes - mi = ts2rcLSE[0, 0, m][1] - if mi < self.m_block_size: - sMaxValidSplit[mi] = max_valid_split[m] - - # =============================== - # Step 5: Store final LSE to gmem - # =============================== - - if const_expr(mLSE is not None): - if const_expr(cu_seqlens is None): - # mLSE_cur = mLSE[None, None, batch_idx] - mLSE_cur = utils.coord_offset_i64(mLSE, batch_idx, dim=2) - else: - # mLSE_cur = cute.domain_offset((offset, 0), mLSE) - mLSE_cur = utils.domain_offset_i64((offset, 0), mLSE) - if k_block == 0: # Only first k_block writes LSE when mLSE is provided - for m in cutlass.range(cute.size(ts2rrLSE, mode=[2]), unroll_full=True): - if ts2rcLSE[0, 0, m][0] == 0: # Only thread responsible for s=0 writes - mi = ts2rcLSE[0, 0, m][1] - idx = m_block * self.m_block_size + mi - if idx < max_idx: - if const_expr(not varlen): - head_idx, m_idx = divmod(idx, seqlen_divmod) - else: - head_idx = idx // seqlen - m_idx = idx - head_idx * seqlen - mLSE_cur[m_idx, head_idx] = lse_sum[m] - - # =============================== - # Step 6: Read O_partial and accumulate final O - # =============================== - - cute.arch.sync_threads() - - # Get max valid split for this thread - thr_max_valid_split = sMaxValidSplit[tOcO[0, 0, 0][0]] - for m in cutlass.range(1, cute.size(tOcO, mode=[1])): - thr_max_valid_split = max(thr_max_valid_split, sMaxValidSplit[tOcO[0, m, 0][0]]) - - tOrO_partial = cute.make_fragment_like(tOsO_partial[None, None, None, 0]) - tOrO = cute.make_fragment_like(tOrO_partial, Float32) - tOrO.fill(0.0) - - stage_load = self.stages - 1 - stage_compute = 0 - - # Main accumulation loop - for s in cutlass.range(thr_max_valid_split + 1, unroll=4): - # Get scales for this split - scale = cute.make_fragment(num_rows, Float32) - for m in cutlass.range(num_rows, unroll_full=True): - scale[m] = sLSE[s, tOcO[0, m, 0][0]] # Get scale from smem - - # Load next stage if needed - split_to_load = s + self.stages - 1 - if split_to_load <= thr_max_valid_split: - load_O_partial(split_to_load, stage_load) - cute.arch.cp_async_commit_group() - stage_load = 0 if stage_load == self.stages - 1 else stage_load + 1 - - # Wait for the current stage to be ready - cute.arch.cp_async_wait_group(self.stages - 1) - # We don't need __syncthreads() because each thread is just reading its own data from smem - # Copy from smem to registers - cute.autovec_copy(tOsO_partial[None, None, None, stage_compute], tOrO_partial) - stage_compute = 0 if stage_compute == self.stages - 1 else stage_compute + 1 - - # Accumulate scaled partial results - for m in cutlass.range(num_rows, unroll_full=True): - if tOhidx[m] >= 0 and scale[m] > 0.0: - tOrO[None, m, None].store( - tOrO[None, m, None].load() - + scale[m] * tOrO_partial[None, m, None].load().to(Float32) - ) - - # =============================== - # Step 7: Write final O to gmem - # =============================== - - rO = cute.make_fragment_like(tOrO, self.dtype) - rO.store(tOrO.load().to(self.dtype)) - if const_expr(cu_seqlens is None): - # mO_cur = mO[None, None, None, batch_idx] - mO_cur = utils.coord_offset_i64(mO, batch_idx, dim=3) - else: - # mO_cur = cute.domain_offset((offset, 0, 0), mO) - mO_cur = utils.domain_offset_i64((offset, 0, 0), mO) - mO_cur = utils.domain_offset_aligned((0, k_block * self.k_block_size, 0), mO_cur) - elems_per_store = const_expr(cute.size(gmem_tiled_copy_O.layout_tv_tiled[1])) - # mO_cur_copy = cute.tiled_divide(mO_cur, (1, elems_per_store,)) - gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx) - # Write final results - for m in cutlass.range(num_rows, unroll_full=True): - if tOhidx[m] >= 0: - mO_cur_copy = cute.tiled_divide( - mO_cur[tOmidx[m], None, tOhidx[m]], (elems_per_store,) - ) - for k in cutlass.range(cute.size(tOcO, mode=[2]), unroll_full=True): - k_idx = tOcO[0, 0, k][1] // elems_per_store - if const_expr(self.is_even_k) or tOpO[k]: - cute.copy(gmem_thr_copy_O, rO[None, m, k], mO_cur_copy[None, k_idx]) - - @cute.jit - def load_O_partial( - self, - gmem_tiled_copy_O_partial: cute.TiledCopy, - tOrOptr: cute.Tensor, - tOsO_partial: cute.Tensor, - tOhidx: cute.Tensor, - tOpO: cute.Tensor, - tOcO: cute.Tensor, - mO_cur_partial_layout: cute.Layout, - split: Int32, - stage: Int32, - ) -> None: - elems_per_load = const_expr(cute.size(gmem_tiled_copy_O_partial.layout_tv_tiled[1])) - tOsO_partial_cur = tOsO_partial[None, None, None, stage] - for m in cutlass.range(cute.size(tOcO, [1]), unroll_full=True): - if tOhidx[m] >= 0: - o_gmem_ptr = cute.make_ptr( - tOsO_partial.element_type, tOrOptr[m], cute.AddressSpace.gmem, assumed_align=16 - ) - mO_partial_cur = cute.make_tensor( - o_gmem_ptr, cute.slice_(mO_cur_partial_layout, (0, None, None, 0)) - ) - mO_partial_cur_copy = cute.tiled_divide(mO_partial_cur, (elems_per_load,)) - for k in cutlass.range(cute.size(tOcO, mode=[2]), unroll_full=True): - k_idx = tOcO[0, 0, k][1] // elems_per_load - if const_expr(self.is_even_k) or tOpO[k]: - cute.copy( - gmem_tiled_copy_O_partial, - # mO_partial_cur_copy[None, k_idx, split], - utils.coord_offset_i64(mO_partial_cur_copy, split, dim=2)[None, k_idx], - tOsO_partial_cur[None, m, k], - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_sm100.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_sm100.py deleted file mode 100644 index d6cb154c2154..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/flash_fwd_sm100.py +++ /dev/null @@ -1,2815 +0,0 @@ -# Supported features: -# - BF16 & FP16 dtype -# - noncausal & causal attention -# - MHA, GQA, MQA -# - hdim 64, 96, 128, (192, 128). -# - varlen -# - sliding window -# - split-kv -# Unsupported features that will be added later: -# - page size != 128 -# - more hdim (192, 256) -# Based on the cutlass example and cute-dsl example: -# https://github.com/NVIDIA/cutlass/tree/main/examples/77_blackwell_fmha -# https://github.com/NVIDIA/cutlass/blob/main/examples/python/CuTeDSL/blackwell/fmha.py - -import enum -import math -from typing import Tuple, Callable, Optional, Literal -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass import Float32, Int32, const_expr -from cutlass.cute.nvgpu import cpasync -import cutlass.cute.nvgpu.tcgen05 as tcgen05 -import cutlass.utils.blackwell_helpers as sm100_utils_basic - -from .paged_kv import PagedKVManager -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.copy_utils as copy_utils -from .mask import AttentionMask -from .softmax import SoftmaxSm100, apply_score_mod_inner -from .seqlen_info import SeqlenInfoQK -from .block_info import BlockInfo -from .block_sparsity import BlockSparseTensors -from .block_sparse_utils import ( - get_total_block_count, - produce_block_sparse_loads_sm100, - softmax_block_sparse_sm100, - handle_block_sparse_empty_tile_correction_sm100, -) -from .pack_gqa import PackGQA -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.blackwell_helpers as sm100_utils -from cutlass.cute import FastDivmodDivisor -from .tile_scheduler import ( - TileSchedulerArguments, - SingleTileScheduler, - StaticPersistentTileScheduler, - SingleTileLPTScheduler, - SingleTileVarlenScheduler, - ParamsBase, -) - - -class NamedBarrierFwd(enum.IntEnum): - Epilogue = enum.auto() # starts from 1 as barrier 0 is reserved for sync_threads() - - -# WarpSchedulerWG1 = enum.auto() -# WarpSchedulerWG2 = enum.auto() -# WarpSchedulerWG3 = enum.auto() -# PFull = enum.auto() -# PEmpty = enum.auto() - - -class FlashAttentionForwardSm100: - arch = 100 - - def __init__( - self, - # dtype: Type[cutlass.Numeric], - head_dim: int, - head_dim_v: Optional[int] = None, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, - is_causal: bool = False, - is_local: bool = False, - is_split_kv: bool = False, - pack_gqa: bool = False, - m_block_size: int = 128, - n_block_size: int = 128, - q_stage: cutlass.Constexpr[int] = 2, - is_persistent: bool = True, - score_mod: cutlass.Constexpr | None = None, - mask_mod: cutlass.Constexpr | None = None, - has_aux_tensors: cutlass.Constexpr = False, - paged_kv_non_tma: bool = False, - is_varlen_q: bool = False, - ): - self.use_tma_KV = not paged_kv_non_tma - # self.dtype = dtype - # padding head_dim to a multiple of 16 as k_block_size - hdim_multiple_of = 16 - self.head_dim_padded = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of) - head_dim_v = head_dim_v if head_dim_v is not None else head_dim - self.same_hdim_kv = head_dim == head_dim_v - self.head_dim_v_padded = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of) - self.same_hdim_kv_padded = self.head_dim_padded == self.head_dim_v_padded - self.check_hdim_oob = head_dim != self.head_dim_padded - self.check_hdim_v_oob = head_dim_v != self.head_dim_v_padded - self.m_block_size = m_block_size - self.n_block_size = n_block_size - self.q_stage = q_stage - assert self.q_stage in [1, 2] - - # 2 Q tile per CTA - self.cta_tiler = (self.q_stage * m_block_size, n_block_size, self.head_dim_padded) - self.mma_tiler_qk = (m_block_size, n_block_size, self.head_dim_padded) - self.mma_tiler_pv = (m_block_size, self.head_dim_v_padded, n_block_size) - self.qk_acc_dtype = Float32 - self.pv_acc_dtype = Float32 - self.cluster_shape_mn = (1, 1) - self.is_persistent = is_persistent - self.is_causal = is_causal - self.is_local = is_local - self.is_varlen_q = is_varlen_q - self.use_correction_warps_for_epi = is_varlen_q - self.qhead_per_kvhead = qhead_per_kvhead - self.is_split_kv = is_split_kv - self.pack_gqa = pack_gqa - if pack_gqa: - assert m_block_size % self.qhead_per_kvhead == 0, ( - "For PackGQA, m_block_size must be divisible by qhead_per_kvhead" - ) - assert not (self.is_split_kv and self.head_dim_v_padded >= 192), ( - "SplitKV is not supported for hdim >= 192" - ) - self.score_mod = score_mod - self.mask_mod = mask_mod - if cutlass.const_expr(has_aux_tensors): - self.vec_size: cutlass.Constexpr = 1 - else: - self.vec_size: cutlass.Constexpr = 2 - # Does S1 need to wait for S0 to finish - # self.s0_s1_barrier = self.head_dim_padded in [64, 96] and (not self.is_causal and not self.is_local) - self.s0_s1_barrier = False - self.overlap_sO_sQ = (self.head_dim_padded == 192 and self.head_dim_v_padded >= 64) or ( - self.head_dim_v_padded >= 128 and self.is_split_kv - ) - if self.overlap_sO_sQ: - self.is_persistent = False - - assert self.use_tma_KV or not (self.check_hdim_oob or self.check_hdim_v_oob), ( - "Paged KV does not support irregular head dim" - ) - - self.softmax0_warp_ids = (0, 1, 2, 3) - self.softmax1_warp_ids = (4, 5, 6, 7) - self.correction_warp_ids = (8, 9, 10, 11) - self.mma_warp_id = 12 - self.epilogue_warp_ids = (13,) - self.load_warp_ids = (14,) - self.empty_warp_ids = (15,) - SM100_TMEM_CAPACITY_COLUMNS = 512 - self.tmem_alloc_cols = SM100_TMEM_CAPACITY_COLUMNS - - self.threads_per_cta = cute.arch.WARP_SIZE * len( - ( - *self.softmax0_warp_ids, - *self.softmax1_warp_ids, - *self.correction_warp_ids, - self.mma_warp_id, - *self.load_warp_ids, - *self.epilogue_warp_ids, - *self.empty_warp_ids, - ) - ) - - if self.q_stage == 1: - if not self.use_tma_KV: - self.empty_warp_ids = self.empty_warp_ids + self.load_warp_ids - self.load_warp_ids = self.softmax1_warp_ids - else: - self.empty_warp_ids = self.empty_warp_ids + self.softmax1_warp_ids - self.softmax1_warp_ids = () - elif not self.use_tma_KV: - self.load_warp_ids = (14, 15) - self.empty_warp_ids = () - - if self.use_correction_warps_for_epi: - self.empty_warp_ids = self.empty_warp_ids + self.epilogue_warp_ids - self.epilogue_warp_ids = self.correction_warp_ids - elif self.is_varlen_q: # fallback - self.epilogue_warp_ids = (13, 14) - - self.tmem_s_offset = [0, self.n_block_size] # e.g., 0, 128 - self.tmem_o_offset = [ - self.tmem_s_offset[-1] + self.n_block_size + i * self.head_dim_v_padded - for i in range(self.q_stage) - ] # e.g., 256, 384 - self.tmem_total = self.tmem_o_offset[-1] + self.head_dim_v_padded - assert self.tmem_total <= SM100_TMEM_CAPACITY_COLUMNS - self.tmem_s_to_p_offset = self.n_block_size // 2 - self.tmem_p_offset = [ - self.tmem_s_offset[i] + self.tmem_s_to_p_offset for i in range(2) - ] # 0, 128 - - # vec buffer for row_max & row_sum - self.tmem_vec_offset = self.tmem_s_offset - - if self.head_dim_padded < 96: - self.num_regs_softmax = 200 - self.num_regs_correction = 64 - self.num_regs_other = 48 - else: - # self.num_regs_softmax = 192 if self.is_causal or self.is_local else 184 - self.num_regs_softmax = 200 - # self.num_regs_softmax = 176 - # self.num_regs_correction = 96 - # self.num_regs_correction = 80 - # self.num_regs_correction = 64 if self.is_causal or self.is_local else 80 - self.num_regs_correction = 64 - # self.num_regs_other = 32 - # self.num_regs_other = 64 - # self.num_regs_other = 80 - self.num_regs_other = 48 - # self.num_regs_other = 96 if self.is_causal or self.is_local else 80 - # self.num_regs_other = 64 if self.is_causal or self.is_local else 80 - self.num_regs_empty = 24 - - self.buffer_align_bytes = 1024 - - def _setup_attributes(self): - """Set up configurations and parameters for the FMHA kernel operation. - - This method initializes and configures various attributes required for the - execution of the fused multi-head attention kernel, mainly about the pipeline stages: - - - Sets up staging parameters for Q, K, V inputs and accumulator data - - Configures pipeline stages for softmax, correction, and epilogue operations - """ - - self.kv_stage = 4 if self.q_dtype.width == 8 or self.q_stage == 1 else 3 - self.acc_stage = 1 - # For hdim 192,128, we don't have enough smem to store all 3 stages of KV: - # 128 x 192 x 2 bytes x 3 stages = 144KB, and we need 96KB for Q. - # Instead we store smem as [smem_large, smem_small, smem_large], where smem_large is - # 128 x 192 and smem_small is 128 x 128. We set the stride between the stages to be - # 128 * 160, so that indexing the 0th and 2nd stages will get the right address, - # but for the 1st stage we need to add or subtract (depending on phase) 128 x 64. - self.uneven_kv_smem = ( - self.head_dim_padded == 192 and self.head_dim_v_padded == 128 and self.kv_stage == 3 - ) - self.uneven_kv_smem_offset = ( - self.m_block_size * (self.head_dim_padded - self.head_dim_v_padded) // 2 - if self.uneven_kv_smem - else 0 - ) - assert self.uneven_kv_smem_offset % 1024 == 0 - - @cute.jit - def __call__( - self, - mQ: cute.Tensor, # (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q - mK: cute.Tensor, # (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, h_k, d) if there is page_table - mV: cute.Tensor, # (b_k, s_k, h_k, dv) or (total_k, h_k, dv) if there is cu_seqlens_k or (num_pages, page_size, h_k, dv) if there is page_table - mO: cute.Tensor, # (b, s_q, h, dv) or (total_q, h, dv) if there is cu_seqlens_q - mLSE: Optional[cute.Tensor], - softmax_scale: Float32, - stream: cuda.CUstream, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - mPageTable: Optional[cute.Tensor] = None, # (b_k, max_num_pages_per_seq) - window_size_left: Int32 | int | None = None, - window_size_right: Int32 | int | None = None, - learnable_sink: Optional[cute.Tensor] = None, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - aux_tensors: Optional[list] = None, - ): - """Execute the Fused Multi-Head Attention operation on the provided tensors. - - This method prepares the input tensors for processing, validates their shapes and types, - configures the computation parameters, and launches the CUDA kernel. - - The method handles: - 1. Tensor layout transformations for specific memory access patterns - 2. Validation of tensor shapes and data types - 3. Initialization of hardware-specific parameters and memory layouts - 4. Configuration of TMA (Tensor Memory Access) operations - 5. Grid and work scheduling computation - 6. Kernel launch with appropriate parameters - """ - # setup static attributes before smem/grid/tma computation - self.q_dtype = mQ.element_type - self.k_dtype = mK.element_type - self.v_dtype = mV.element_type - self.o_dtype = mO.element_type - # Assume all strides are divisible by 128 bits except the last stride - new_stride = lambda t: ( - *(cute.assume(s, divby=128 // t.element_type.width) for s in t.stride[:-1]), - t.stride[-1], - ) - mQ, mK, mV, mO = [ - cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=new_stride(t))) - for t in (mQ, mK, mV, mO) - ] - Q_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1] - mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=Q_layout_transpose)) - # (s_k, d, h_k, b_k) or (total_k, d, h_k) if there's cu_seqlens_k or (page_size, d, h_k, num_pages) if there's page_table - KV_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensK is None) else [0, 2, 1] - mK, mV = [ - cute.make_tensor(t.iterator, cute.select(t.layout, mode=KV_layout_transpose)) - for t in (mK, mV) - ] - if const_expr(self.is_split_kv): - O_layout_transpose = ( - [2, 4, 3, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 3, 2, 0] - ) - LSE_layout_transpose = [3, 2, 1, 0] if const_expr(mCuSeqlensQ is None) else [2, 1, 0] - num_splits = mO.shape[0] - else: - O_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1] - LSE_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0] - num_splits = Int32(1) - mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=O_layout_transpose)) - mLSE = ( - cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=LSE_layout_transpose)) - if const_expr(mLSE is not None) - else None - ) - # (s, d, h, b) -> (d, s, h, b) - V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2] - mV = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose)) - - self.q_major_mode = cutlass.utils.LayoutEnum.from_tensor(mQ).mma_major_mode() - self.k_major_mode = cutlass.utils.LayoutEnum.from_tensor(mK).mma_major_mode() - self.v_major_mode = cutlass.utils.LayoutEnum.from_tensor(mV).mma_major_mode() - self.o_layout = cutlass.utils.LayoutEnum.from_tensor(mO) - - if const_expr(self.q_major_mode != tcgen05.OperandMajorMode.K): - raise RuntimeError("The layout of mQ is not supported") - if const_expr(self.k_major_mode != tcgen05.OperandMajorMode.K): - raise RuntimeError("The layout of mK is not supported") - if const_expr(self.v_major_mode != tcgen05.OperandMajorMode.MN): - raise RuntimeError("The layout of mV is not supported") - - # check type consistency - if const_expr(self.q_dtype != self.k_dtype): - raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}") - if const_expr(self.q_dtype != self.v_dtype): - raise TypeError(f"Type mismatch: {self.q_dtype} != {self.v_dtype}") - self._setup_attributes() - self.use_tma_O = self.arch >= 90 and mCuSeqlensQ is None and mSeqUsedQ is None - # This can be tuned - self.e2e_freq = 16 - if const_expr( - self.head_dim_padded > 64 and not self.is_causal and not self.is_local and self.pack_gqa - ): - self.e2e_freq = 32 if mCuSeqlensQ is not None or mSeqUsedQ is not None else 10 - - cta_group = tcgen05.CtaGroup.ONE - # the intermediate tensor p is from tmem & mK-major - p_source = tcgen05.OperandSource.TMEM - p_major_mode = tcgen05.OperandMajorMode.K - tiled_mma_qk = sm100_utils_basic.make_trivial_tiled_mma( - self.q_dtype, - self.q_major_mode, - self.k_major_mode, - self.qk_acc_dtype, - cta_group, - self.mma_tiler_qk[:2], - ) - tiled_mma_pv = sm100_utils_basic.make_trivial_tiled_mma( - self.v_dtype, - p_major_mode, - self.v_major_mode, - self.pv_acc_dtype, - cta_group, - self.mma_tiler_pv[:2], - p_source, - ) - - self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) - self.cluster_layout_vmnk = cute.tiled_divide( - cute.make_layout(self.cluster_shape_mnk), - (tiled_mma_qk.thr_id.shape,), - ) - - self.epi_tile = self.mma_tiler_pv[:2] - - sQ_layout = sm100_utils_basic.make_smem_layout_a( - tiled_mma_qk, - self.mma_tiler_qk, - self.q_dtype, - self.q_stage, - ) - sK_layout = sm100_utils_basic.make_smem_layout_b( - tiled_mma_qk, - self.mma_tiler_qk, - self.k_dtype, - self.kv_stage, - ) - tP_layout = sm100_utils_basic.make_smem_layout_a( - tiled_mma_pv, - self.mma_tiler_pv, - self.q_dtype, - self.acc_stage, - ) - sV_layout = sm100_utils_basic.make_smem_layout_b( - tiled_mma_pv, - self.mma_tiler_pv, - self.v_dtype, - self.kv_stage, - ) - sO_layout = sm100_utils_basic.make_smem_layout_epi( - self.o_dtype, - self.o_layout, - self.epi_tile, - self.q_stage, - ) - if const_expr(not self.same_hdim_kv_padded): - # sK and sV are using the same physical smem so we need to adjust the stride so that they line up - stride_sK = const_expr( - max(sK_layout.outer.stride[-1], 0) - ) # take max to turn tuple to Int32 - stride_sV = const_expr(max(sV_layout.outer.stride[-1], 0)) - stage_stride = const_expr( - max(stride_sK, stride_sV) - if not self.uneven_kv_smem - else (stride_sK + stride_sV) // 2 - ) - sK_layout = cute.make_composed_layout( - sK_layout.inner, - 0, - cute.make_layout( - (*sK_layout.outer.shape[:-1], self.kv_stage), - stride=(*sK_layout.outer.stride[:-1], stage_stride), - ), - ) - sV_layout = cute.make_composed_layout( - sV_layout.inner, - 0, - cute.make_layout( - (*sV_layout.outer.shape[:-1], self.kv_stage), - stride=(*sV_layout.outer.stride[:-1], stage_stride), - ), - ) - - if const_expr(self.pack_gqa): - shape_Q_packed = ( - (self.qhead_per_kvhead, mQ.shape[0]), - mQ.shape[1], - mK.shape[2], - *mQ.shape[3:], - ) - stride_Q_packed = ( - (mQ.stride[2], mQ.stride[0]), - mQ.stride[1], - mQ.stride[2] * self.qhead_per_kvhead, - *mQ.stride[3:], - ) - mQ = cute.make_tensor( - mQ.iterator, cute.make_layout(shape_Q_packed, stride=stride_Q_packed) - ) - shape_O_packed = ( - (self.qhead_per_kvhead, mO.shape[0]), - mO.shape[1], - mK.shape[2], - *mO.shape[3:], - ) - stride_O_packed = ( - (mO.stride[2], mO.stride[0]), - mO.stride[1], - mO.stride[2] * self.qhead_per_kvhead, - *mO.stride[3:], - ) - mO = cute.make_tensor( - mO.iterator, cute.make_layout(shape_O_packed, stride=stride_O_packed) - ) - if const_expr(mLSE is not None): - shape_LSE_packed = ( - (self.qhead_per_kvhead, mLSE.shape[0]), - mK.shape[2], - *mLSE.shape[2:], - ) - stride_LSE_packed = ( - (mLSE.stride[1], mLSE.stride[0]), - mLSE.stride[1] * self.qhead_per_kvhead, - *mLSE.stride[2:], - ) - mLSE = cute.make_tensor( - mLSE.iterator, cute.make_layout(shape_LSE_packed, stride=stride_LSE_packed) - ) - - self.tma_copy_bytes = { - name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1, 2])) - for name, mX, layout in [ - ("Q", mQ, sQ_layout), - ("K", mK, sK_layout), - ("V", mV, sV_layout), - ] - } - - # TMA load for Q - tma_load_op = cpasync.CopyBulkTensorTileG2SOp(cta_group) - tma_store_op = cpasync.CopyBulkTensorTileS2GOp() - - tma_atom_Q, mQ = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - mQ, - cute.select(sQ_layout, mode=[0, 1, 2]), - self.mma_tiler_qk, - tiled_mma_qk, - self.cluster_layout_vmnk.shape, - ) - - if const_expr(self.use_tma_KV): - # TMA load for K - tma_atom_K, mK = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - mK, - cute.select(sK_layout, mode=[0, 1, 2]), - self.mma_tiler_qk, - tiled_mma_qk, - self.cluster_layout_vmnk.shape, - ) - # TMA load for V - tma_atom_V, mV = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - mV, - cute.select(sV_layout, mode=[0, 1, 2]), - self.mma_tiler_pv, - tiled_mma_pv, - self.cluster_layout_vmnk.shape, - ) - else: - tma_atom_K = None - tma_atom_V = None - - o_cta_v_layout = cute.composition(cute.make_identity_layout(mO.shape), self.epi_tile) - - self.num_epilogue_threads = cute.arch.WARP_SIZE * len(self.epilogue_warp_ids) - if const_expr(self.use_tma_O): - tma_atom_O, mO = cpasync.make_tiled_tma_atom( - tma_store_op, - mO, - cute.select(sO_layout, mode=[0, 1]), - o_cta_v_layout, - ) - gmem_tiled_copy_O = None - else: - tma_atom_O = None - universal_copy_bits = 128 - async_copy_elems = universal_copy_bits // self.o_dtype.width - atom_universal_copy = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.o_dtype, - num_bits_per_copy=universal_copy_bits, - ) - tO_shape_dim_1 = sO_layout.outer.shape[1][0] // async_copy_elems - tO_layout = cute.make_ordered_layout( - (self.num_epilogue_threads // tO_shape_dim_1, tO_shape_dim_1), - order=(1, 0), - ) - # So that we don't have to check if we overshoot kBlockM when we store O - assert self.m_block_size % tO_layout.shape[0] == 0 - vO_layout = cute.make_layout((1, async_copy_elems)) - gmem_tiled_copy_O = cute.make_tiled_copy_tv(atom_universal_copy, tO_layout, vO_layout) - - if const_expr(mCuSeqlensQ is not None or mSeqUsedQ is not None): - TileScheduler = SingleTileVarlenScheduler - else: - if const_expr(self.is_causal or self.is_local): - TileScheduler = SingleTileLPTScheduler - else: - TileScheduler = ( - SingleTileScheduler - if const_expr(not self.is_persistent) - else StaticPersistentTileScheduler - ) - tile_sched_args = TileSchedulerArguments( - cute.ceil_div(cute.size(mQ.shape[0]), self.cta_tiler[0]), - cute.size(mQ.shape[2]), - cute.size(mQ.shape[3]) - if const_expr(mCuSeqlensQ is None) - else cute.size(mCuSeqlensQ.shape[0] - 1), - num_splits, - cute.size(mK.shape[0]) - if const_expr(mPageTable is None) - else mK.shape[0] * mPageTable.shape[1], - mQ.shape[1], - mV.shape[0], # Note that this is different from Sm90 since we transpose mV in Sm100 - total_q=cute.size(mQ.shape[0]) - if const_expr(mCuSeqlensQ is not None) - else cute.size(mQ.shape[0]) * cute.size(mQ.shape[3]), - tile_shape_mn=self.cta_tiler[:2], - mCuSeqlensQ=mCuSeqlensQ, - mSeqUsedQ=mSeqUsedQ, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - element_size=self.k_dtype.width // 8, - is_persistent=self.is_persistent, - lpt=self.is_causal or self.is_local, - is_split_kv=self.is_split_kv, - ) - tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args) - self.tile_scheduler_cls = TileScheduler - grid_dim = TileScheduler.get_grid_shape(tile_sched_params) - - self.mbar_load_q_full_offset = 0 - self.mbar_load_q_empty_offset = self.mbar_load_q_full_offset + self.q_stage - self.mbar_load_kv_full_offset = self.mbar_load_q_empty_offset + self.q_stage - self.mbar_load_kv_empty_offset = self.mbar_load_kv_full_offset + self.kv_stage - self.mbar_P_full_O_rescaled_offset = self.mbar_load_kv_empty_offset + self.kv_stage - self.mbar_S_full_offset = self.mbar_P_full_O_rescaled_offset + self.q_stage - self.mbar_O_full_offset = self.mbar_S_full_offset + self.q_stage - self.mbar_softmax_corr_full_offset = self.mbar_O_full_offset + self.q_stage - self.mbar_softmax_corr_empty_offset = self.mbar_softmax_corr_full_offset + self.q_stage - self.mbar_corr_epi_full_offset = self.mbar_softmax_corr_empty_offset + self.q_stage - self.mbar_corr_epi_empty_offset = self.mbar_corr_epi_full_offset + self.q_stage - self.mbar_s0_s1_sequence_offset = self.mbar_corr_epi_empty_offset + self.q_stage - self.mbar_tmem_dealloc_offset = self.mbar_s0_s1_sequence_offset + 8 - self.mbar_P_full_2_offset = self.mbar_tmem_dealloc_offset + 1 - self.mbar_total = self.mbar_P_full_2_offset + self.q_stage - - sO_size = cute.cosize(sO_layout) if const_expr(not self.overlap_sO_sQ) else 0 - sQ_size = ( - cute.cosize(sQ_layout) - if const_expr(not self.overlap_sO_sQ) - else cutlass.max( - cute.cosize(sQ_layout), - cute.cosize(sO_layout) * self.o_dtype.width // self.q_dtype.width, - ) - ) - - @cute.struct - class SharedStorage: - # m_barriers for pipelines - mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.mbar_total] - # Tmem holding buffer - tmem_holding_buf: Int32 - # Smem tensors - # store row max and row sum - sScale: cute.struct.MemRange[Float32, self.q_stage * self.m_block_size * 2] - sO: cute.struct.Align[ - cute.struct.MemRange[self.o_dtype, sO_size], - self.buffer_align_bytes, - ] - sQ: cute.struct.Align[ - cute.struct.MemRange[self.q_dtype, sQ_size], - self.buffer_align_bytes, - ] - sK: cute.struct.Align[ - # cute.cosize(sK_layout) is correct even in the case of self.uneven_kv_smem - cute.struct.MemRange[self.k_dtype, cute.cosize(sK_layout)], - self.buffer_align_bytes, - ] - - self.shared_storage = SharedStorage - - LOG2_E = math.log2(math.e) - if const_expr(self.score_mod is None): - softmax_scale_log2 = softmax_scale * LOG2_E - softmax_scale = None - else: - # NB: If a users passes in a score mod, we want to apply the score-mod in the sm_scaled qk - # But in the original base 10. We hijack softmax_scale_log2 to just be the change of base - # and correctly apply the softmax_scale prior to score_mod in the softmax step - softmax_scale_log2 = LOG2_E - softmax_scale = softmax_scale - - if const_expr(window_size_left is not None): - window_size_left = Int32(window_size_left) - if const_expr(window_size_right is not None): - window_size_right = Int32(window_size_right) - - fastdiv_mods = None - if cutlass.const_expr(aux_tensors is not None): - seqlen_q = cute.size(mQ.shape[0]) // ( - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1 - ) - seqlen_k = ( - cute.size(mK.shape[0]) - if const_expr(mPageTable is None) - else mK.shape[0] * mPageTable.shape[1] - ) - seqlen_q_divmod = FastDivmodDivisor(seqlen_q) - seqlen_k_divmod = FastDivmodDivisor(seqlen_k) - fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod) - - self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None) - if cutlass.const_expr(self.use_block_sparsity and mPageTable is not None): - raise NotImplementedError("Block sparsity + paged KV not supported on SM100") - - # Launch the kernel synchronously - self.kernel( - mQ, - mK, - mV, - mO, - mLSE, - mCuSeqlensQ, - mCuSeqlensK, - mSeqUsedQ, - mSeqUsedK, - mPageTable, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - tma_atom_O, - softmax_scale_log2, - softmax_scale, - window_size_left, - window_size_right, - learnable_sink, - blocksparse_tensors, - sQ_layout, - sK_layout, - tP_layout, - sV_layout, - sO_layout, - gmem_tiled_copy_O, - tiled_mma_qk, - tiled_mma_pv, - tile_sched_params, - num_splits, - aux_tensors, - fastdiv_mods, - ).launch( - grid=grid_dim, - block=[self.threads_per_cta, 1, 1], - cluster=self.cluster_shape_mnk, - smem=self.shared_storage.size_in_bytes(), - stream=stream, - min_blocks_per_mp=1, - ) - - # GPU device kernel - @cute.kernel - def kernel( - self, - mQ: cute.Tensor, # (s_q, d, h, b) or (total_q, d, h) if there is cu_seqlens_q - mK: cute.Tensor, # (s_k, d, h_k, b_k) or (total_k, d, h_k) if there is cu_seqlens_k or (page_size, d, h_k, num_pages) if there is page_table - mV: cute.Tensor, # (d, s_k, h_k, b_k) or (d, total_k, h_k) if there is cu_seqlens_k or (d, page_size, h_k, num_pages) if there is page_table - mO: cute.Tensor, - mLSE: Optional[cute.Tensor], - mCuSeqlensQ: Optional[cute.Tensor], - mCuSeqlensK: Optional[cute.Tensor], - mSeqUsedQ: Optional[cute.Tensor], - mSeqUsedK: Optional[cute.Tensor], - mPageTable: Optional[cute.Tensor], - tma_atom_Q: cute.CopyAtom, - tma_atom_K: Optional[cute.CopyAtom], - tma_atom_V: Optional[cute.CopyAtom], - tma_atom_O: Optional[cute.CopyAtom], - softmax_scale_log2: Float32, - softmax_scale: Float32 | None, - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - learnable_sink: Optional[cute.Tensor], - blocksparse_tensors: Optional[BlockSparseTensors], - sQ_layout: cute.ComposedLayout, - sK_layout: cute.ComposedLayout, - tP_layout: cute.ComposedLayout, - sV_layout: cute.ComposedLayout, - sO_layout: cute.ComposedLayout, - gmem_tiled_copy_O: Optional[cute.TiledCopy], - tiled_mma_qk: cute.TiledMma, - tiled_mma_pv: cute.TiledMma, - tile_sched_params: ParamsBase, - num_splits: Int32, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - ): - """The device kernel implementation of the Fused Multi-Head Attention. - - This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation: - 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA - 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V) - 3. Softmax warps: Compute softmax normalization on attention scores - 4. Correction warps: Apply adjustments to intermediate results - 5. Epilogue warp: Handles final output transformation and storage - - The kernel implements a complex pipeline with overlapping computation and memory operations, - using tensor memory access (TMA) for efficient data loading, warp specialization for different - computation phases, and optional attention masking. - """ - - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - - # Prefetch tma descriptor - if warp_idx == 0: - cpasync.prefetch_descriptor(tma_atom_Q) - if const_expr(tma_atom_K is not None): - cpasync.prefetch_descriptor(tma_atom_K) - if const_expr(tma_atom_V is not None): - cpasync.prefetch_descriptor(tma_atom_V) - if const_expr(tma_atom_O is not None): - cpasync.prefetch_descriptor(tma_atom_O) - - # Alloc - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(self.shared_storage) - - mbar_ptr = storage.mbar_ptr.data_ptr() - # Use the first N warps to initialize barriers - if warp_idx == 1: - # Init "full" barrier with number of producers, "empty" barrier with number of consumers - for i in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_init(mbar_ptr + self.mbar_load_q_full_offset + i, 1) - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_load_q_empty_offset + i, len([self.mma_warp_id]) - ) - if warp_idx == 2: - for i in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_softmax_corr_empty_offset + i, cute.arch.WARP_SIZE * 4 - ) - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_softmax_corr_full_offset + i, cute.arch.WARP_SIZE * 4 - ) - if warp_idx == 3: - if const_expr(self.s0_s1_barrier): - for i in cutlass.range_constexpr(8): - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_s0_s1_sequence_offset + i, cute.arch.WARP_SIZE - ) - if const_expr(not self.use_correction_warps_for_epi) and warp_idx == 4: - for i in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_corr_epi_full_offset + i, - cute.arch.WARP_SIZE * len(self.correction_warp_ids), - ) - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_corr_epi_empty_offset + i, - cute.arch.WARP_SIZE * len(self.epilogue_warp_ids), - ) - if warp_idx == 5: - for i in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_P_full_O_rescaled_offset + i, - cute.arch.WARP_SIZE - * (len(self.softmax0_warp_ids) + len(self.correction_warp_ids)), - ) - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_S_full_offset + i, len([self.mma_warp_id]) - ) - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_O_full_offset + i, len([self.mma_warp_id]) - ) - if warp_idx == 6: - for i in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_P_full_2_offset + i, - cute.arch.WARP_SIZE * len(self.softmax0_warp_ids), - ) - if warp_idx == 7: - cute.arch.mbarrier_init( - mbar_ptr + self.mbar_tmem_dealloc_offset, - cute.arch.WARP_SIZE - * len( - ( - *self.softmax0_warp_ids, - *self.softmax1_warp_ids, - *self.correction_warp_ids, - ) - ), - ) - # Relying on pipeline_kv constructor to call mbarrier_init_fence and sync - pipeline_kv = self.make_and_init_load_kv_pipeline(mbar_ptr + self.mbar_load_kv_full_offset) - - # Generate smem tensor Q/K/V/O - # (MMA, MMA_Q, MMA_D, PIPE) - sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner) - # (MMA, MMA_K, MMA_D, PIPE) - sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner) - # (MMA, MMA_K, MMA_D, PIPE) - # Strip swizzle info to reuse smem - sV = cute.make_tensor(cute.recast_ptr(sK.iterator, sV_layout.inner), sV_layout.outer) - if const_expr(not self.overlap_sO_sQ): - sO = storage.sO.get_tensor(sO_layout.outer, swizzle=sO_layout.inner) - else: - sO = cute.make_tensor( - cute.recast_ptr(sQ.iterator, sO_layout.inner, self.o_dtype), sO_layout.outer - ) - - sScale = storage.sScale.get_tensor(cute.make_layout(self.q_stage * self.m_block_size * 2)) - - thr_mma_qk = tiled_mma_qk.get_slice(0) # default 1SM - thr_mma_pv = tiled_mma_pv.get_slice(0) # default 1SM - - qk_acc_shape = thr_mma_qk.partition_shape_C(self.mma_tiler_qk[:2]) - tStS_fake = thr_mma_qk.make_fragment_C(qk_acc_shape) - # This is a fake tensor, by right need to retrieve tmem_ptr. But we know that we always - # request 512 columns of tmem, so we know that it starts at 0. - tmem_ptr = cute.make_ptr(Float32, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16) - tStS = cute.make_tensor(tmem_ptr, tStS_fake.layout) - - pv_acc_shape = thr_mma_pv.partition_shape_C(self.mma_tiler_pv[:2]) - tOtO = thr_mma_pv.make_fragment_C(pv_acc_shape) - - tStSs = tuple( - cute.make_tensor(tStS.iterator + self.tmem_s_offset[stage], tStS.layout) - for stage in range(self.q_stage) - ) - tOtOs = tuple( - cute.make_tensor(tOtO.iterator + self.tmem_o_offset[stage], tOtO.layout) - for stage in range(self.q_stage) - ) - - tP = cute.make_tensor(tStS.iterator, tP_layout.outer) - tOrP = thr_mma_pv.make_fragment_A(tP)[None, None, None, 0] - - tOrPs = [ - cute.make_tensor( - tOrP.iterator - + self.qk_acc_dtype.width // self.q_dtype.width * self.tmem_p_offset[stage], - tOrP.layout, - ) - for stage in range(self.q_stage) - ] - - block_info = BlockInfo( - # This is cta_tiler, not mma_tiler_qk, since we move by block by (2 * mma_tiler[0], mma_tiler[1]) - self.cta_tiler[0], - self.cta_tiler[1], - self.is_causal, - self.is_local, - self.is_split_kv, - window_size_left, - window_size_right, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - SeqlenInfoCls = partial( - SeqlenInfoQK.create, - seqlen_q_static=mQ.shape[0] if const_expr(not self.pack_gqa) else mQ.shape[0][1], - seqlen_k_static=mK.shape[0] - if const_expr(mPageTable is None) - else mK.shape[0] * mPageTable.shape[1], - mCuSeqlensQ=mCuSeqlensQ, - mCuSeqlensK=mCuSeqlensK, - mSeqUsedQ=mSeqUsedQ, - mSeqUsedK=mSeqUsedK, - ) - AttentionMaskCls = partial( - AttentionMask, - self.m_block_size, - self.n_block_size, - window_size_left=window_size_left, - window_size_right=window_size_right, - qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - TileSchedulerCls = partial(self.tile_scheduler_cls.create, tile_sched_params) - - # /////////////////////////////////////////////////////////////////////////////// - # EMPTY - # /////////////////////////////////////////////////////////////////////////////// - for i in cutlass.range_constexpr(len(self.empty_warp_ids)): - if warp_idx == self.empty_warp_ids[i]: - cute.arch.warpgroup_reg_dealloc(self.num_regs_empty) - - # /////////////////////////////////////////////////////////////////////////////// - # LOAD - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx >= self.load_warp_ids[0] and warp_idx <= self.load_warp_ids[-1]: - cute.arch.warpgroup_reg_dealloc(self.num_regs_other) - self.load( - thr_mma_qk, - thr_mma_pv, - mQ, - mK, - mV, - sQ, - sK, - sV, - mPageTable, - tma_atom_Q, - tma_atom_K, - tma_atom_V, - pipeline_kv, - mbar_ptr, - block_info, - num_splits, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - ) - - # /////////////////////////////////////////////////////////////////////////////// - # MMA - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.mma_warp_id: - # if warp_idx == self.mma_warp_id or warp_idx == self.empty_warp_ids: - cute.arch.warpgroup_reg_dealloc(self.num_regs_other) - # Alloc tmem buffer - tmem_alloc_cols = Int32(self.tmem_alloc_cols) - if warp_idx == self.mma_warp_id: - cute.arch.alloc_tmem(tmem_alloc_cols, storage.tmem_holding_buf) - cute.arch.sync_warp() - - self.mma( - tiled_mma_qk, - tiled_mma_pv, - sQ, - sK, - sV, - tStSs, - tOtOs, - tOrPs, - pipeline_kv, - mbar_ptr, - block_info, - num_splits, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - ) - - # if warp_idx == self.mma_warp_id: - # dealloc tmem buffer - cute.arch.relinquish_tmem_alloc_permit() - cute.arch.mbarrier_wait(mbar_ptr + self.mbar_tmem_dealloc_offset, 0) - tmem_alloc_cols = Int32(self.tmem_alloc_cols) - # Retrieving tmem ptr and make acc - tmem_ptr = cute.arch.retrieve_tmem_ptr( - Float32, - alignment=16, - ptr_to_buffer_holding_addr=storage.tmem_holding_buf, - ) - cute.arch.dealloc_tmem(tmem_ptr, tmem_alloc_cols) - - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue - # /////////////////////////////////////////////////////////////////////////////// - if const_expr(not self.use_correction_warps_for_epi): - if warp_idx >= self.epilogue_warp_ids[0] and warp_idx <= self.epilogue_warp_ids[-1]: - cute.arch.warpgroup_reg_dealloc(self.num_regs_other) - self.epilogue_s2g( - mO, - sO, - gmem_tiled_copy_O, - tma_atom_O, - mbar_ptr, - block_info, - num_splits, - SeqlenInfoCls, - TileSchedulerCls, - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Softmax - # /////////////////////////////////////////////////////////////////////////////// - if (const_expr(self.q_stage == 2) and warp_idx <= self.softmax1_warp_ids[-1]) or ( - const_expr(self.q_stage == 1) and warp_idx <= self.softmax0_warp_ids[-1] - ): - # increase register after decreasing - cute.arch.warpgroup_reg_alloc(self.num_regs_softmax) - softmax_loop = partial( - self.softmax_loop, - softmax_scale_log2=softmax_scale_log2, - softmax_scale=softmax_scale, - thr_mma_qk=thr_mma_qk, - sScale=sScale, - mLSE=mLSE, - learnable_sink=learnable_sink, - mbar_ptr=mbar_ptr, - block_info=block_info, - num_splits=num_splits, - SeqlenInfoCls=SeqlenInfoCls, - AttentionMaskCls=AttentionMaskCls, - TileSchedulerCls=TileSchedulerCls, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - blocksparse_tensors=blocksparse_tensors, - ) - - if const_expr(not self.s0_s1_barrier): - stage = Int32( - 0 - if const_expr(self.q_stage == 1) or warp_idx < self.softmax1_warp_ids[0] - else 1 - ) - softmax_loop( - stage=stage, - tStSi=cute.make_tensor( - tStS.iterator - + (self.tmem_s_offset[0] if stage == 0 else self.tmem_s_offset[1]), - tStS.layout, - ), - ) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset) - else: - # If there's s0_s1_barrier, it's faster to have 2 WGs having different code - if warp_idx < self.softmax1_warp_ids[0]: - tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[0], tStS.layout) - softmax_loop(stage=0, tStSi=tStSi) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset) - if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]: - tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[1], tStS.layout) - softmax_loop(stage=1, tStSi=tStSi) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset) - - # /////////////////////////////////////////////////////////////////////////////// - # Correction - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id: - cute.arch.warpgroup_reg_dealloc(self.num_regs_correction) - self.correction_loop( - thr_mma_qk, - thr_mma_pv, - tStS, - tOtOs, - sScale, - mO, - mLSE, - sO, - learnable_sink, - gmem_tiled_copy_O, - tma_atom_O, - mbar_ptr, - softmax_scale_log2, - block_info, - num_splits, - SeqlenInfoCls, - TileSchedulerCls, - blocksparse_tensors, - ) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset) - - return - - @cute.jit - def load( - self, - thr_mma_qk: cute.core.ThrMma, - thr_mma_pv: cute.core.ThrMma, - mQ: cute.Tensor, - mK: cute.Tensor, - mV: cute.Tensor, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - mPageTable: Optional[cute.Tensor], - tma_atom_Q: cute.CopyAtom, - tma_atom_K: Optional[cute.CopyAtom], - tma_atom_V: Optional[cute.CopyAtom], - pipeline_kv: cutlass.pipeline.PipelineAsync, - mbar_ptr: cute.Pointer, - block_info: BlockInfo, - num_splits: Int32, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors], - ): - num_load_threads = len(self.load_warp_ids) * cute.arch.WARP_SIZE - tidx = cute.arch.thread_idx()[0] % num_load_threads - q_producer_phase = Int32(1) - kv_producer_state = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Producer, self.kv_stage - ) - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - mQ_cur = seqlen.offset_batch_Q(mQ, batch_idx, dim=3)[None, None, head_idx] - gQ = cute.local_tile(mQ_cur, cute.select(self.mma_tiler_qk, mode=[0, 2]), (None, 0)) - - head_idx_kv = ( - head_idx // self.qhead_per_kvhead if const_expr(not self.pack_gqa) else head_idx - ) - if const_expr(mPageTable is None): - if const_expr(not seqlen.has_cu_seqlens_k): - mK_cur, mV_cur = [t[None, None, head_idx_kv, batch_idx] for t in (mK, mV)] - else: - mK_cur = cute.domain_offset((seqlen.offset_k, 0), mK[None, None, head_idx_kv]) - mV_cur = cute.domain_offset((0, seqlen.offset_k), mV[None, None, head_idx_kv]) - gK = cute.local_tile(mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0)) - gV = cute.local_tile(mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None)) - else: - # Need to keep batch coord None since we'll index into it with page idx - mK_cur, mV_cur = [t[None, None, head_idx_kv, None] for t in (mK, mV)] - gK = cute.local_tile( - mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0, None) - ) - gV = cute.local_tile( - mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None, None) - ) - tSgQ = thr_mma_qk.partition_A(gQ) - tSgK = thr_mma_qk.partition_B(gK) - tOgV = thr_mma_pv.partition_B(gV) - load_Q_fn, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_Q, 0, cute.make_layout(1), tSgQ, sQ - ) - - if const_expr(self.use_tma_KV): - tKsK, tKgK = cpasync.tma_partition( - tma_atom_K, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sK, 0, 3), - cute.group_modes(tSgK, 0, 3), - ) - tVsV, tVgV = cpasync.tma_partition( - tma_atom_V, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sV, 0, 3), - cute.group_modes(tOgV, 0, 3), - ) - paged_kv_manager = None - else: - page_size = mK.shape[0] - paged_kv_manager = PagedKVManager.create( - mPageTable, - mK, - mV, - FastDivmodDivisor(page_size), - batch_idx, - head_idx_kv, - tidx, - seqlen.seqlen_k, - 0, # leftpad_k - self.n_block_size, - self.head_dim_padded, - self.head_dim_v_padded, - num_load_threads, - mK.element_type, - ) - tKsK, tKgK = None, None - tVsV, tVgV = None, None - - load_Q = partial( - self.load_Q, - load_Q_fn, - mbar_ptr + self.mbar_load_q_full_offset, - mbar_ptr + self.mbar_load_q_empty_offset, - phase=q_producer_phase, - ) - # We have to use mbarrier directly in the load for KV instead of replying on - # pipeline_kv, because we could have different number of TMA bytes for K and V - load_K = partial( - self.load_KV, - tma_atom_K, - tKgK, - tKsK, - paged_kv_manager, - sK, - mbar_ptr + self.mbar_load_kv_full_offset, - mbar_ptr + self.mbar_load_kv_empty_offset, - K_or_V="K", - ) - load_V = partial( - self.load_KV, - tma_atom_V, - tVgV, - tVsV, - paged_kv_manager, - sV, - mbar_ptr + self.mbar_load_kv_full_offset, - mbar_ptr + self.mbar_load_kv_empty_offset, - K_or_V="V", - ) - - if const_expr(not self.use_block_sparsity): - n_block_min, n_block_max = block_info.get_n_block_min_max( - seqlen, m_block, split_idx, num_splits - ) - if const_expr(not self.is_split_kv) or n_block_min < n_block_max: - if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE: - load_Q(block=self.q_stage * m_block + 0, stage=0) # Q0 - n_block_first = n_block_max - 1 if n_block_max > 0 else 0 - page_idx = ( - mPageTable[batch_idx, n_block_first] - if const_expr(mPageTable is not None and self.use_tma_KV) - else None - ) - if const_expr(not self.use_tma_KV): - paged_kv_manager.load_page_table(n_block_first) - load_K( - block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx - ) # K0 - kv_producer_state.advance() - if const_expr(self.q_stage == 2) and ( - const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE - ): - load_Q(block=self.q_stage * m_block + 1, stage=1) # Q1 - q_producer_phase ^= 1 - load_V( - block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx - ) # V0 - kv_producer_state.advance() - for i in cutlass.range(n_block_max - 1 - n_block_min, unroll=1): - n_block = n_block_max - 2 - i - page_idx = ( - mPageTable[batch_idx, n_block] - if const_expr(mPageTable is not None and self.use_tma_KV) - else None - ) - if const_expr(not self.use_tma_KV): - paged_kv_manager.load_page_table(n_block) - # if cute.arch.thread_idx()[0] % 32 == 0: cute.printf("n_block = {}, page_idx = {}", n_block, page_idx) - load_K( - block=n_block, producer_state=kv_producer_state, page_idx=page_idx - ) # Ki - kv_producer_state.advance() - load_V( - block=n_block, producer_state=kv_producer_state, page_idx=page_idx - ) # Vi - kv_producer_state.advance() - - else: - kv_producer_state, q_producer_phase = produce_block_sparse_loads_sm100( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - kv_producer_state, - load_Q, - load_K, - load_V, - pipeline_kv, - self.q_stage, - q_producer_phase, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - - tile_scheduler.prefetch_next_work() - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def mma( - self, - tiled_mma_qk: cute.core.ThrMma, - tiled_mma_pv: cute.core.ThrMma, - sQ: cute.Tensor, - sK: cute.Tensor, - sV: cute.Tensor, - tStSs: Tuple[cute.Tensor, cute.Tensor], - tOtOs: tuple[cute.Tensor], - tOrPs: Tuple[cute.Tensor, cute.Tensor], - pipeline_kv: cutlass.pipeline.PipelineAsync, - mbar_ptr: cute.Pointer, - block_info: BlockInfo, - num_splits: Int32, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors], - ): - tSrQ = tiled_mma_qk.make_fragment_A(sQ) - tSrK = tiled_mma_qk.make_fragment_B(sK) - tOrV = tiled_mma_pv.make_fragment_B(sV) - if const_expr(self.q_stage == 2): - tSrQs = (tSrQ[None, None, None, 0], tSrQ[None, None, None, 1]) - else: - tSrQs = (tSrQ[None, None, None, 0],) - - qk_mma_op, pv_mma_op = tiled_mma_qk.op, tiled_mma_pv.op - - gemm_Si = [ - partial( - sm100_utils.gemm_ptx_partial, - qk_mma_op, - self.tmem_s_offset[stage], - tSrQs[stage], - sA=sQ[None, None, None, stage], - zero_init=True, - ) - for stage in range(self.q_stage) - ] - gemm_Pi = [ - partial( - sm100_utils.gemm_ptx_partial, - pv_mma_op, - self.tmem_o_offset[stage], - tOrPs[stage], - sA=None, - ) - for stage in range(self.q_stage) - ] - - mma_q_consumer_phase = Int32(0) - mma_kv_consumer_state = cutlass.pipeline.make_pipeline_state( - cutlass.pipeline.PipelineUserType.Consumer, self.kv_stage - ) - P_full_O_rescaled_phase = Int32(0) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - - block_iter_count = Int32(0) - process_tile = False - - if const_expr(self.use_block_sparsity): - block_iter_count = get_total_block_count( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - process_tile = block_iter_count > Int32(0) - else: - n_block_min, n_block_max = block_info.get_n_block_min_max( - seqlen, m_block, split_idx, num_splits - ) - block_iter_count = n_block_max - n_block_min - if const_expr(not self.is_split_kv): - process_tile = True - else: - process_tile = n_block_min < n_block_max - - if process_tile: - for stage in cutlass.range_constexpr(self.q_stage): - # GEMM_QK00 (Q0 * K0 -> S0) or GEMM_QK01 (Q1 * K0 -> S1) - # 1. wait for Q0 / Q1 - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_load_q_full_offset + stage, mma_q_consumer_phase - ) - # 2. wait for K0 - if const_expr(stage == 0): - pipeline_kv.consumer_wait(mma_kv_consumer_state) - tSrKi = tSrK[None, None, None, mma_kv_consumer_state.index] - # We don't need to acquire empty S0 / S1. - # For the first iteration, we don't need to wait as we're guaranteed S0 / S1 - # are empty. For subsequent iterations, the wait happened at the end - # of the while loop. - # 3. gemm - # tiled_mma_qk = sm100_utils.gemm(tiled_mma_qk, tStSs[stage], tSrQs[stage], tSrKi, zero_init=True) - sK_cur = sK[None, None, None, mma_kv_consumer_state.index] - if const_expr(self.uneven_kv_smem): - sK_cur = self.offset_kv_smem( - sK_cur, mma_kv_consumer_state.index, mma_kv_consumer_state.phase - ) - gemm_Si[stage](tCrB=tSrKi, sB=sK_cur) - # 4. release S0 / S1 - with cute.arch.elect_one(): - tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage) - mma_q_consumer_phase ^= 1 - # 5. release K0 - pipeline_kv.consumer_release(mma_kv_consumer_state) - mma_kv_consumer_state.advance() - # End of GEMM (Q1 * K0 -> S1) - # Note: Q0 & Q1 are still needed in the seqlen_kv loop - # so we need to release them after the seqlen_kv loop - - # O hasn't been accumulated yet, its first MMA calculation doesn't need to accumulate - block_loop_count = block_iter_count - 1 - O_should_accumulate = False - for i in cutlass.range(block_loop_count, unroll=1): - # GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop - # 1. wait for V0 - pipeline_kv.consumer_wait(mma_kv_consumer_state) - mma_kv_release_state = mma_kv_consumer_state.clone() - Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase - tOrVi = tOrV[None, None, None, Vi_index] - for stage in cutlass.range_constexpr(self.q_stage): - # 2. acquire corrected O0/O1_partial and P0 / P1 - # For the first iteration in this work tile, waiting for O0/O1_partial - # means that the correction warps has finished reading tO during - # the last iteration of the previous work tile has finished. - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage, - P_full_O_rescaled_phase, - ) - # 3. gemm - # sm100_utils.gemm(tiled_mma_pv, tOtO0, tOrP0, tOrVi, zero_init=True) - # gemm_Pi[stage](tCrB=tOrVi, sB=sV[None, None, None, Vi_index], zero_init=not O_should_accumulate) - sV_cur = sV[None, None, None, Vi_index] - if const_expr(self.uneven_kv_smem): - sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase) - gemm_Pi[stage]( - tCrB=tOrVi, - sB=sV_cur, - zero_init=not O_should_accumulate, - mbar_ptr=mbar_ptr + self.mbar_P_full_2_offset + stage, - mbar_phase=P_full_O_rescaled_phase, - ) - # 4. release accumulated O0_partial / O1_partial - # Don't need to signal O_full to the correction warps anymore since the - # correction warps wait for the softmax warps anyway. By the time the softmax - # warps finished, S_i for the next iteration must have been done, so O_i-1 - # must have been done as well. - # with cute.arch.elect_one(): - # tcgen05.commit(mbar_ptr + self.mbar_O_full_offset + stage) - # 5. release V(i-1) - if const_expr(stage == self.q_stage - 1): - pipeline_kv.consumer_release(mma_kv_release_state) - mma_kv_release_state.advance() - # End of GEMM_PV00 (P0 * V0 -> O0_partial) - - # GEMM_QK0i (Q0 * Ki -> S0) - # 1. wait for Ki - if const_expr(stage == 0): - mma_kv_consumer_state.advance() - pipeline_kv.consumer_wait(mma_kv_consumer_state) - Ki_index, Ki_phase = ( - mma_kv_consumer_state.index, - mma_kv_consumer_state.phase, - ) - # 2. gemm - # Don't need to wait for the softmax warp to have finished reading the previous - # Si, since this gemm is scheduled after the PV gemm, which guaranteed that Si - # has been read and Pi has been written. - # tiled_mma_qk = sm100_utils.gemm(tiled_mma_qk, tStSs[stage], tSrQs[stage], tSrK[None, None, None, Ki_index], zero_init=True) - sK_cur = sK[None, None, None, Ki_index] - if const_expr(self.uneven_kv_smem): - sK_cur = self.offset_kv_smem(sK_cur, Ki_index, Ki_phase) - gemm_Si[stage](tCrB=tSrK[None, None, None, Ki_index], sB=sK_cur) - # 3. release S0 - with cute.arch.elect_one(): - tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage) - # End of GEMM_QK0i (Q0 * Ki -> S0) - # 4. release Ki - pipeline_kv.consumer_release(mma_kv_consumer_state) - mma_kv_consumer_state.advance() - P_full_O_rescaled_phase ^= 1 - O_should_accumulate = True - # End of seqlen_kv loop - - # release Q0 & Q1 - with cute.arch.elect_one(): - for stage in cutlass.range_constexpr(self.q_stage): - tcgen05.commit(mbar_ptr + self.mbar_load_q_empty_offset + stage) - - # GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop - # 1. wait for V0 - pipeline_kv.consumer_wait(mma_kv_consumer_state) - Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase - tOrVi = tOrV[None, None, None, Vi_index] - for stage in cutlass.range_constexpr(self.q_stage): - # 2. acquire corrected Oi_partial and Pi - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage, - P_full_O_rescaled_phase, - ) - # 3. gemm - # sm100_utils.gemm(tiled_mma_pv, tOtO0, tOrP0, tOrVi, zero_init=True) - # gemm_Pi[stage](tCrB=tOrVi, sB=sV[None, None, None, Vi_index], zero_init=not O_should_accumulate) - sV_cur = sV[None, None, None, Vi_index] - if const_expr(self.uneven_kv_smem): - sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase) - gemm_Pi[stage]( - tCrB=tOrVi, - sB=sV_cur, - zero_init=not O_should_accumulate, - mbar_ptr=mbar_ptr + self.mbar_P_full_2_offset + stage, - mbar_phase=P_full_O_rescaled_phase, - ) - # 4. release accumulated O0_partial - # We do need O_full here since for the last tile, by the time the softmax warp - # has signaled to the correction warps, the softmax warp has just finished compute - # the row sum of the current tile. It does not guarantee that the 1st tile - # of the next work tile has been computed yet. - with cute.arch.elect_one(): - tcgen05.commit(mbar_ptr + self.mbar_O_full_offset + stage) - # End of GEMM_PV00 (P0 * V0 -> O0_partial) - P_full_O_rescaled_phase ^= 1 - # 5. release Vi_end - pipeline_kv.consumer_release(mma_kv_consumer_state) - mma_kv_consumer_state.advance() - # End of GEMM_PV1(i_end) (P1 * Vi_end -> O1) - - # Advance to next tile - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - # End of persistent scheduler loop - - # for both softmax0 and softmax1 warp group - @cute.jit - def softmax_loop( - self, - stage: int | Int32, - softmax_scale_log2: Float32, - softmax_scale: Float32, - thr_mma_qk: cute.core.ThrMma, - tStSi: cute.Tensor, - sScale: cute.Tensor, - mLSE: Optional[cute.Tensor], - learnable_sink: Optional[cute.Tensor], - mbar_ptr: cute.Pointer, - block_info: BlockInfo, - num_splits: Int32, - SeqlenInfoCls: Callable, - AttentionMaskCls: Callable, - TileSchedulerCls: Callable, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - """Compute softmax on attention scores from QK matrix multiplication. - - This method handles the softmax computation for either the first or second half of the - attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum - and sum values needed for stable softmax computation, applies optional masking, and - transforms raw attention scores into probability distributions. - - The implementation uses specialized memory access patterns and efficient math operations - for computing exp(x) using exp2 functions. It also coordinates pipeline - synchronization between MMA, correction, and sequence processing stages. - """ - tidx = cute.arch.thread_idx()[0] % ( - cute.arch.WARP_SIZE - # * (len(self.softmax0_warp_ids) if stage == 0 else len(self.softmax1_warp_ids) - * (len(self.softmax0_warp_ids)) - ) - - tStScale = cute.composition(tStSi, cute.make_layout((self.m_block_size, 1))) - tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2])) - tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1))) - - tilePlikeFP32 = self.mma_tiler_qk[1] // 32 * self.v_dtype.width - tStP_layout = cute.composition( - tStSi.layout, cute.make_layout((self.m_block_size, tilePlikeFP32)) - ) - tStP = cute.make_tensor(tStSi.iterator + self.tmem_s_to_p_offset, tStP_layout) - - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), - Float32, - ) - thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi).get_slice(tidx) - tStS_t2r = thr_tmem_load.partition_S(tStSi) - - tmem_store_scale_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)), - Float32, - ) - thr_tmem_store_scale = tcgen05.make_tmem_copy(tmem_store_scale_atom, tStScale).get_slice( - tidx - ) - - tStScale_r2t = thr_tmem_store_scale.partition_D(tStScale) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16)), - Float32, - ) - thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStP).get_slice(tidx) - tStP_r2t = thr_tmem_store.partition_D(tStP) - - mma_si_consumer_phase = Int32(0) - si_corr_producer_phase = Int32(1) - s0_s1_sequence_phase = Int32(1 if stage == 0 else 0) - - # self.warp_scheduler_barrier_init() - - warp_idx_in_wg = cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4 - mbar_s0_s1_sequence_offset = self.mbar_s0_s1_sequence_offset + warp_idx_in_wg - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - n_block_min, n_block_max = block_info.get_n_block_min_max( - seqlen, m_block, split_idx, num_splits - ) - - mask = AttentionMaskCls(seqlen) - shared_mask_kwargs = dict( - m_block=self.q_stage * m_block + stage, - thr_mma=thr_mma_qk, - thr_tmem_load=thr_tmem_load, - mask_causal=self.is_causal, - mask_local=self.is_local, - batch_idx=batch_idx, - head_idx=head_idx, - aux_tensors=aux_tensors, - ) - - # Recompute fastdiv_mods if necessary - recompute_fastdiv_mods_q = cutlass.const_expr( - aux_tensors is not None and (seqlen.has_cu_seqlens_q or seqlen.has_seqused_q) - ) - recompute_fastdiv_mods_k = cutlass.const_expr( - aux_tensors is not None and (seqlen.has_cu_seqlens_k or seqlen.has_seqused_k) - ) - - if cutlass.const_expr(fastdiv_mods is not None): - seqlen_q_divmod, seqlen_k_divmod = fastdiv_mods - fastdiv_mods = ( - seqlen_q_divmod - if not recompute_fastdiv_mods_q - else FastDivmodDivisor(seqlen.seqlen_q), - seqlen_k_divmod - if not recompute_fastdiv_mods_k - else FastDivmodDivisor(seqlen.seqlen_k), - ) - - mask_mod = self.mask_mod if const_expr(self.mask_mod is not None) else None - mask_fn = partial( - mask.apply_mask_sm100, - mask_mod=mask_mod, - fastdiv_mods=fastdiv_mods, - **shared_mask_kwargs, - ) - if const_expr(self.use_block_sparsity): - # Full blocks dont need mask_mod - mask_fn_none = partial( - mask.apply_mask_sm100, - mask_mod=None, - fastdiv_mods=fastdiv_mods, - **shared_mask_kwargs, - ) - else: - mask_fn_none = None - - softmax = SoftmaxSm100.create( - softmax_scale_log2, - rescale_threshold=8.0 if const_expr(self.q_dtype.width == 16) else 0.0, - softmax_scale=softmax_scale, - ) - softmax.reset() - - if const_expr(self.use_block_sparsity): - tile_block_count = get_total_block_count( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - has_work = tile_block_count > Int32(0) - else: - tile_block_count = n_block_max - n_block_min - has_work = const_expr(not self.is_split_kv) or tile_block_count > Int32(0) - - softmax_step = partial( - self.softmax_step, - softmax=softmax, - mbar_ptr=mbar_ptr, - mbar_s0_s1_sequence_offset=mbar_s0_s1_sequence_offset, - thr_mma_qk=thr_mma_qk, - thr_tmem_load=thr_tmem_load, - thr_tmem_store=thr_tmem_store, - thr_tmem_store_scale=thr_tmem_store_scale, - tStS_t2r=tStS_t2r, - tStScale_r2t=tStScale_r2t, - tStP_r2t=tStP_r2t, - sScale=sScale, - stage=stage, - batch_idx=batch_idx, - head_idx=head_idx, - m_block=self.q_stage * m_block + stage, - seqlen=seqlen, - aux_tensors=aux_tensors, - fastdiv_mods=fastdiv_mods, - ) - - if has_work: - # Softmax acts as the producer: wait until correction signals the stage is empty - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase - ) - si_corr_producer_phase ^= 1 - - # Block sparse or dense iteration - if const_expr(self.use_block_sparsity): - # When aux_tensors exist, Q indices beyond seqlen_q must be wrapped to avoid - # OOB aux_tensor access. Only edge tiles (where m_tile_end > seqlen_q) need this. - if const_expr(aux_tensors is not None): - m_tile_end = (self.q_stage * m_block + stage + 1) * self.m_block_size - check_m_boundary = m_tile_end > seqlen.seqlen_q - else: - check_m_boundary = False - ( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - empty_tile, - ) = softmax_block_sparse_sm100( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - softmax_step, - mask_fn, - mask_fn_none, - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - mbar_ptr, - self.mbar_softmax_corr_full_offset, - self.mbar_softmax_corr_empty_offset, - self.mbar_P_full_O_rescaled_offset, - self.mbar_P_full_2_offset, - self.q_stage, - Int32(stage), - check_m_boundary, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - if not empty_tile: - sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0] - if const_expr(mLSE is not None or learnable_sink is not None): - sScale[tidx + stage * self.m_block_size + self.m_block_size * 2] = ( - softmax.row_max[0] - ) - # if tidx == 0: - # cute.printf("softmax row sum stage %d: %f, row_max = %f\n", stage, softmax.row_sum[0], softmax.row_max[0]) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage) - # if tidx == 0: cute.printf("softmax row sum stage %d: %f\n", stage, softmax.row_sum[0]) - else: - if const_expr(not self.is_split_kv) or tile_block_count > Int32(0): - mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = ( - softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - n_block_max - 1, - is_first=True, - mask_fn=partial(mask_fn, mask_seqlen=True), - ) - ) - n_block_max -= 1 - # Next couple of iterations with causal masking - if const_expr(self.is_causal or self.is_local): - n_block_min_causal_local_mask = ( - block_info.get_n_block_min_causal_local_mask( - seqlen, m_block, n_block_min - ) - ) - for n_tile in cutlass.range( - n_block_max - n_block_min_causal_local_mask, unroll=1 - ): - n_block = n_block_max - 1 - n_tile - mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = ( - softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - n_block, - mask_fn=partial(mask_fn, mask_seqlen=False), - ) - ) - n_block_max = cutlass.min(n_block_max, n_block_min_causal_local_mask) - # The remaining iterations have no masking (but may still need mask_mod) - n_block_min_before_local_mask = block_info.get_n_block_min_before_local_mask( - seqlen, m_block, n_block_min - ) - for n_tile in cutlass.range( - n_block_max - n_block_min_before_local_mask, unroll=1 - ): - n_block = n_block_max - n_tile - 1 - if const_expr(self.mask_mod is not None): - mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = ( - softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - n_block, - mask_fn=partial(mask_fn, mask_seqlen=False), - ) - ) - else: - mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = ( - softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - n_block, - ) - ) - # Separate iterations with local masking on the left - if const_expr(self.is_local and block_info.window_size_left is not None): - n_block_max = cutlass.min(n_block_max, n_block_min_before_local_mask) - for n_tile in cutlass.range(0, n_block_max - n_block_min, unroll=1): - n_block = n_block_max - 1 - n_tile - mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = ( - softmax_step( - mma_si_consumer_phase, - si_corr_producer_phase, - s0_s1_sequence_phase, - n_block, - mask_fn=partial(mask_fn, mask_seqlen=False), - ) - ) - # Now that we no longer already have the 1st iteration, need mask_seqlen=True here - - # Dense path always writes scale / signals - sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0] - if const_expr(mLSE is not None or learnable_sink is not None): - sScale[tidx + stage * self.m_block_size + self.m_block_size * 2] = ( - softmax.row_max[0] - ) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage) - - # # Write LSE to gmem - # if const_expr(mLSE is not None): - # acc_O_mn_row_is_zero_or_nan = softmax.row_sum[0] == 0.0 or softmax.row_sum[0] != softmax.row_sum[0] - # scale = ( - # cute.arch.rcp_approx(softmax.row_sum[0] if not acc_O_mn_row_is_zero_or_nan else 1.0) - # ) - # LN2 = math.log(2.0) - # lse = ( - # (softmax.row_max[0] * softmax.scale_log2 + utils.log2f(softmax.row_sum[0])) * LN2 - # if not acc_O_mn_row_is_zero_or_nan else -Float32.inf - # ) - # if const_expr(not seqlen.has_cu_seqlens_q): - # mLSE_cur = mLSE[None, head_idx, batch_idx] - # else: - # mLSE_cur = cute.domain_offset((seqlen.offset_q,), mLSE[None, head_idx]) - # gLSE = cute.local_tile(mLSE_cur, (self.m_block_size,), (m_block * 2 + stage,)) - # if tidx < seqlen.seqlen_q - (m_block * 2 + stage) * self.m_block_size: - # gLSE[tidx] = lse - - # Advance to next tile - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def softmax_step( - self, - mma_si_consumer_phase: Int32, - si_corr_producer_phase: Int32, - s0_s1_sequence_phase: Int32, - n_block: Int32, - softmax: SoftmaxSm100, - mbar_ptr: cute.Pointer, - mbar_s0_s1_sequence_offset: Int32, - thr_mma_qk: cute.core.ThrMma, - thr_tmem_load: cute.CopyAtom, - thr_tmem_store: cute.CopyAtom, - thr_tmem_store_scale: cute.CopyAtom, - tStS_t2r: cute.Tensor, - tStScale_r2t: cute.Tensor, - tStP_r2t: cute.Tensor, - sScale: cute.Tensor, - stage: int | Int32, - batch_idx: Int32, - head_idx: Int32, - m_block: Int32, - seqlen, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - mask_fn: Optional[Callable] = None, - is_first: bool = False, - ) -> Tuple[cute.Int32, cute.Int32, cute.Int32]: - """Perform a single step of the softmax computation on a block of attention scores. - - This method processes one block of the attention matrix, computing numerically stable - softmax by first finding the row maximum, subtracting it from all elements, applying - exponential function, and then normalizing by the sum of exponentials. It also handles - optional masking of attention scores. - - The method involves several key operations: - 1. Loading attention scores from tensor memory - 2. Applying optional masking based on position - 3. Computing row-wise maximum values for numerical stability - 4. Transforming scores using exp2(x*scale - max*scale) - 5. Computing row sums for normalization - 6. Coordinating pipeline synchronization between different processing stages - """ - tilePlikeFP32 = self.mma_tiler_qk[1] // Float32.width * self.v_dtype.width - tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2])) - tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1))) - tScP = cute.composition(tScS, cute.make_layout((self.m_block_size, tilePlikeFP32))) - - # Wait for Si - cute.arch.mbarrier_wait(mbar_ptr + self.mbar_S_full_offset + stage, mma_si_consumer_phase) - tSrS_t2r = cute.make_fragment(thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype) - cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r) - if cutlass.const_expr(self.score_mod is not None): - self.apply_score_mod( - tSrS_t2r, - thr_tmem_load, - thr_mma_qk, - batch_idx, - head_idx, - m_block, - n_block, - softmax, - seqlen, - aux_tensors, - fastdiv_mods, - ) - - if const_expr(mask_fn is not None): - mask_fn(tSrS_t2r, n_block=n_block) - row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first) - - if const_expr(not is_first): - # tSrScale_r2t = cute.make_fragment(thr_tmem_store_scale.partition_S(tScScale).shape, Float32) - # tSrScale_r2t[0] = acc_scale - # cute.copy(thr_tmem_store_scale, tSrScale_r2t, tStScale_r2t) - # cute.arch.fence_view_async_tmem_store() - thread_idx = thr_tmem_load.thr_idx - sScale[thread_idx + stage * self.m_block_size] = acc_scale - # if thread_idx == 0: cute.printf("softmax acc_scale stage %d: %f, row_max = %f\n", stage, acc_scale, row_max) - # Notify correction wg that row_max is ready - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage) - - # if thread_idx == 0 and stage == 0: cute.print_tensor(tSrS_t2r) - # print(tSrS_t2r) - softmax.scale_subtract_rowmax(tSrS_t2r, row_max) - # Sequence barrier wait - if const_expr(self.s0_s1_barrier): - cute.arch.mbarrier_wait( - mbar_ptr + mbar_s0_s1_sequence_offset + stage * 4, s0_s1_sequence_phase - ) - tSrP_r2t_f32 = cute.make_fragment(thr_tmem_store.partition_S(tScP).shape, Float32) - tSrP_r2t = cute.make_tensor( - cute.recast_ptr(tSrP_r2t_f32.iterator, dtype=self.q_dtype), - tSrS_t2r.layout, - ) - # softmax.scale_apply_exp2_convert(tSrS_t2r, row_max, tSrP_r2t) - softmax.apply_exp2_convert( - tSrS_t2r, - tSrP_r2t, - e2e=mask_fn is None and self.head_dim_padded <= 128, - e2e_freq=self.e2e_freq, - ) - # Sequence barrier arrive - if const_expr(self.s0_s1_barrier): - cute.arch.mbarrier_arrive(mbar_ptr + mbar_s0_s1_sequence_offset + (1 - stage) * 4) - # print(tSrP_r2t_f32, tStP_r2t) - # cute.copy(thr_tmem_store, tSrP_r2t_f32, tStP_r2t) - for i in cutlass.range_constexpr(cute.size(tStP_r2t.shape[2]) // 4 * 3): - cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i]) - cute.arch.fence_view_async_tmem_store() - # Notify mma warp that P is ready - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage) - for i in cutlass.range_constexpr( - cute.size(tStP_r2t.shape[2]) // 4 * 3, cute.size(tStP_r2t.shape[2]) - ): - cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i]) - cute.arch.fence_view_async_tmem_store() - # Notify mma warp that the 2nd half of P is ready - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_2_offset + stage) - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase - ) - softmax.update_row_sum(tSrS_t2r.load(), acc_scale, is_first) - # acc_scale = cute.arch.exp2(acc_scale_) - return mma_si_consumer_phase ^ 1, si_corr_producer_phase ^ 1, s0_s1_sequence_phase ^ 1 - - @cute.jit - def correction_loop( - self, - thr_mma_qk: cute.core.ThrMma, - thr_mma_pv: cute.core.ThrMma, - tStS: cute.Tensor, - tOtOs: tuple[cute.Tensor], - sScale: cute.Tensor, - mO: cute.Tensor, - mLSE: cute.Tensor, - sO: cute.Tensor, - learnable_sink: Optional[cute.Tensor], - gmem_tiled_copy_O: cute.TiledCopy, - tma_atom_O: cute.CopyAtom, - mbar_ptr: cute.Pointer, - softmax_scale_log2: Float32, - block_info: BlockInfo, - num_splits: Int32, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - blocksparse_tensors: Optional[BlockSparseTensors] = None, - ): - tidx = cute.arch.thread_idx()[0] % (cute.arch.WARP_SIZE * len(self.correction_warp_ids)) - tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2])) - tStScale_layout = cute.composition(tStS.layout, cute.make_layout((self.m_block_size, 1))) - tStScales = tuple( - cute.make_tensor(tStS.iterator + self.tmem_vec_offset[stage], tStScale_layout) - for stage in range(self.q_stage) - ) - tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1))) - tmem_load_v_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)), - self.qk_acc_dtype, - ) - thr_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStScales[0]).get_slice(tidx) - - tStScales_t2r = [ - thr_tmem_load_vec.partition_S(tStScales[stage]) for stage in range(self.q_stage) - ] - tSrScale_t2r_shape = thr_tmem_load_vec.partition_D(tScScale).shape - - # First iter: no correction is required - for stage in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage) - - softmax_corr_consumer_phase = Int32(0) - o_corr_consumer_phase = Int32(0) - corr_epi_producer_phase = Int32(1) - - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - n_block_min, n_block_max = block_info.get_n_block_min_max( - seqlen, m_block, split_idx, num_splits - ) - - if const_expr(self.is_split_kv): - mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[ - None, None, head_idx, split_idx - ] - else: - mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx] - gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0)) - - # Default LSE to -inf for invalid split_idx tiles - stats = [ - ( - 0.0, - -Float32.inf - if const_expr(mLSE is not None or learnable_sink is not None) - else None, - True, - ) - ] * self.q_stage - - if const_expr(self.use_block_sparsity): - total_block_count = get_total_block_count( - blocksparse_tensors, - batch_idx, - head_idx, - m_block, - self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1, - ) - has_work = total_block_count > Int32(0) - else: - total_block_count = n_block_max - n_block_min - has_work = const_expr(not self.is_split_kv) or total_block_count > Int32(0) - - if has_work: - # Ignore first signal from softmax as no correction is required - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_full_offset + 0, softmax_corr_consumer_phase - ) - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + 0) - if const_expr(self.q_stage == 2): - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_full_offset + 1, - softmax_corr_consumer_phase, - ) - softmax_corr_consumer_phase ^= 1 - - tSrScale_t2r = cute.make_fragment(tSrScale_t2r_shape, Float32) - for i in cutlass.range(total_block_count - 1, unroll=1): - for stage in cutlass.range_constexpr(self.q_stage): - # wait for S0 / S1 - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_full_offset + stage, - softmax_corr_consumer_phase, - ) - # cute.copy(tiled_tmem_load_vec, tStScales_t2r[stage], tSrScale_t2r) - # cute.arch.fence_view_async_tmem_load() - # scale = tSrScale_t2r[0] - scale = sScale[tidx + stage * self.m_block_size] - should_rescale = cute.arch.vote_ballot_sync(scale < 1.0) != 0 - # should_rescale = True - # if tidx == 0: cute.printf("Correction scale i = %d, for stage %d: %f, should_rescale = %d\n", i, stage, scale, should_rescale) - # Don't need O_full anymore, since by the time softmax has signaled the correction - # warps, S_i must have been done, so O_i-1 must have been done as well. - # cute.arch.mbarrier_wait(mbar_ptr + self.mbar_O_full_offset + stage, o_corr_consumer_phase) - if should_rescale: - self.correction_rescale(thr_mma_pv, tOtOs[stage], tidx, scale) - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage - ) - if const_expr(self.q_stage == 2): - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_softmax_corr_empty_offset + (1 - stage) - ) - else: - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_softmax_corr_empty_offset + stage - ) - softmax_corr_consumer_phase ^= 1 - # o_corr_consumer_phase ^= 1 - if const_expr(self.q_stage == 2): - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + 1) - # End of seqlen_corr_loop_steps - - # Even in the case of self.overlap_sO_sQ, we can write to stage 0 of sO without - # additional sync because the MMA in the top half must have been done. - # Similarly we can write to stage 1 of sO without additional sync. - learnable_sink_val = [None] * self.q_stage - if const_expr(learnable_sink is not None): - if const_expr(not self.pack_gqa): - sink_val = Float32(learnable_sink[head_idx]) - learnable_sink_val = [sink_val] * self.q_stage - else: # Each thread might have a different sink value due to different q_head - for stage in cutlass.range_constexpr(self.q_stage): - q_head_idx = ( - (self.q_stage * m_block + stage) * self.m_block_size + tidx - ) % self.qhead_per_kvhead + head_idx * self.qhead_per_kvhead - learnable_sink_val[stage] = Float32(learnable_sink[q_head_idx]) - for stage in cutlass.range_constexpr(self.q_stage): - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_softmax_corr_full_offset + stage, - softmax_corr_consumer_phase, - ) - # cute.copy(tiled_tmem_load_vec, tStScales_t2r[stage], tSrScale_t2r) - # cute.arch.fence_view_async_tmem_load() - # scale = tSrScale_t2r[0] - row_sum = sScale[tidx + stage * self.m_block_size] - if const_expr(mLSE is not None or learnable_sink is not None): - row_max = sScale[tidx + stage * self.m_block_size + self.m_block_size * 2] - else: - row_max = None - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_softmax_corr_empty_offset + stage - ) - if const_expr(learnable_sink is not None): - LOG2_E = math.log2(math.e) - sink_val = learnable_sink_val[stage] - if const_expr(not self.is_split_kv) or split_idx == 0: - if row_max == -Float32.inf: - # It's possible to have an empty row with splitKV. - row_max = sink_val * (LOG2_E / softmax_scale_log2) - row_sum = Float32(1.0) - else: - row_sum += utils.exp2f( - sink_val * LOG2_E - row_max * softmax_scale_log2 - ) - acc_O_mn_row_is_zero_or_nan = row_sum == 0.0 or row_sum != row_sum - stats[stage] = (row_sum, row_max, acc_O_mn_row_is_zero_or_nan) - scale = cute.arch.rcp_approx( - row_sum if not acc_O_mn_row_is_zero_or_nan else 1.0 - ) - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_O_full_offset + stage, o_corr_consumer_phase - ) - if const_expr(not self.use_correction_warps_for_epi): - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_corr_epi_empty_offset + stage, - corr_epi_producer_phase, - ) - self.correction_epilogue( - thr_mma_pv, - tOtOs[stage], - tidx, - stage, - m_block, - seqlen.seqlen_q, - scale, - sO[None, None, stage], - mO_cur, - gO, - gmem_tiled_copy_O, - ) - if const_expr(not self.use_correction_warps_for_epi): - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_full_offset + stage) - # Signal for the next work tile that O buffers in tmem are already read, so - # mma warp can write to them - cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage) - # if tidx == 0: cute.printf("Correction final scale for stage %d: %f\n", stage, scale) - - o_corr_consumer_phase ^= 1 - softmax_corr_consumer_phase ^= 1 - corr_epi_producer_phase ^= 1 - else: - # WARNING: we need some code before the const_expr, see https://github.com/NVIDIA/cutlass/issues/2781 - if const_expr(self.use_correction_warps_for_epi): - gmem_tiled_copy_O_for_empty_tile = gmem_tiled_copy_O - else: - gmem_tiled_copy_O_for_empty_tile = None - if const_expr(self.use_block_sparsity): - ( - softmax_corr_consumer_phase, - o_corr_consumer_phase, - corr_epi_producer_phase, - ) = handle_block_sparse_empty_tile_correction_sm100( - tidx, - self.q_stage, - self.m_block_size, - self.qhead_per_kvhead, - self.pack_gqa, - self.is_split_kv, - learnable_sink, - mLSE, - seqlen, - m_block, - head_idx, - batch_idx, - split_idx, - sScale, - stats, - self.correction_epilogue, - thr_mma_pv, - tOtOs, - sO, - mbar_ptr, - self.mbar_softmax_corr_full_offset, - self.mbar_softmax_corr_empty_offset, - self.mbar_P_full_O_rescaled_offset, - self.mbar_P_full_2_offset, - self.mbar_corr_epi_full_offset, - self.mbar_corr_epi_empty_offset, - softmax_corr_consumer_phase, - o_corr_consumer_phase, - corr_epi_producer_phase, - softmax_scale_log2, - mO_cur, - gO, - gmem_tiled_copy_O_for_empty_tile, - ) - - if const_expr(mLSE is not None): - if const_expr(not seqlen.has_cu_seqlens_q): - if const_expr(self.is_split_kv): - mLSE_cur = mLSE[None, head_idx, batch_idx, split_idx] - else: - mLSE_cur = mLSE[None, head_idx, batch_idx] - else: - offset = ( - seqlen.offset_q if const_expr(not self.pack_gqa) else (0, seqlen.offset_q) - ) - if const_expr(self.is_split_kv): - mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx, split_idx]) - else: - mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx]) - for stage in cutlass.range_constexpr(self.q_stage): - gLSE = cute.local_tile( - mLSE_cur, (self.m_block_size,), (self.q_stage * m_block + stage,) - ) - row_sum, row_max, acc_O_mn_row_is_zero_or_nan = stats[stage] - # if tidx == 0 and stage <= 1: - # cute.printf("row_sum = {}, row_max = {}, acc_O_mn_row_is_zero_or_nan = {}\n", row_sum, row_max, acc_O_mn_row_is_zero_or_nan) - LN2 = math.log(2.0) - lse = ( - (row_max * softmax_scale_log2 + utils.log2f(row_sum)) * LN2 - if not acc_O_mn_row_is_zero_or_nan - else -Float32.inf - ) - seqlen_q = ( - seqlen.seqlen_q - if const_expr(not self.pack_gqa) - else seqlen.seqlen_q * self.qhead_per_kvhead - ) - if tidx < seqlen_q - (self.q_stage * m_block + stage) * self.m_block_size: - # This actually just works with PackGQA too - gLSE[tidx] = lse - - # Advance to next tile - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def correction_rescale( - self, - thr_mma: cute.core.ThrMma, - tOtO: cute.Tensor, - tidx: Int32, - scale: Float32, - ): - """Rescale intermediate attention results based on softmax normalization factor. - - This method performs a crucial correction step in the attention computation pipeline. - When processing attention in blocks, the softmax normalization factors may change - as new blocks are processed. This method rescales previously computed partial - output values to account for updated normalization factors. - - The implementation uses efficient tensor memory operations to: - 1. Load existing partial attention output from tensor memory - 2. Apply the scaling factor to all elements - 3. Store the rescaled results back to tensor memory - """ - tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2])) - corr_tile_size = 16 # tuneable parameter - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tOtO_i = cute.composition(tOtO, cute.make_layout((self.m_block_size, corr_tile_size))) - tOcO_i = cute.composition(tOcO, cute.make_layout((self.m_block_size, corr_tile_size))) - thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i).get_slice(tidx) - thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i).get_slice(tidx) - tOtO_t2r = thr_tmem_load.partition_S(tOtO_i) - tOrO_t2r_shape = thr_tmem_load.partition_D(tOcO_i).shape - tOtO_r2t = thr_tmem_store.partition_D(tOtO_i) - - frg_count = self.head_dim_v_padded // corr_tile_size - tOrO_frg = cute.make_fragment((tOrO_t2r_shape, frg_count), self.pv_acc_dtype) - for i in cutlass.range_constexpr(frg_count): - tOrO_frg = cute.make_fragment(tOrO_t2r_shape, self.pv_acc_dtype) - tOtO_t2r_i = cute.make_tensor(tOtO_t2r.iterator + i * corr_tile_size, tOtO_t2r.layout) - cute.copy(thr_tmem_load, tOtO_t2r_i, tOrO_frg) - for j in cutlass.range(0, cute.size(tOrO_frg), 2, unroll_full=True): - tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2( - (tOrO_frg[j], tOrO_frg[j + 1]), - (scale, scale), - ) - tOtO_r2t_i = cute.make_tensor(tOtO_r2t.iterator + i * corr_tile_size, tOtO_r2t.layout) - cute.copy(thr_tmem_store, tOrO_frg, tOtO_r2t_i) - cute.arch.fence_view_async_tmem_store() - - @cute.jit - def correction_epilogue( - self, - thr_mma: cute.core.ThrMma, - tOtO: cute.Tensor, - tidx: Int32, - stage: Int32, - m_block: Int32, - seqlen_q: Int32, - scale: Float32, - sO: cute.Tensor, - mO_cur: Optional[cute.Tensor] = None, - gO: Optional[cute.Tensor] = None, - gmem_tiled_copy_O: Optional[cute.TiledCopy] = None, - ): - """Apply final scaling and transformation to attention output before writing to global memory. - - This correction_epilogue function handles the final processing step for attention output values. - It applies a scaling factor to the accumulated attention results and prepares the - data for efficient transfer back to global memory. - - The method performs: - 1. Loading of accumulated attention results from tensor memory - 2. Application of the final output scaling factor - 3. Type conversion if necessary (typically from higher precision accumulator to output precision) - 4. Reorganization of data for optimal memory access patterns - 5. Preparation for efficient TMA store operations - - :param thr_mma: Thread MMA operation for the computation - :type thr_mma: cute.core.ThrMma - :param tOtO: Tensor containing accumulated attention output - :type tOtO: cute.Tensor - :param scale: Final scaling factor to apply to the output - :type scale: Float32 - :param sO: Shared memory tensor for the final output - :type sO: cute.Tensor - """ - - corr_tile_size = 32 * 8 // self.o_dtype.width - tOsO = thr_mma.partition_C(sO) - tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2])) - - tOtO_i = cute.logical_divide(tOtO, cute.make_layout((self.m_block_size, corr_tile_size))) - tOcO_i = cute.logical_divide(tOcO, cute.make_layout((self.m_block_size, corr_tile_size))) - tOsO_i = cute.logical_divide(tOsO, cute.make_layout((self.m_block_size, corr_tile_size))) - - epi_subtile = (self.epi_tile[0], corr_tile_size) - tmem_copy_atom = sm100_utils_basic.get_tmem_load_op( - self.mma_tiler_pv, - self.o_layout, - self.o_dtype, - self.pv_acc_dtype, - epi_subtile, - use_2cta_instrs=False, - ) - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]).get_slice( - tidx - ) - thr_tmem_load = tiled_tmem_load.get_slice(tidx) - smem_copy_atom = sm100_utils_basic.get_smem_store_op( - self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load - ) - tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load) - - tOtO_t2r = thr_tmem_load.partition_S(tOtO_i[(None, None), None]) - tOsO_s2r = thr_tmem_load.partition_D(tOsO_i[(None, None), None]) - tOcO_t2r = thr_tmem_load.partition_D(tOcO_i[(None, None), None]) - for i in cutlass.range_constexpr(self.head_dim_v_padded // corr_tile_size): - tOtO_t2r_i = tOtO_t2r[None, 0, 0, i] - tOsO_r2s_i = tOsO_s2r[None, 0, 0, i] - tOrO_frg = cute.make_fragment(tOcO_t2r[None, 0, 0, i].shape, self.pv_acc_dtype) - cute.copy(tiled_tmem_load, tOtO_t2r_i, tOrO_frg) - for j in cutlass.range_constexpr(0, cute.size(tOrO_frg), 2): - tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2( - (tOrO_frg[j], tOrO_frg[j + 1]), - (scale, scale), - ) - tOrO_frg_cvt = cute.make_fragment(tOrO_frg.shape, self.o_dtype) - tOrO_frg_cvt.store(tOrO_frg.load().to(self.o_dtype)) - cute.copy(tiled_smem_store, tOrO_frg_cvt, tOsO_r2s_i) - # fence view async shared - cute.arch.fence_proxy( - "async.shared", - space="cta", - ) - - if const_expr(self.use_correction_warps_for_epi): - assert not self.use_tma_O - assert gmem_tiled_copy_O is not None - cute.arch.barrier( - barrier_id=int(NamedBarrierFwd.Epilogue), - number_of_threads=len(self.epilogue_warp_ids) * cute.arch.WARP_SIZE, - ) - gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx) - tOsO = gmem_thr_copy_O.partition_S(sO) - cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded)) - tOgO = gmem_thr_copy_O.partition_D(gO) - tOcO = gmem_thr_copy_O.partition_S(cO) - t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO) - tOpO = utils.predicate_k(tOcO, limit=mO_cur.shape[1]) - pack_gqa = PackGQA( - self.m_block_size, - self.head_dim_v_padded, - self.check_hdim_v_oob, - self.qhead_per_kvhead, - ) - - # load acc O from smem to rmem for wider vectorization - tOrO = cute.make_fragment_like(tOsO, self.o_dtype) - cute.autovec_copy(tOsO, tOrO) - # copy acc O from rmem to gmem - if const_expr(not self.pack_gqa): - for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])): - if ( - t0OcO[0, rest_m, 0][0] - < seqlen_q - - (self.q_stage * m_block + stage) * self.m_block_size - - tOcO[0][0] - ): - cute.copy( - gmem_tiled_copy_O, - tOrO[None, rest_m, None], - tOgO[None, rest_m, None, self.q_stage * m_block + stage], - pred=tOpO[None, rest_m, None] - if const_expr(self.check_hdim_v_oob) - else None, - ) - else: - pack_gqa.store_O( - mO_cur, - tOrO, - gmem_tiled_copy_O, - tidx, - self.q_stage * m_block + stage, - seqlen_q, - ) - - @cute.jit - def epilogue_s2g( - self, - mO: cute.Tensor, - sO: cute.Tensor, - gmem_tiled_copy_O: cute.TiledCopy, - tma_atom_O: Optional[cute.CopyAtom], - mbar_ptr: cute.Pointer, - block_info: BlockInfo, - num_splits: int, - SeqlenInfoCls: Callable, - TileSchedulerCls: Callable, - ): - epi_consumer_phase = Int32(0) - tile_scheduler = TileSchedulerCls() - work_tile = tile_scheduler.initial_work_tile_info() - while work_tile.is_valid_tile: - m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx - seqlen = SeqlenInfoCls(batch_idx) - n_block_min, n_block_max = block_info.get_n_block_min_max( - seqlen, m_block, split_idx, num_splits - ) - - if const_expr(not self.is_split_kv) or n_block_min < n_block_max: - if const_expr(self.is_split_kv): - mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[ - None, None, head_idx, split_idx - ] - else: - mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx] - gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0)) - if const_expr(self.use_tma_O): - store_O, _, _ = copy_utils.tma_get_copy_fn( - tma_atom_O, 0, cute.make_layout(1), sO, gO - ) - for stage in cutlass.range_constexpr(self.q_stage): - # wait from corr, issue tma store on smem - # 1. wait for O0 / O1 final - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase - ) - # 2. copy O0 / O1 to gmem - store_O(src_idx=stage, dst_idx=self.q_stage * m_block + stage) - cute.arch.cp_async_bulk_commit_group() - for stage in cutlass.range_constexpr(self.q_stage): - # Ensure O0 / O1 buffer is ready to be released - if const_expr(self.q_stage == 2): - cute.arch.cp_async_bulk_wait_group(1 - stage, read=True) - else: - cute.arch.cp_async_bulk_wait_group(0, read=True) - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_corr_epi_empty_offset + stage - ) - else: - tidx = cute.arch.thread_idx()[0] % ( - cute.arch.WARP_SIZE * len(self.epilogue_warp_ids) - ) - gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx) - tOsO = gmem_thr_copy_O.partition_S(sO) - cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded)) - tOgO = gmem_thr_copy_O.partition_D(gO) - tOcO = gmem_thr_copy_O.partition_S(cO) - t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO) - tOpO = utils.predicate_k(tOcO, limit=mO.shape[1]) - pack_gqa = PackGQA( - self.m_block_size, - self.head_dim_v_padded, - self.check_hdim_v_oob, - self.qhead_per_kvhead, - ) - for stage in cutlass.range_constexpr(self.q_stage): - # wait from corr, issue tma store on smem - # 1. wait for O0 / O1 final - cute.arch.mbarrier_wait( - mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase - ) - # 2. copy O0 / O1 to gmem - # load acc O from smem to rmem for wider vectorization - tOrO = cute.make_fragment_like(tOsO[None, None, None, 0], self.o_dtype) - cute.autovec_copy(tOsO[None, None, None, stage], tOrO) - # copy acc O from rmem to gmem - if const_expr(not self.pack_gqa): - for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])): - if ( - t0OcO[0, rest_m, 0][0] - < seqlen.seqlen_q - - (self.q_stage * m_block + stage) * self.m_block_size - - tOcO[0][0] - ): - cute.copy( - gmem_tiled_copy_O, - tOrO[None, rest_m, None], - tOgO[None, rest_m, None, self.q_stage * m_block + stage], - pred=tOpO[None, rest_m, None] - if const_expr(self.check_hdim_v_oob) - else None, - ) - else: - pack_gqa.store_O( - mO_cur, - tOrO, - gmem_tiled_copy_O, - tidx, - self.q_stage * m_block + stage, - seqlen.seqlen_q, - ) - cute.arch.mbarrier_arrive( - mbar_ptr + self.mbar_corr_epi_empty_offset + stage - ) - - epi_consumer_phase ^= 1 - - # Advance to next tile - tile_scheduler.advance_to_next_work() - work_tile = tile_scheduler.get_current_work() - - def load_Q( - self, - load_Q_fn: Callable, - mbar_full_ptr: cute.Pointer, - mbar_empty_ptr: cute.Pointer, - block: Int32, - stage: int, - phase: Int32, - ): - cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase) - with cute.arch.elect_one(): - cute.arch.mbarrier_arrive_and_expect_tx(mbar_full_ptr + stage, self.tma_copy_bytes["Q"]) - load_Q_fn(src_idx=block, dst_idx=stage, tma_bar_ptr=mbar_full_ptr + stage) - - @cute.jit - def load_KV( - self, - tma_atom: Optional[cute.CopyAtom], - tXgX: Optional[cute.Tensor], - tXsX: Optional[cute.Tensor], - paged_kv_manager: Optional[PagedKVManager], - sX: cute.Tensor, - mbar_full_ptr: cute.Pointer, - mbar_empty_ptr: cute.Pointer, - block: Int32, - producer_state: cutlass.pipeline.PipelineState, - K_or_V: Literal["K", "V"], - page_idx: Optional[Int32] = None, - ): - assert K_or_V in ("K", "V") - stage, phase = producer_state.index, producer_state.phase - cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase) - if const_expr(K_or_V == "K" and self.uneven_kv_smem): - # Before this round, the smem location was occupied by V, which is smaller than - # K. So we need to wait for the stage after that (stage 1) to be empty as well. - if stage == 0: - cute.arch.mbarrier_wait(mbar_empty_ptr + 1, phase) - - if const_expr(self.use_tma_KV): - assert tXgX is not None and tXsX is not None and tma_atom is not None - with cute.arch.elect_one(): - cute.arch.mbarrier_arrive_and_expect_tx( - mbar_full_ptr + stage, - self.tma_copy_bytes[K_or_V], - ) - tXsX_cur = tXsX[None, stage] - if const_expr(self.uneven_kv_smem): - # Since this is the producer_state, the phase starts at 1, so we have to invert it - tXsX_cur = self.offset_kv_smem(tXsX_cur, stage, phase ^ 1) - # Currently we assume that page_size == n_block_size so we index into tXgX with block = 0 - tXgX_cur = ( - tXgX[None, block] if const_expr(page_idx is None) else tXgX[None, 0, page_idx] - ) - cute.copy(tma_atom, tXgX_cur, tXsX_cur, tma_bar_ptr=mbar_full_ptr + stage) - else: - assert paged_kv_manager is not None - paged_kv_manager.load_KV(block, sX[None, None, None, stage], K_or_V) - cute.arch.cp_async_commit_group() - cute.arch.cp_async_mbarrier_arrive_noinc(mbar_full_ptr + stage) - - @cute.jit - def offset_kv_smem(self, sX: cute.Tensor, stage: Int32, phase: Int32): - if const_expr(self.uneven_kv_smem): - # smem layout is [smem_large, smem_small, smem_large], and the current stride is - # (smem_large + smem_small) // 2. So for stage == 1, move right by offset if - # phase == 0, or left by offset if phase == 1. - offset = 0 if stage != 1 else self.uneven_kv_smem_offset * (1 - 2 * phase) - return cute.make_tensor(sX.iterator + offset, sX.layout) - else: - return sX - - def make_and_init_load_kv_pipeline(self, load_kv_mbar_ptr): - load_kv_consumer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len([self.mma_warp_id]) - ) - if self.use_tma_KV: - load_kv_producer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len(self.load_warp_ids) - ) - return cutlass.pipeline.PipelineTmaUmma.create( - barrier_storage=load_kv_mbar_ptr, - num_stages=self.kv_stage, - producer_group=load_kv_producer_group, - consumer_group=load_kv_consumer_group, - tx_count=self.tma_copy_bytes["K"], - ) - else: - load_kv_producer_group = cutlass.pipeline.CooperativeGroup( - cutlass.pipeline.Agent.Thread, len(self.load_warp_ids) * cute.arch.WARP_SIZE - ) - return cutlass.pipeline.PipelineAsyncUmma.create( - num_stages=self.kv_stage, - producer_group=load_kv_producer_group, - consumer_group=load_kv_consumer_group, - barrier_storage=load_kv_mbar_ptr, - ) - - # @cute.jit - # def warp_scheduler_barrier_init(self): - # warp_group_idx = utils.canonical_warp_group_idx(sync=False) - # if warp_group_idx == 0: - # cute.arch.barrier_arrive( - # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1), number_of_threads=2 * 128, - # ) - - # def warp_scheduler_barrier_sync(self): - # cute.arch.barrier( - # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + utils.canonical_warp_group_idx(sync=False), - # number_of_threads=2 * 128 - # ) - - # def warp_scheduler_barrier_arrive(self): - # cur_wg = utils.canonical_warp_group_idx(sync=False) - # next_wg = 1 - cur_wg - # cute.arch.barrier_arrive( - # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + next_wg, number_of_threads=2 * 128, - # ) - - @cute.jit - def apply_score_mod( - self, - tSrS_t2r, - thr_tmem_load, - thr_mma_qk, - batch_idx, - head_idx, - m_block, - n_block, - softmax, - seqlen: SeqlenInfoQK, - aux_tensors=None, - fastdiv_mods=(None, None), - ): - """Apply score modification for SM100 (constant q_idx).""" - # Prepare index tensor with extra partition - cS = cute.make_identity_tensor((self.m_block_size, self.n_block_size)) - cS = cute.domain_offset((m_block * self.m_block_size, n_block * self.n_block_size), cS) - tScS = thr_mma_qk.partition_C(cS) - tScS_t2r = thr_tmem_load.partition_D(tScS) - - # Shared q_idx for all scores - q_idx_logical = tScS_t2r[0][0] - - # For Pack-GQA, compute the logical head index for this tile - if cutlass.const_expr(self.pack_gqa): - # Building up the logical q_head idx: final_q_head = kv_head * qhead_per_kvhead + (q_physical % qhead_per_kvhead) - q_physical = q_idx_logical - q_idx_logical = q_physical // self.qhead_per_kvhead - head_offset = q_physical - q_idx_logical * self.qhead_per_kvhead - head_idx = head_idx * self.qhead_per_kvhead + head_offset - - if cutlass.const_expr(aux_tensors is not None): - seqlen_q_divmod, _ = fastdiv_mods - _, q_idx_logical = divmod(q_idx_logical, seqlen_q_divmod) - - apply_score_mod_inner( - tSrS_t2r, - tScS_t2r, - self.score_mod, - batch_idx, - head_idx, - softmax.softmax_scale, - self.vec_size, - self.qk_acc_dtype, - aux_tensors, - fastdiv_mods, - seqlen_info=seqlen, - constant_q_idx=q_idx_logical, - qhead_per_kvhead=self.qhead_per_kvhead if cutlass.const_expr(self.pack_gqa) else 1, - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/hopper_helpers.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/hopper_helpers.py deleted file mode 100644 index c6a1c301904d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/hopper_helpers.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (c) 2025, Tri Dao. -from typing import Type, Union, Optional -import cutlass -import cutlass.cute as cute -from cutlass import Int32, Float32, Boolean, const_expr -from cutlass.cute.nvgpu import warpgroup -from cutlass.cutlass_dsl import Numeric, dsl_user_op -from cutlass.utils import LayoutEnum -import cutlass.utils.hopper_helpers as sm90_utils_og - - -@cute.jit -def gemm( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - zero_init: cutlass.Constexpr[bool] = False, - wg_wait: cutlass.Constexpr[int] = 0, - # A_in_regs: cutlass.Constexpr[bool] = False, - swap_AB: cutlass.Constexpr[bool] = False, -) -> None: - if const_expr(swap_AB): - gemm(tiled_mma, acc, tCrB, tCrA, zero_init=zero_init, wg_wait=wg_wait, swap_AB=False) - else: - warpgroup.fence() - # We make a new mma_atom since we'll be modifying its attribute (accumulate). - # Otherwise the compiler complains "operand #0 does not dominate this use" - mma_atom = cute.make_mma_atom(tiled_mma.op) - mma_atom.set(warpgroup.Field.ACCUMULATE, not zero_init) - for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])): - cute.gemm(mma_atom, acc, tCrA[None, None, k], tCrB[None, None, k], acc) - mma_atom.set(warpgroup.Field.ACCUMULATE, True) - warpgroup.commit_group() - if const_expr(wg_wait >= 0): - warpgroup.wait_group(wg_wait) - - -def gemm_zero_init( - tiled_mma: cute.TiledMma, - shape: cute.Shape, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - A_idx: Optional[Int32] = None, - B_idx: Optional[Int32] = None, - wg_wait: int = -1, - swap_AB: bool = False, -) -> cute.Tensor: - if const_expr(swap_AB): - return gemm_zero_init( - tiled_mma, shape[::-1], tCrB, tCrA, B_idx, A_idx, wg_wait, swap_AB=False - ) - else: - acc = cute.make_fragment(tiled_mma.partition_shape_C(shape), Float32) - rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx] - rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx] - gemm(tiled_mma, acc, rA, rB, zero_init=True, wg_wait=wg_wait) - return acc - - -def gemm_w_idx( - tiled_mma: cute.TiledMma, - acc: cute.Tensor, - tCrA: cute.Tensor, - tCrB: cute.Tensor, - zero_init: Boolean, - A_idx: Optional[Int32] = None, - B_idx: Optional[Int32] = None, - wg_wait: int = -1, - swap_AB: bool = False, -) -> None: - if const_expr(swap_AB): - gemm_w_idx(tiled_mma, acc, tCrB, tCrA, zero_init, B_idx, A_idx, wg_wait, swap_AB=False) - else: - rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx] - rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx] - gemm(tiled_mma, acc, rA, rB, zero_init=zero_init, wg_wait=wg_wait) - - -@dsl_user_op -def make_smem_layout( - dtype: Type[Numeric], - layout: LayoutEnum, - shape: cute.Shape, - stage: Optional[int] = None, - *, - loc=None, - ip=None, -) -> Union[cute.Layout, cute.ComposedLayout]: - major_mode_size = shape[1] if layout.is_n_major_c() else shape[0] - smem_layout_atom = warpgroup.make_smem_layout_atom( - sm90_utils_og.get_smem_layout_atom(layout, dtype, major_mode_size), - dtype, - ) - order = (1, 0, 2) if const_expr(layout.is_m_major_c()) else (0, 1, 2) - smem_layout_staged = cute.tile_to_shape( - smem_layout_atom, - cute.append(shape, stage) if const_expr(stage is not None) else shape, - order=order if const_expr(stage is not None) else order[:2], - ) - return smem_layout_staged diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/interface.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/interface.py deleted file mode 100644 index 7aa61a5faf2e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/interface.py +++ /dev/null @@ -1,1831 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -# [2025-07-04] Version in Cute-DSL, for Hopper and Blackwell. You'll need install nvidia-cutlass-dsl==4.2.0. - -# Supported features: -# - BF16 & FP16 dtype -# - noncausal & causal attention -# - MHA, GQA, MQA -# - hdim 64, 96, 128. -# - (hdim_qk, hdim_v) = (192, 128) for Blackwell (i.e. DeepSeek shape) -# - varlen -# - sliding window -# - bwd pass for Ampere (will also run on Hopper/Blackwell, but will be slow) - -# Features not supported yet: -# - split (i.e. FlashDecoding) -# - tuned block sizes -# - paged KV -# - append KV to existing KV cache -# - FP8 -# - bwd pass optimized for Hopper/Blackwell - -import math -from functools import lru_cache -from typing import Optional, Tuple, Callable - -import torch - - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .cute_dsl_utils import to_cute_tensor -from .flash_fwd import FlashAttentionForwardSm90 -from .flash_fwd_sm100 import FlashAttentionForwardSm100 -from .flash_bwd_preprocess import FlashAttentionBackwardPreprocess -from .flash_bwd import FlashAttentionBackwardSm80 -from .flash_bwd_sm90 import FlashAttentionBackwardSm90 -from .flash_bwd_sm100 import FlashAttentionBackwardSm100 -from .flash_bwd_postprocess import FlashAttentionBackwardPostprocess -from .flash_fwd_combine import FlashAttentionForwardCombine - -from .block_sparsity import ( - BlockSparseTensorsTorch, - to_cute_block_sparse_tensors, - normalize_block_sparse_tensors, - get_block_sparse_expected_shapes, - get_block_sparse_expected_shapes_bwd, -) - - -@lru_cache(maxsize=None) -def _get_device_capability(): - """Cached device capability check.""" - return torch.cuda.get_device_capability()[0] - - -def maybe_contiguous(x): - return x.contiguous() if x is not None and x.stride(-1) != 1 else x - - -def _validate_tensor(t, name, expected_shape, expected_dtype, expected_device): - assert t.shape == expected_shape, f"{name} shape {t.shape} != expected {expected_shape}" - assert t.dtype == expected_dtype, f"{name} dtype {t.dtype} != expected {expected_dtype}" - assert t.device == expected_device, f"{name} device {t.device} != expected {expected_device}" - assert t.is_cuda, f"{name} must be on CUDA" - - -torch2cute_dtype_map = { - torch.float16: cutlass.Float16, - torch.bfloat16: cutlass.BFloat16, - torch.float32: cutlass.Float32, -} - - -def num_splits_heuristic(total_mblocks, num_SMs, num_n_blocks, max_splits): - # If num_n_blocks is too small, use 1 split. For example, we never split for hdim = 128 and seqlen_k = 512. - if num_n_blocks <= 4: - return 1 - - # NOTE: We should revisit this heuristic after persistence is supported for split KV. - # Sometimes, it's ideal to over-schedule splits for better efficiency. - return min(num_SMs // total_mblocks, max_splits, num_n_blocks) - - -def _flash_attn_fwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - cu_seqlens_q: Optional[torch.Tensor] = None, - cu_seqlens_k: Optional[torch.Tensor] = None, - seqused_q: Optional[torch.Tensor] = None, - seqused_k: Optional[torch.Tensor] = None, - max_seqlen_q: Optional[int] = None, - max_seqlen_k: Optional[int] = None, - page_table: Optional[torch.Tensor] = None, - softmax_scale: Optional[float] = None, - causal: bool = False, - softcap: Optional[float] = None, - window_size_left: Optional[int] = None, - window_size_right: Optional[int] = None, - learnable_sink: Optional[torch.Tensor] = None, - # m_block_size: int = 128, - # n_block_size: int = 64, - # num_threads: int = 128, - m_block_size: int = 128, - n_block_size: int = 128, - num_threads: int = 384, - num_splits: int = 1, - pack_gqa: Optional[bool] = None, - _compute_capability: Optional[int] = None, - score_mod: Optional[Callable] = None, - mask_mod: Optional[Callable] = None, - block_sparse_tensors: Optional[BlockSparseTensorsTorch] = None, - return_lse: bool = False, - out: Optional[torch.Tensor] = None, - lse: Optional[torch.Tensor] = None, - aux_tensors: Optional[list[torch.Tensor]] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward pass for FlashAttention. - - Args: - ... - score_mod: A callable that takes the attention scores and applies a modification. - mask_mod: A callable that takes token position information and selectively masks - block_sparse_tensors: A tuple of tensors used for block sparsity. - return_lse: Whether to return the log softmax of the attention scores. If set to True will always calculate - out: Optional pre-allocated output tensor. If None, will be allocated internally. - lse: Optional pre-allocated log-sum-exp tensor. If None, will be allocated when needed. - aux_tensors: Some score_mods will want to read from global aux_tensors. This is how we thread them through to the inner kernel. - """ - q, k, v = [maybe_contiguous(t) for t in (q, k, v)] - num_head, head_dim = q.shape[-2:] - if cu_seqlens_q is None: - batch_size, seqlen_q = q.shape[:2] - total_q = batch_size * seqlen_q - else: - batch_size = cu_seqlens_q.shape[0] - 1 - seqlen_q = None - total_q = q.shape[0] - if page_table is not None: - assert cu_seqlens_k is None, "page_table is not supported with cu_seqlens_k" - assert page_table.dtype == torch.int32, "page_table must be int32" - assert page_table.stride(-1) == 1, "page_table must be contiguous in the last dimension" - max_num_pages_per_seq = page_table.shape[1] - assert page_table.shape == (batch_size, max_num_pages_per_seq) - num_pages, page_size = k.shape[:2] - seqlen_k = num_pages * page_size - else: - num_pages, page_size = None, None - seqlen_k = k.shape[-3] - num_head_kv = k.shape[-2] - head_dim_v = v.shape[-1] - if cu_seqlens_k is None: - if page_table is None: - assert k.shape == (batch_size, seqlen_k, num_head_kv, head_dim) - assert v.shape == (batch_size, seqlen_k, num_head_kv, head_dim_v) - else: - assert k.shape == (num_pages, page_size, num_head_kv, head_dim) - assert v.shape == (num_pages, page_size, num_head_kv, head_dim_v) - else: - assert k.shape == (seqlen_k, num_head_kv, head_dim) - assert v.shape == (seqlen_k, num_head_kv, head_dim_v) - assert cu_seqlens_k.shape == (batch_size + 1,), ( - "cu_seqlens_k must have shape (batch_size + 1,)" - ) - - if cu_seqlens_q is not None: - assert cu_seqlens_q.shape == (batch_size + 1,), ( - "cu_seqlens_q must have shape (batch_size + 1,)" - ) - assert seqused_q is None or seqused_q.shape == (batch_size,), ( - "seqused_q must have shape (batch_size,)" - ) - assert seqused_k is None or seqused_k.shape == (batch_size,), ( - "seqused_k must have shape (batch_size,)" - ) - assert q.dtype in [torch.float16, torch.bfloat16], "inputs must be float16 or bfloat16" - assert q.dtype == k.dtype == v.dtype, "inputs must have the same dtype" - for t in [cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k]: - if t is not None: - assert t.dtype == torch.int32, ( - "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be int32" - ) - assert t.stride(0) == 1, ( - "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be contiguous" - ) - if learnable_sink is not None: - assert learnable_sink.shape == (num_head,) - assert learnable_sink.dtype == torch.bfloat16, "learnable_sink must be bfloat16" - - assert all( - t is None or t.is_cuda - for t in ( - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - page_table, - learnable_sink, - ) - ), "inputs must be on CUDA device" - assert num_head % num_head_kv == 0, "num_head must be divisible by num_head_kv" - assert head_dim <= 256, "head_dim must be less than or equal to 256" - alignment = 16 // q.element_size() - assert head_dim % alignment == 0, f"head_dim must be divisible by {alignment}" - assert head_dim_v % alignment == 0, f"head_dim_v must be divisible by {alignment}" - if softmax_scale is None: - softmax_scale = 1.0 / math.sqrt(head_dim) - if softcap == 0.0: - softcap = None - qhead_per_kvhead = num_head // num_head_kv - if pack_gqa is None: - pack_gqa = qhead_per_kvhead > 1 - - out_torch_dtype = q.dtype - device = q.device - q_batch_seqlen_shape = (batch_size, seqlen_q) if cu_seqlens_q is None else (total_q,) - lse_shape = (batch_size, num_head, seqlen_q) if cu_seqlens_q is None else (num_head, total_q) - requires_grad = q.requires_grad or k.requires_grad or v.requires_grad - - if out is None: - out = torch.empty( - *q_batch_seqlen_shape, num_head, head_dim_v, dtype=out_torch_dtype, device=device - ) - else: - _validate_tensor( - out, "out", (*q_batch_seqlen_shape, num_head, head_dim_v), out_torch_dtype, device - ) - - if lse is None: - lse = ( - torch.empty(lse_shape, dtype=torch.float32, device=device) - if requires_grad or return_lse - else None - ) - elif lse is not None: - _validate_tensor(lse, "lse", lse_shape, torch.float32, device) - - dtype = torch2cute_dtype_map[q.dtype] - compute_capability = ( - _get_device_capability() if _compute_capability is None else _compute_capability - ) - - assert compute_capability in [9, 10, 11], ( - "Unsupported compute capability. Supported: 9.x, 10.x, 11.x" - ) - - use_block_sparsity = block_sparse_tensors is not None - - if mask_mod is None: - if causal: - window_size_right = 0 - local = window_size_left is not None or window_size_right is not None - if window_size_left is not None or window_size_right is not None: - if window_size_left is None and window_size_right == 0: - causal, local = True, False - window_size_right = None - else: - causal, local = False, True - else: - causal, local = False, False - - current_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - - if compute_capability == 9: # TODO: tune block size according to hdim. - if head_dim == head_dim_v == 128 and not causal and not local and not use_block_sparsity: - n_block_size = 192 - - if compute_capability in [10, 11]: - if pack_gqa and (128 % qhead_per_kvhead != 0): - pack_gqa = False - # TODO: fix GQA + SplitKV + non-varlen - if pack_gqa and num_splits != 1 and cu_seqlens_q is None: - pack_gqa = False - - if max_seqlen_q is None: - max_seqlen_q = seqlen_q if cu_seqlens_q is None else total_q - if max_seqlen_k is None: - max_seqlen_k = seqlen_k - seqlen_q_packgqa = max_seqlen_q * qhead_per_kvhead - if compute_capability == 10: - q_stage = 2 if seqlen_q_packgqa > m_block_size else 1 - else: - q_stage = 1 - - if num_splits < 1: - m_block_size_effective = q_stage * m_block_size - seqlen_k_loaded = ( - max_seqlen_k - if not local - else max(0, min(max_seqlen_k, window_size_right + window_size_left + 1 + m_block_size)) - ) - num_n_blocks = (seqlen_k_loaded + n_block_size - 1) // n_block_size - num_m_blocks = (seqlen_q_packgqa + m_block_size_effective - 1) // m_block_size_effective - total_mblocks = batch_size * num_head_kv * num_m_blocks - num_splits = num_splits_heuristic( - total_mblocks, - torch.cuda.get_device_properties(device).multi_processor_count, - num_n_blocks, - 128, - ) - - is_split_kv = num_splits > 1 - if is_split_kv: - out_partial = torch.empty( - num_splits, - *q_batch_seqlen_shape, - num_head, - head_dim_v, - dtype=torch.float32, - device=device, - ) - lse_partial = torch.empty(num_splits, *lse_shape, dtype=torch.float32, device=device) - - # hash score and mask mods for compile cache - score_mod_hash = utils.hash_callable(score_mod) if score_mod is not None else False - mask_mod_hash = utils.hash_callable(mask_mod) if mask_mod is not None else False - - if softcap is not None: - assert score_mod is None, "softcap and score_mod cannot be used together" - score_mod = utils.create_softcap_scoremod(softcap) - - is_varlen = ( - cu_seqlens_q is not None - or cu_seqlens_k is not None - or seqused_q is not None - or seqused_k is not None - ) - - if mask_mod is not None: - if is_varlen: - raise NotImplementedError( - "mask_mod with aux_tensors is not yet supported for varlen sequences. This will be fixed in a future PR." - ) - - if use_block_sparsity: - if is_varlen: - raise NotImplementedError( - "Block sparsity is not yet supported for varlen sequences. This will be fixed in a future PR." - ) - # NB: pack_gqa requires block sparse head dim == 1 (broadcasted) - if pack_gqa and block_sparse_tensors.mask_block_cnt.shape[1] != 1: - pack_gqa = False - if is_split_kv: - raise NotImplementedError( - "Block sparsity is not yet supported with SplitKV. TODO: partition sparse block lists per split." - ) - - compile_key = ( - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - causal, - score_mod_hash, - mask_mod_hash, - use_block_sparsity, - len(aux_tensors) if aux_tensors is not None else 0, - lse is None, - cu_seqlens_q is None, - cu_seqlens_k is None, - seqused_q is None, - seqused_k is None, - page_table is not None, - window_size_left is not None, - window_size_right is not None, - learnable_sink is not None, - m_block_size, - n_block_size, - q_stage, - num_threads, - is_split_kv, - pack_gqa, - compute_capability, - page_size not in [None, 128], # paged KV non-TMA - ) - if compile_key not in _flash_attn_fwd.compile_cache: - ( - cu_seqlens_q_tensor, - cu_seqlens_k_tensor, - seqused_q_tensor, - seqused_k_tensor, - learnable_sink_tensor, - ) = [ - to_cute_tensor(t, assumed_align=4, leading_dim=0) if t is not None else None - for t in (cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k, learnable_sink) - ] - page_table_tensor = ( - to_cute_tensor(page_table, assumed_align=4, leading_dim=1) - if page_table is not None - else None - ) - q_tensor, k_tensor, v_tensor, o_tensor = [ - to_cute_tensor(t) for t in (q, k, v, out if not is_split_kv else out_partial) - ] - if is_split_kv: - lse_tensor = to_cute_tensor(lse_partial, assumed_align=4) - elif lse is not None: - lse_tensor = to_cute_tensor(lse, assumed_align=4) - else: - lse_tensor = None - - sparse_tensors = None - if block_sparse_tensors is not None: - if seqlen_q is None: - raise ValueError( - "Block sparsity requires fixed-length sequences (seqlen_q must be known)." - ) - expected_count_shape, expected_index_shape = get_block_sparse_expected_shapes( - batch_size, - num_head, - seqlen_q, - seqlen_k, - m_block_size, - n_block_size, - q_stage, - ) - compile_time_normalized = normalize_block_sparse_tensors( - block_sparse_tensors, - expected_count_shape=expected_count_shape, - expected_index_shape=expected_index_shape, - ) - sparse_tensors = to_cute_block_sparse_tensors(compile_time_normalized) - - cute_aux_tensors = None - if aux_tensors is not None: - cute_aux_tensors = [ - to_cute_tensor(buf, assumed_align=None, fully_dynamic=True) for buf in aux_tensors - ] - - if compute_capability == 9: - assert page_table is None, "paged KV not supported on SM 9.0" - assert not is_split_kv, "SplitKV not supported on SM 9.0" - # fa_fwd = FlashAttentionForwardSm80( - fa_fwd = FlashAttentionForwardSm90( - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - is_causal=causal, - is_local=local, - pack_gqa=pack_gqa, - tile_m=m_block_size, - tile_n=n_block_size, - # num_stages=1, - num_stages=2, - num_threads=num_threads, - Q_in_regs=False, - intra_wg_overlap=True, - mma_pv_is_rs=True, - mask_mod=mask_mod, - score_mod=score_mod, - has_aux_tensors=aux_tensors is not None, - ) - elif compute_capability in [10, 11]: - fa_fwd = FlashAttentionForwardSm100( - head_dim, - head_dim_v, - qhead_per_kvhead=qhead_per_kvhead, - is_causal=causal, - is_local=local, - is_split_kv=is_split_kv, - pack_gqa=pack_gqa, - m_block_size=m_block_size, - n_block_size=n_block_size, - q_stage=q_stage, - is_persistent=not causal - and not local - and cu_seqlens_q is None - and seqused_q is None - and not is_split_kv, - score_mod=score_mod, - mask_mod=mask_mod, - has_aux_tensors=aux_tensors is not None, - paged_kv_non_tma=page_size not in [None, 128], - is_varlen_q=cu_seqlens_q is not None or seqused_q is not None, - ) - else: - raise ValueError( - f"Unsupported compute capability: {compute_capability}. Supported: 9.x, 10.x, 11.x" - ) - # TODO: check @can_implement - _flash_attn_fwd.compile_cache[compile_key] = cute.compile( - fa_fwd, - q_tensor, - k_tensor, - v_tensor, - o_tensor, - lse_tensor, - softmax_scale, - current_stream, - cu_seqlens_q_tensor, - cu_seqlens_k_tensor, - seqused_q_tensor, - seqused_k_tensor, - page_table_tensor, - window_size_left, - window_size_right, - learnable_sink_tensor, - sparse_tensors, - cute_aux_tensors, - options="--enable-tvm-ffi", - ) - - # Expand block sparse tensors to match actual head count (may be broadcast from 1) - normalized_block_sparse_tensors = None - if block_sparse_tensors is not None: - expected_count_shape, expected_index_shape = get_block_sparse_expected_shapes( - batch_size, - num_head, - seqlen_q, - seqlen_k, - m_block_size, - n_block_size, - q_stage, - ) - normalized_block_sparse_tensors = normalize_block_sparse_tensors( - block_sparse_tensors, - expected_count_shape=expected_count_shape, - expected_index_shape=expected_index_shape, - ) - _flash_attn_fwd.compile_cache[compile_key]( - q, - k, - v, - out if not is_split_kv else out_partial, - lse_partial if is_split_kv else lse, - softmax_scale, - current_stream, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - page_table, - window_size_left, - window_size_right, - learnable_sink, - normalized_block_sparse_tensors, - aux_tensors, - ) - if is_split_kv: - _flash_attn_fwd_combine( - out_partial, - lse_partial.transpose(-1, -2), - out, - lse.transpose(-1, -2) if lse is not None else None, - cu_seqlens_q, - seqused_q, - ) - return out, lse - - -_flash_attn_fwd.compile_cache = {} - - -def _flash_attn_bwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - out: torch.Tensor, - dout: torch.Tensor, - lse: torch.Tensor, - softmax_scale: Optional[float] = None, - causal: bool = False, - softcap: float = 0.0, - window_size_left: Optional[int] = None, - window_size_right: Optional[int] = None, - m_block_size: int = 64, - n_block_size: int = 128, - num_threads: int = 256, - pack_gqa: bool = False, - num_stages_Q: int = 2, - num_stages_dO: int = 2, - SdP_swapAB: bool = False, - dKV_swapAB: bool = False, - dQ_swapAB: bool = False, - AtomLayoutMSdP: int = 2, - AtomLayoutNdKV: int = 2, - AtomLayoutMdQ: int = 2, - V_in_regs: bool = False, - cu_seqlens_q: Optional[torch.Tensor] = None, - cu_seqlens_k: Optional[torch.Tensor] = None, - seqused_q: Optional[torch.Tensor] = None, - seqused_k: Optional[torch.Tensor] = None, - max_seqlen_q: Optional[int] = None, - max_seqlen_k: Optional[int] = None, - deterministic: bool = False, - dq: Optional[torch.Tensor] = None, - dk: Optional[torch.Tensor] = None, - dv: Optional[torch.Tensor] = None, - score_mod: Optional[Callable] = None, - score_mod_bwd: Optional[Callable] = None, - mask_mod: Optional[Callable] = None, - aux_tensors: Optional[list[torch.Tensor]] = None, - block_sparse_tensors: Optional[BlockSparseTensorsTorch] = None, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - compute_capability = _get_device_capability() - assert compute_capability in [9, 10, 11], ( - "Unsupported compute capability. Supported: 9.x, 10.x, 11.x" - ) - - if compute_capability == 9: - m_block_size = 80 if not causal else 64 - n_block_size = 128 - num_stages_Q = 2 - num_stages_dO = 2 - num_stages_PdS = 2 - SdP_swapAB = True - dKV_swapAB = False - dQ_swapAB = not causal - AtomLayoutMSdP = 1 - AtomLayoutNdKV = 2 - AtomLayoutMdQ = 1 - cluster_size = 1 - assert window_size_left is None and window_size_right is None, ( - "local not supported yet on 9.x" - ) - else: - m_block_size = 128 - n_block_size = 128 - dQ_swapAB = False - dKV_swapAB = False - AtomLayoutMdQ = 1 - AtomLayoutNdKV = 1 - # TODO: support cluster size 2 - cluster_size = 1 - q, k, v, out, dout, lse, cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k = [ - maybe_contiguous(t) - for t in (q, k, v, out, dout, lse, cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k) - ] - num_head, head_dim = q.shape[-2:] - if cu_seqlens_q is None: - batch_size, seqlen_q = q.shape[:2] - total_q = batch_size * seqlen_q - else: - batch_size = cu_seqlens_q.shape[0] - 1 - total_q = q.shape[0] - seqlen_q = max_seqlen_q if max_seqlen_q is not None else total_q - - if cu_seqlens_k is None: - batch_size, seqlen_k = k.shape[:2] - total_k = batch_size * seqlen_k - else: - batch_size = cu_seqlens_k.shape[0] - 1 - total_k = k.shape[0] - seqlen_k = max_seqlen_k if max_seqlen_k is not None else total_k - - num_head_kv = k.shape[-2] - head_dim_v = v.shape[-1] - - if causal: - window_size_right = 0 - local = window_size_left is not None or window_size_right is not None - if local: - if window_size_left is None and window_size_right == 0: - causal, local = True, False - window_size_right = None - else: - causal, local = False, True - - use_block_sparsity = block_sparse_tensors is not None - - # SM90 block-sparse backward: tile_m=64 is the GCD between a m_block_size that fits, - # the base block_m of 128 from forward, and block-sparse size for subtiling. - if compute_capability == 9 and use_block_sparsity: - m_block_size = 64 - # dQ_swapAB tuning: use False when m_block_size=64 (same as causal case) - dQ_swapAB = False - - # NB: this could be derived from the block_sparse_tensors but for now we hardcode it to 2 - subtile_factor = 2 - sparse_block_size_q = subtile_factor * m_block_size - - seqlen_q_rounded = (seqlen_q + m_block_size - 1) // m_block_size * m_block_size - seqlen_k_rounded = (seqlen_k + n_block_size - 1) // n_block_size * n_block_size - - if cu_seqlens_k is None: - assert k.shape == (batch_size, seqlen_k, num_head_kv, head_dim) - assert v.shape == (batch_size, seqlen_k, num_head_kv, head_dim_v) - else: - assert k.shape == (total_k, num_head_kv, head_dim) - assert v.shape == (total_k, num_head_kv, head_dim_v) - assert cu_seqlens_k.shape == (batch_size + 1,), ( - "cu_seqlens_k must have shape (batch_size + 1,)" - ) - - if cu_seqlens_q is not None: - assert cu_seqlens_q.shape == (batch_size + 1,), ( - "cu_seqlens_q must have shape (batch_size + 1,)" - ) - - assert out.shape == (total_q, num_head, head_dim_v) - assert dout.shape == (total_q, num_head, head_dim_v) - assert lse.shape == (num_head, total_q), "lse must have shape (num_head, total_q)" - else: - assert out.shape == (batch_size, seqlen_q, num_head, head_dim_v) - assert dout.shape == (batch_size, seqlen_q, num_head, head_dim_v) - assert lse.shape == (batch_size, num_head, seqlen_q), ( - "lse must have shape (batch_size, num_head, seqlen_q)" - ) - - assert q.dtype in [torch.float16, torch.bfloat16], "inputs must be float16 or bfloat16" - assert q.dtype == k.dtype == v.dtype == out.dtype == dout.dtype, ( - "inputs must have the same dtype" - ) - for t in [cu_seqlens_q, cu_seqlens_k]: - if t is not None: - assert t.dtype == torch.int32, "cu_seqlens_q, cu_seqlens_k must be int32" - assert lse.dtype == torch.float32, "lse must be float32" - assert all( - t is None or t.is_cuda for t in (q, k, v, out, dout, lse, cu_seqlens_q, cu_seqlens_k) - ), "inputs must be on CUDA device" - assert num_head % num_head_kv == 0, "num_head must be divisible by num_head_kv" - assert head_dim <= 256, "head_dim must be less than or equal to 256" - alignment = 16 // q.element_size() - assert head_dim % alignment == 0, f"head_dim must be divisible by {alignment}" - assert head_dim_v % alignment == 0, f"head_dim_v must be divisible by {alignment}" - if softmax_scale is None: - softmax_scale = 1.0 / math.sqrt(head_dim) - qhead_per_kvhead = num_head // num_head_kv - if pack_gqa is None: - pack_gqa = qhead_per_kvhead > 1 - # pack_gqa backward not yet supported in bwd - pack_gqa = False - if compute_capability not in [10, 11]: - assert deterministic is False, "bwd deterministic only supported for sm100/sm110 for now" - - if score_mod is not None: - assert score_mod_bwd is not None, "score_mod_bwd is required when score_mod is provided" - assert softcap == 0.0, ( - "softcap and score_mod are mutually exclusive (different log2 scaling)" - ) - assert cu_seqlens_q is None and cu_seqlens_k is None, ( - "varlen + score_mod not supported in bwd yet" - ) - - device = q.device - out_torch_dtype = q.dtype - - if dq is None: - dq = torch.empty_like(q) - else: - _validate_tensor(dq, "dq", q.shape, out_torch_dtype, device) - - if dk is None: - dk = torch.empty_like(k) - else: - _validate_tensor(dk, "dk", k.shape, out_torch_dtype, device) - - if dv is None: - dv = torch.empty_like(v) - else: - _validate_tensor(dv, "dv", v.shape, out_torch_dtype, device) - - head_dim_rounded = (head_dim + 32 - 1) // 32 * 32 - - if cu_seqlens_q is None: - dq_accum = torch.empty( - batch_size, - num_head, - seqlen_q_rounded * head_dim_rounded, - dtype=torch.float32, - device=device, - ) - dpsum = torch.empty( - batch_size, num_head, seqlen_q_rounded, dtype=torch.float32, device=device - ) - lse_log2 = torch.empty( - batch_size, num_head, seqlen_q_rounded, dtype=torch.float32, device=device - ) - else: - total_q_rounded_padded = ( - (total_q + cu_seqlens_q.shape[0] * m_block_size - 1) // m_block_size * m_block_size - ) - dq_accum = torch.empty( - num_head, total_q_rounded_padded * head_dim_rounded, dtype=torch.float32, device=device - ) - dpsum = torch.empty(num_head, total_q_rounded_padded, dtype=torch.float32, device=device) - lse_log2 = torch.empty(num_head, total_q_rounded_padded, dtype=torch.float32, device=device) - - dKV_postprocess = qhead_per_kvhead > 1 - if dKV_postprocess: - head_dim_v_rounded = (head_dim_v + 32 - 1) // 32 * 32 - if cu_seqlens_k is None: - num_n_blocks = seqlen_k_rounded // n_block_size - if cluster_size == 2 and num_n_blocks % cluster_size != 0: - seqlen_k_rounded = seqlen_k_rounded + n_block_size - dk_accum = torch.zeros( - batch_size, - num_head_kv, - seqlen_k_rounded * head_dim_rounded, - dtype=torch.float32, - device=device, - ) - dv_accum = torch.zeros( - batch_size, - num_head_kv, - seqlen_k_rounded * head_dim_v_rounded, - dtype=torch.float32, - device=device, - ) - else: - total_k_rounded_padded = ( - (total_k + cu_seqlens_k.shape[0] * n_block_size - 1) // n_block_size * n_block_size - ) - num_n_blocks = total_k_rounded_padded // n_block_size - if cluster_size == 2 and num_n_blocks % cluster_size != 0: - total_k_rounded_padded = total_k_rounded_padded + n_block_size - dk_accum = torch.zeros( - num_head_kv, - total_k_rounded_padded * head_dim_rounded, - dtype=torch.float32, - device=device, - ) - dv_accum = torch.zeros( - num_head_kv, - total_k_rounded_padded * head_dim_v_rounded, - dtype=torch.float32, - device=device, - ) - - dtype = torch2cute_dtype_map[q.dtype] - current_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - - if deterministic: - dQ_semaphore = torch.zeros( - batch_size, - num_head, - seqlen_q_rounded // m_block_size, - 1, - dtype=torch.int32, - device="cuda", - ) - else: - dQ_semaphore = None - - if deterministic and qhead_per_kvhead > 1: - dK_semaphore = torch.zeros( - batch_size, - num_head_kv, - seqlen_k_rounded // n_block_size, - 2, - dtype=torch.int32, - device="cuda", - ) - dV_semaphore = torch.zeros( - batch_size, - num_head_kv, - seqlen_k_rounded // n_block_size, - 2, - dtype=torch.int32, - device="cuda", - ) - else: - dK_semaphore = None - dV_semaphore = None - - # Preprocess kernel: compute (o * dout).sum(dim=-1), lse * log2_e, and zero out dq_accum. - compile_key_pre = ( - compute_capability, - dtype, - head_dim_v, - m_block_size, - num_threads, - cu_seqlens_q is None, - seqused_q is None, - ) - if compile_key_pre not in _flash_attn_bwd.compile_cache_pre: - o_tensor, do_tensor = [to_cute_tensor(t) for t in (out, dout)] - dq_accum_tensor, dpsum_tensor, lse_log2_tensor = [ - to_cute_tensor(t) for t in (dq_accum, dpsum, lse_log2) - ] - lse_tensor = to_cute_tensor(lse, assumed_align=4) - cu_seqlens_q_tensor, seqused_q_tensor = [ - to_cute_tensor(t, assumed_align=4) if t is not None else None - for t in (cu_seqlens_q, seqused_q) - ] - arch = compute_capability * 10 - fa_bwd_pre = FlashAttentionBackwardPreprocess( - dtype, - head_dim_v, - arch, - m_block_size, - num_threads=num_threads, - ) - # TODO: check @can_implement - _flash_attn_bwd.compile_cache_pre[compile_key_pre] = cute.compile( - fa_bwd_pre, - o_tensor, - do_tensor, - dpsum_tensor, - lse_tensor, - lse_log2_tensor, - dq_accum_tensor, - cu_seqlens_q_tensor, - seqused_q_tensor, - current_stream, - options="--enable-tvm-ffi", - ) - _flash_attn_bwd.compile_cache_pre[compile_key_pre]( - out, - dout, - dpsum, - lse, - lse_log2, - dq_accum, - cu_seqlens_q, - seqused_q, - current_stream, - ) - - # NB num_threads application for 3 kernels - # There are pre, main, post processing kernels, currenlty num_threads is only actually - # used for the pre proc, and then we hard code to 384 for the main and post proc, and we do - # before cache key gen - num_threads = 384 - - # Backward kernel: compute dk, dv, dq_accum. - score_mod_hash = utils.hash_callable(score_mod) if score_mod else False - score_mod_bwd_hash = utils.hash_callable(score_mod_bwd) if score_mod_bwd else False - mask_mod_hash = utils.hash_callable(mask_mod) if mask_mod else False - num_aux_tensors = len(aux_tensors) if aux_tensors else 0 - cute_aux_tensors = None - if aux_tensors is not None: - cute_aux_tensors = [ - to_cute_tensor(buf, assumed_align=None, fully_dynamic=True) for buf in aux_tensors - ] - - if compute_capability == 9: - compile_key = ( - compute_capability, - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - causal, - softcap != 0.0, - m_block_size, - n_block_size, - num_threads, - pack_gqa, - num_stages_Q, - num_stages_dO, - SdP_swapAB, - dKV_swapAB, - dQ_swapAB, - AtomLayoutMSdP, - AtomLayoutNdKV, - AtomLayoutMdQ, - V_in_regs, - cu_seqlens_q is None, - cu_seqlens_k is None, - seqused_q is None, - seqused_k is None, - score_mod_hash, - score_mod_bwd_hash, - mask_mod_hash, - num_aux_tensors, - use_block_sparsity, - ) - else: - compile_key = ( - compute_capability, - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - causal, - window_size_left is not None, - window_size_right is not None, - softcap != 0.0, - m_block_size, - n_block_size, - num_threads, - pack_gqa, - cluster_size, - deterministic, - score_mod_hash, - score_mod_bwd_hash, - mask_mod_hash, - num_aux_tensors, - use_block_sparsity, - cu_seqlens_q is None, - cu_seqlens_k is None, - seqused_q is None, - seqused_k is None, - ) - if compile_key not in _flash_attn_bwd.compile_cache: - q_tensor, k_tensor, v_tensor, do_tensor, dq_tensor, dk_tensor, dv_tensor = [ - to_cute_tensor(t) for t in (q, k, v, dout, dq, dk, dv) - ] - dq_accum_tensor, dpsum_tensor, lse_log2_tensor = [ - to_cute_tensor(t) for t in (dq_accum, dpsum, lse_log2) - ] - if dKV_postprocess: - dk_accum_tensor, dv_accum_tensor = [to_cute_tensor(t) for t in (dk_accum, dv_accum)] - cu_seqlens_q_tensor, cu_seqlens_k_tensor, seqused_q_tensor, seqused_k_tensor = [ - to_cute_tensor(t, assumed_align=4) if t is not None else None - for t in (cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k) - ] - dQ_semaphore_tensor, dK_semaphore_tensor, dV_semaphore_tensor = [ - utils.convert_from_dlpack_leading_static( - t.detach(), leading_dim=3, alignment=4, stride_order=t.dim_order() - ) - if t is not None - else None - for t in (dQ_semaphore, dK_semaphore, dV_semaphore) - ] - fa_bwd_sm80 = FlashAttentionBackwardSm80( - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - m_block_size, - n_block_size, - num_stages_Q, - num_stages_dO, - num_threads, - pack_gqa, - causal, - SdP_swapAB, - dKV_swapAB, - dQ_swapAB, - AtomLayoutMSdP, - AtomLayoutNdKV, - AtomLayoutMdQ, - V_in_regs=V_in_regs, - ) - if compute_capability == 9: - fa_bwd_obj = FlashAttentionBackwardSm90( - dtype, - head_dim, - head_dim_v, - qhead_per_kvhead, - causal, - m_block_size, - n_block_size, - num_stages_Q, - num_stages_dO, - num_stages_PdS, - SdP_swapAB, - dKV_swapAB, - dQ_swapAB, - AtomLayoutMSdP, - AtomLayoutNdKV, - AtomLayoutMdQ, - num_threads, - V_in_regs=V_in_regs, - score_mod=score_mod, - score_mod_bwd=score_mod_bwd, - mask_mod=mask_mod, - has_aux_tensors=aux_tensors is not None, - subtile_factor=subtile_factor, - ) - else: - fa_bwd_obj = FlashAttentionBackwardSm100( - head_dim, - head_dim_v, - is_causal=causal, - is_local=local, - qhead_per_kvhead=qhead_per_kvhead, - # tile_m=m_block_size, - # tile_n=n_block_size, - cluster_size=cluster_size, - # cluster_size=1, - deterministic=deterministic, - score_mod=score_mod, - score_mod_bwd=score_mod_bwd, - mask_mod=mask_mod, - has_aux_tensors=aux_tensors is not None, - subtile_factor=subtile_factor, - ) - - # Block sparse tensors for backward use Q-direction indexing (transposed from forward). - # sparse_block_size_q = subtile_factor * tile_m matches BlockMask granularity. - sparse_tensors_compile = None - if block_sparse_tensors is not None: - expected_count_shape, expected_index_shape = get_block_sparse_expected_shapes_bwd( - batch_size, - num_head, - seqlen_q, - seqlen_k, - m_block_size, - n_block_size, - subtile_factor, - ) - compile_time_normalized = normalize_block_sparse_tensors( - block_sparse_tensors, - expected_count_shape=expected_count_shape, - expected_index_shape=expected_index_shape, - context="_flash_attn_bwd", - hint=lambda: ( - f"Backward expects Q-direction block-sparse tensors (q_mask_cnt/q_mask_idx, and optionally full_q_cnt/full_q_idx). " - f"Regenerate the backward BlockMask with BLOCK_SIZE=({sparse_block_size_q}, {n_block_size}) " - f"(sparse_block_size_q={sparse_block_size_q})." - ), - ) - sparse_tensors_compile = to_cute_block_sparse_tensors(compile_time_normalized) - - # TODO: check @can_implement - _flash_attn_bwd.compile_cache[compile_key] = cute.compile( - fa_bwd_obj, - q_tensor, - k_tensor, - v_tensor, - do_tensor, - lse_log2_tensor, - dpsum_tensor, - dq_accum_tensor, - dk_tensor if not dKV_postprocess else dk_accum_tensor, - dv_tensor if not dKV_postprocess else dv_accum_tensor, - softmax_scale, - current_stream, - cu_seqlens_q_tensor, - cu_seqlens_k_tensor, - seqused_q_tensor, - seqused_k_tensor, - None, # softcap - not yet supported in backward - window_size_left, - window_size_right, - dQ_semaphore_tensor, - dK_semaphore_tensor, - dV_semaphore_tensor, - cute_aux_tensors, - sparse_tensors_compile, - options="--enable-tvm-ffi", - ) - # Runtime normalization of block sparse tensors for both SM90 and SM100 - normalized_block_sparse_tensors = None - if block_sparse_tensors is not None: - expected_count_shape, expected_index_shape = get_block_sparse_expected_shapes_bwd( - batch_size, - num_head, - seqlen_q, - seqlen_k, - m_block_size, - n_block_size, - subtile_factor, - ) - normalized_block_sparse_tensors = normalize_block_sparse_tensors( - block_sparse_tensors, - expected_count_shape=expected_count_shape, - expected_index_shape=expected_index_shape, - context="_flash_attn_bwd", - hint=lambda: ( - f"Backward expects Q-direction block-sparse tensors (q_mask_cnt/q_mask_idx, and optionally full_q_cnt/full_q_idx). " - f"Regenerate the backward BlockMask with BLOCK_SIZE=({sparse_block_size_q}, {n_block_size}) " - f"(sparse_block_size_q={sparse_block_size_q})." - ), - ) - - _flash_attn_bwd.compile_cache[compile_key]( - q, - k, - v, - dout, - lse_log2, - dpsum, - dq_accum, - dk if not dKV_postprocess else dk_accum, - dv if not dKV_postprocess else dv_accum, - softmax_scale, - current_stream, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - None, # softcap - not yet supported in backward - window_size_left, - window_size_right, - dQ_semaphore, - dK_semaphore, - dV_semaphore, - aux_tensors, - normalized_block_sparse_tensors, - ) - - num_threads = 256 if compute_capability == 9 else 128 - arch = compute_capability * 10 - # Postprocess kernel: convert dq_accum from float32 to dq in bf16/fp16 - compile_key_post = ( - compute_capability, - dtype, - head_dim, - m_block_size, - num_threads, - AtomLayoutMdQ, - dQ_swapAB, - cu_seqlens_q is None, - seqused_q is None, - ) - if compile_key_post not in _flash_attn_bwd.compile_cache_post: - dq_accum_tensor = to_cute_tensor(dq_accum) - dq_tensor = to_cute_tensor(dq) - cu_seqlens_q_tensor, seqused_q_tensor = [ - to_cute_tensor(t, assumed_align=4) if t is not None else None - for t in (cu_seqlens_q, seqused_q) - ] - fa_bwd_post = FlashAttentionBackwardPostprocess( - dtype, head_dim, arch, m_block_size, num_threads, AtomLayoutMdQ, dQ_swapAB - ) - # TODO: check @can_implement - _flash_attn_bwd.compile_cache_post[compile_key_post] = cute.compile( - fa_bwd_post, - dq_accum_tensor, - dq_tensor, - softmax_scale, - cu_seqlens_q_tensor, - seqused_q_tensor, - current_stream, - options="--enable-tvm-ffi", - ) - _flash_attn_bwd.compile_cache_post[compile_key_post]( - dq_accum, - dq, - softmax_scale, - cu_seqlens_q, - seqused_q, - current_stream, - ) - - if dKV_postprocess: - # Postprocess kernel: convert dk_accum & dv_accum from float32 to bf16/fp16 - compile_key_post = ( - compute_capability, - dtype, - head_dim, - n_block_size, - num_threads, - AtomLayoutNdKV, - dKV_swapAB, - cu_seqlens_k is None, - seqused_k is None, - ) - if compile_key_post not in _flash_attn_bwd.compile_cache_post: - dk_accum_tensor = to_cute_tensor(dk_accum) - dk_tensor = to_cute_tensor(dk) - cu_seqlens_k_tensor, seqused_k_tensor = [ - to_cute_tensor(t, assumed_align=4) if t is not None else None - for t in (cu_seqlens_k, seqused_k) - ] - arch = compute_capability * 10 - fa_bwd_post = FlashAttentionBackwardPostprocess( - dtype, head_dim, arch, n_block_size, num_threads, AtomLayoutNdKV, dKV_swapAB - ) - # TODO: check @can_implement - _flash_attn_bwd.compile_cache_post[compile_key_post] = cute.compile( - fa_bwd_post, - dk_accum_tensor, - dk_tensor, - softmax_scale, - cu_seqlens_k_tensor, - seqused_k_tensor, - current_stream, - options="--enable-tvm-ffi", - ) - _flash_attn_bwd.compile_cache_post[compile_key_post]( - dk_accum, - dk, - softmax_scale, - cu_seqlens_k, - seqused_k, - current_stream, - ) - compile_key_post = ( - compute_capability, - dtype, - head_dim_v, - n_block_size, - num_threads, - AtomLayoutNdKV, - dKV_swapAB, - cu_seqlens_k is None, - seqused_k is None, - ) - if compile_key_post not in _flash_attn_bwd.compile_cache_post: - dv_accum_tensor = to_cute_tensor(dv_accum) - dv_tensor = to_cute_tensor(dv) - cu_seqlens_k_tensor, seqused_k_tensor = [ - to_cute_tensor(t, assumed_align=4) if t is not None else None - for t in (cu_seqlens_k, seqused_k) - ] - arch = compute_capability * 10 - fa_bwd_post = FlashAttentionBackwardPostprocess( - dtype, head_dim_v, arch, n_block_size, num_threads, AtomLayoutNdKV, dKV_swapAB - ) - # TODO: check @can_implement - _flash_attn_bwd.compile_cache_post[compile_key_post] = cute.compile( - fa_bwd_post, - dv_accum_tensor, - dv_tensor, - cutlass.Float32(1.0), - cu_seqlens_k_tensor, - seqused_k_tensor, - current_stream, - options="--enable-tvm-ffi", - ) - _flash_attn_bwd.compile_cache_post[compile_key_post]( - dv_accum, - dv, - 1.0, - cu_seqlens_k, - seqused_k, - current_stream, - ) - - return dq, dk, dv - - -_flash_attn_bwd.compile_cache_pre = {} -_flash_attn_bwd.compile_cache = {} -_flash_attn_bwd.compile_cache_post = {} - - -class FlashAttnFunc(torch.autograd.Function): - @staticmethod - def forward( - ctx, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - softmax_scale: Optional[float] = None, - causal: bool = False, - window_size: Tuple[Optional[int], Optional[int]] = (None, None), - learnable_sink: Optional[torch.Tensor] = None, - softcap: float = 0.0, - num_splits: int = 1, - pack_gqa: Optional[bool] = None, - deterministic: bool = False, - mask_mod: Optional[Callable] = None, - full_block_cnt: Optional[torch.Tensor] = None, - full_block_idx: Optional[torch.Tensor] = None, - mask_block_cnt: Optional[torch.Tensor] = None, - mask_block_idx: Optional[torch.Tensor] = None, - ): - # Only create block sparse tensors if at least one block sparse parameter is provided - block_sparse_tensors = None - if any( - t is not None for t in [full_block_cnt, full_block_idx, mask_block_cnt, mask_block_idx] - ): - block_sparse_tensors = BlockSparseTensorsTorch( - full_block_cnt=full_block_cnt, - full_block_idx=full_block_idx, - mask_block_cnt=mask_block_cnt, - mask_block_idx=mask_block_idx, - ) - out, lse = _flash_attn_fwd( - q, - k, - v, - softmax_scale=softmax_scale, - causal=causal, - window_size_left=window_size[0], - window_size_right=window_size[1], - learnable_sink=learnable_sink, - softcap=softcap, - num_splits=num_splits, - pack_gqa=pack_gqa, - mask_mod=mask_mod, - block_sparse_tensors=block_sparse_tensors, - ) - ctx.save_for_backward(q, k, v, out, lse) - ctx.softmax_scale = softmax_scale - ctx.causal = causal - ctx.window_size = window_size - ctx.softcap = softcap - ctx.deterministic = deterministic - return out, lse - - @staticmethod - def backward(ctx, dout, *args): - q, k, v, out, lse = ctx.saved_tensors - dq, dk, dv = _flash_attn_bwd( - q, - k, - v, - out, - dout, - lse, - ctx.softmax_scale, - ctx.causal, - ctx.softcap, - window_size_left=ctx.window_size[0], - window_size_right=ctx.window_size[1], - deterministic=ctx.deterministic, - ) - return dq, dk, dv, *((None,) * 20) # Extra Nones is fine - - -class FlashAttnVarlenFunc(torch.autograd.Function): - @staticmethod - def forward( - ctx, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - cu_seqlens_q: Optional[torch.Tensor], - cu_seqlens_k: Optional[torch.Tensor], - seqused_q: Optional[torch.Tensor] = None, - seqused_k: Optional[torch.Tensor] = None, - max_seqlen_q: Optional[int] = None, - max_seqlen_k: Optional[int] = None, - page_table: Optional[torch.Tensor] = None, - softmax_scale: Optional[float] = None, - causal: bool = False, - window_size: Tuple[Optional[int], Optional[int]] = (None, None), - learnable_sink: Optional[torch.Tensor] = None, - softcap: float = 0.0, - num_splits: int = 1, - pack_gqa: Optional[bool] = None, - deterministic: bool = False, - score_mod: Optional[Callable] = None, - aux_tensors: Optional[list] = None, - ): - out, lse = _flash_attn_fwd( - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - max_seqlen_q=max_seqlen_q, - max_seqlen_k=max_seqlen_k, - page_table=page_table, - softmax_scale=softmax_scale, - causal=causal, - window_size_left=window_size[0], - window_size_right=window_size[1], - learnable_sink=learnable_sink, - softcap=softcap, - num_splits=num_splits, - pack_gqa=pack_gqa, - score_mod=score_mod, - aux_tensors=aux_tensors, - ) - ctx.save_for_backward(q, k, v, out, lse, cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k) - ctx.softmax_scale = softmax_scale - ctx.causal = causal - ctx.window_size = window_size - ctx.softcap = softcap - ctx.deterministic = deterministic - ctx.max_seqlen_q = max_seqlen_q - ctx.max_seqlen_k = max_seqlen_k - return out, lse - - @staticmethod - def backward(ctx, dout, *args): - q, k, v, out, lse, cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k = ctx.saved_tensors - assert ctx.softcap == 0.0 - dq, dk, dv = _flash_attn_bwd( - q, - k, - v, - out, - dout, - lse, - ctx.softmax_scale, - ctx.causal, - ctx.softcap, - window_size_left=ctx.window_size[0], - window_size_right=ctx.window_size[1], - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - seqused_q=seqused_q, - seqused_k=seqused_k, - max_seqlen_q=ctx.max_seqlen_q, - max_seqlen_k=ctx.max_seqlen_k, - deterministic=ctx.deterministic, - ) - - return dq, dk, dv, *((None,) * 20) - - -def flash_attn_func( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - softmax_scale: Optional[float] = None, - causal: bool = False, - window_size: Tuple[Optional[int], Optional[int]] = (None, None), - learnable_sink: Optional[torch.Tensor] = None, - softcap: float = 0.0, - num_splits: int = 1, - pack_gqa: Optional[bool] = None, - deterministic: bool = False, - mask_mod: Optional[Callable] = None, - full_block_cnt: Optional[torch.Tensor] = None, - full_block_idx: Optional[torch.Tensor] = None, - mask_block_cnt: Optional[torch.Tensor] = None, - mask_block_idx: Optional[torch.Tensor] = None, -): - return FlashAttnFunc.apply( - q, - k, - v, - softmax_scale, - causal, - window_size, - learnable_sink, - softcap, - num_splits, - pack_gqa, - deterministic, - mask_mod, - full_block_cnt, - full_block_idx, - mask_block_cnt, - mask_block_idx, - ) - - -def flash_attn_varlen_func( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - cu_seqlens_q: Optional[torch.Tensor] = None, - cu_seqlens_k: Optional[torch.Tensor] = None, - max_seqlen_q: Optional[int] = None, - max_seqlen_k: Optional[int] = None, - seqused_q: Optional[torch.Tensor] = None, - seqused_k: Optional[torch.Tensor] = None, - page_table: Optional[torch.Tensor] = None, - softmax_scale: Optional[float] = None, - causal: bool = False, - window_size: Tuple[Optional[int], Optional[int]] = (None, None), - learnable_sink: Optional[torch.Tensor] = None, - softcap: float = 0.0, - num_splits: int = 1, - pack_gqa: Optional[bool] = None, - deterministic: bool = False, - score_mod: Optional[Callable] = None, - aux_tensors: Optional[list] = None, -): - return FlashAttnVarlenFunc.apply( - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - max_seqlen_q, - max_seqlen_k, - page_table, - softmax_scale, - causal, - window_size, - learnable_sink, - softcap, - num_splits, - pack_gqa, - deterministic, - score_mod, - aux_tensors, - ) - - -def _flash_attn_fwd_combine( - out_partial: torch.Tensor, - lse_partial: torch.Tensor, - out: torch.Tensor, - lse: Optional[torch.Tensor] = None, - cu_seqlens: Optional[torch.Tensor] = None, - seqused: Optional[torch.Tensor] = None, - num_splits_dynamic_ptr: Optional[torch.Tensor] = None, - semaphore_to_reset: Optional[torch.Tensor] = None, -) -> None: - """Forward combine kernel for split attention computation. - - Combines partial outputs and log-sum-exp values from multiple splits - of attention computation into final outputs. - - Args: - out_partial: Partial outputs tensor (num_splits, batch, seqlen, nheads, headdim) or - (num_splits, total_q, nheads, headdim) if there's cu_seqlens - lse_partial: Partial LSE tensor (num_splits, batch, seqlen, nheads) or - (num_splits, total_q, nheads) if there's cu_seqlens - out: Output tensor (batch, seqlen, nheads, headdim) or (total_q, nheads, headdim) if there's cu_seqlens - lse: Output LSE tensor (batch, seqlen, nheads) or (total_q, nheads) if there's cu_seqlens. - cu_seqlens: Cumulative sequence lengths for variable length sequences - seqused: Used sequence lengths for each batch - num_splits_dynamic_ptr: Dynamic number of splits per batch - semaphore_to_reset: Semaphore for synchronization - k_block_size: Block size for head dimension - - Returns: - None - """ - # Input validation - assert out_partial.dim() in [4, 5], "out_partial must have 4 or 5 dimensions" - assert lse_partial.dim() in [3, 4], "lse_partial must have 3 or 4 dimensions" - assert out_partial.dtype in [torch.float16, torch.bfloat16, torch.float32], ( - "out_partial must be fp16, bf16, or fp32" - ) - assert lse_partial.dtype == torch.float32, "lse_partial must be fp32" - assert out_partial.is_cuda and lse_partial.is_cuda, "tensors must be on CUDA device" - assert out_partial.stride(-1) == 1, "out_partial must be contiguous in the last dimension" - assert lse_partial.stride(-2) == 1, "lse_partial must be contiguous in the seqlen dimension" - assert lse_partial.shape == out_partial.shape[:-1] - - # Determine if this is variable length based on dimensions - is_varlen = out_partial.dim() == 4 - - # Validate output tensor shapes and types - assert out.shape == out_partial.shape[1:], "out shape mismatch" - if lse is not None: - assert lse.shape == lse_partial.shape[1:], "lse shape mismatch" - assert lse.dtype == torch.float32, "lse must be fp32" - - # Validate optional tensors - for t, name in [ - (cu_seqlens, "cu_seqlens"), - (seqused, "seqused"), - (num_splits_dynamic_ptr, "num_splits_dynamic_ptr"), - ]: - if t is not None: - assert t.dtype == torch.int32, f"{name} must be int32" - assert t.is_cuda, f"{name} must be on CUDA device" - assert t.is_contiguous(), f"{name} must be contiguous" - - head_dim = out_partial.shape[-1] - num_splits = out_partial.shape[0] - assert num_splits <= 256 - # If hdim is 96 or 192, it's faster to round them to 128 or 256 respectively - # so that kBlockM is smaller and we have more parallelism. - k_block_size = 64 if head_dim <= 64 else 128 - # We want kBlockM to be as small as possible to maximize parallelism. - # E.g., if hdim is 64, we want kBlockM to be 16 so that we can use 256 threads, each reading 4 elements (floats). - m_block_size = 8 if k_block_size % 128 == 0 else (16 if k_block_size % 64 == 0 else 32) - log_max_splits = max(math.ceil(math.log2(num_splits)), 4) - if m_block_size == 8: - # If kBlockM == 8 then the minimum number of splits is 32. - # TODO: we can deal w this by using 128 threads instead - log_max_splits = max(log_max_splits, 5) - - current_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - - # Create combine kernel configuration - dtype = torch2cute_dtype_map[out.dtype] - dtype_partial = torch2cute_dtype_map[out_partial.dtype] - - compile_key = ( - dtype, - dtype_partial, - head_dim, - m_block_size, - k_block_size, - log_max_splits, - cu_seqlens is not None, - seqused is not None, - lse is not None, - ) - - if compile_key not in _flash_attn_fwd_combine.compile_cache: - out_partial_tensor = to_cute_tensor(out_partial, leading_dim=4 if not is_varlen else 3) - lse_partial_tensor = to_cute_tensor( - lse_partial, assumed_align=4, leading_dim=lse_partial.ndim - 2 - ) - out_tensor = to_cute_tensor(out, leading_dim=3 if not is_varlen else 2) - lse_tensor = ( - to_cute_tensor(lse, assumed_align=4, leading_dim=lse.ndim - 2) - if lse is not None - else None - ) - - optional_tensors = [ - to_cute_tensor(t, assumed_align=4, leading_dim=0) if t is not None else None - for t in (cu_seqlens, seqused, num_splits_dynamic_ptr, semaphore_to_reset) - ] - cu_seqlens_tensor, seqused_tensor, num_splits_dynamic_tensor, semaphore_tensor = ( - optional_tensors - ) - fa_combine = FlashAttentionForwardCombine( - dtype=dtype, - dtype_partial=dtype_partial, - head_dim=head_dim, - m_block_size=m_block_size, - k_block_size=k_block_size, - log_max_splits=log_max_splits, - ) - - # Check if implementation is supported - if not fa_combine.can_implement( - dtype, - dtype_partial, - head_dim, - m_block_size, - k_block_size, - log_max_splits, - num_threads=256, - ): - raise RuntimeError( - "FlashAttention combine kernel cannot be implemented with given parameters" - ) - - _flash_attn_fwd_combine.compile_cache[compile_key] = cute.compile( - fa_combine, - out_partial_tensor, - lse_partial_tensor, - out_tensor, - lse_tensor, - cu_seqlens_tensor, - seqused_tensor, - num_splits_dynamic_tensor, - semaphore_tensor, - current_stream, - options="--enable-tvm-ffi", - ) - _flash_attn_fwd_combine.compile_cache[compile_key]( - out_partial, - lse_partial, - out, - lse, - cu_seqlens, - seqused, - num_splits_dynamic_ptr, - semaphore_to_reset, - current_stream, - ) - - -_flash_attn_fwd_combine.compile_cache = {} - - -def flash_attn_combine( - out_partial: torch.Tensor, - lse_partial: torch.Tensor, - out: Optional[torch.Tensor] = None, - out_dtype: Optional[torch.dtype] = None, - cu_seqlens: Optional[torch.Tensor] = None, - seqused: Optional[torch.Tensor] = None, - return_lse: bool = True, -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Flash Attention combine function for split attention computation. - - Combines partial outputs and log-sum-exp values from multiple splits - of attention computation into final outputs. This is the main user-facing - interface for the combine kernel. - - Args: - out_partial: Partial outputs tensor with shape: - - (num_splits, batch_size, seqlen, num_heads, head_size) for regular batched input - - (num_splits, total_q, num_heads, head_size) for variable length input - lse_partial: Partial LSE tensor with shape: - - (num_splits, batch_size, seqlen, num_heads) for regular batched input - - (num_splits, total_q, num_heads) for variable length input - out: Optional output tensor. If None, will be created automatically. - out_dtype: Optional output dtype. If None, will use fp16/bf16 based on input. - cu_seqlens: Cumulative sequence lengths for variable length sequences - seqused: Used sequence lengths for each batch - return_lse: Whether to return the combined LSE tensor. Default is True. - - Returns: - Tuple of (out, lse) where: - - out: Combined output tensor with shape (batch_size, seqlen, num_heads, head_size) - or (total_q, num_heads, head_size) for varlen - - lse: Combined log-sum-exp tensor with shape (batch_size, seqlen, num_heads) - or (total_q, num_heads) for varlen. None if return_lse=False - - Note: - This function expects the input tensors to be in the format produced by - split attention computation, where the first dimension is num_splits. - The permuting from user format to kernel format is now done inside the kernel. - """ - # Input validation - assert out_partial.dim() in [4, 5], "out_partial must have 4 or 5 dimensions" - assert lse_partial.dim() in [3, 4], "lse_partial must have 3 or 4 dimensions" - assert out_partial.dtype == torch.float32, "out_partial must be fp32 (from accumulation)" - assert lse_partial.dtype == torch.float32, "lse_partial must be fp32" - - # Determine if this is variable length based on dimensions - is_varlen = out_partial.dim() == 4 - - if is_varlen: - # Variable length: (num_splits, total_q, num_heads, head_size) - num_splits, total_q, num_heads, head_size = out_partial.shape - assert lse_partial.shape == (num_splits, total_q, num_heads), ( - "lse_partial shape mismatch for varlen" - ) - batch_size = 1 # Treat as single batch for varlen - seqlen = total_q - else: - # Regular batched: (num_splits, batch_size, seqlen, num_heads, head_size) - num_splits, batch_size, seqlen, num_heads, head_size = out_partial.shape - assert lse_partial.shape == (num_splits, batch_size, seqlen, num_heads), ( - "lse_partial shape mismatch" - ) - - # Determine output dtype - if out_dtype is None: - out_dtype = out_partial.dtype - - # Create output if not provided - device = out_partial.device - if out is None: - if is_varlen: - out = torch.empty(total_q, num_heads, head_size, dtype=out_dtype, device=device) - else: - out = torch.empty( - batch_size, seqlen, num_heads, head_size, dtype=out_dtype, device=device - ) - - # Create lse output only if requested - if return_lse: - if is_varlen: - lse = torch.empty(num_heads, total_q, dtype=torch.float32, device=device).transpose( - 0, 1 - ) - else: - lse = torch.empty( - batch_size, num_heads, seqlen, dtype=torch.float32, device=device - ).transpose(1, 2) - else: - lse = None - - _flash_attn_fwd_combine( - out_partial, - lse_partial, - out, - lse, - cu_seqlens, - seqused, - ) - return out, lse diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mask.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mask.py deleted file mode 100644 index a5f3aeaaae6c..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mask.py +++ /dev/null @@ -1,609 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -from typing import Optional, Callable -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass import Float32, Int32, const_expr - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .seqlen_info import SeqlenInfoQK - - -@cute.jit -def mask_r2p(X: cute.Tensor, col_limit: Int32, arch: int = 90, rank1: bool = False) -> None: - # Bit manipulation, compiles down to the R2P instruction - # For sm100: we know that tScS_t2r[i][1] == i, for the particular tmem copy atom we're using. - # For sm90: instead of comparing limit to 0, 1, 8, 9, 16, 17, ..., - # we compare a transformed version of limit to 0, 1, 2, 3, 4, 5, ... - if const_expr(arch == 90): - col_limit_transformed = col_limit // 8 * 2 + min(col_limit % 8, 2) - else: - col_limit_transformed = col_limit - ncol = const_expr(cute.size(X.shape[cute.rank(X) - 1]) if not rank1 else cute.size(X.shape)) - # Ideally we'd move by 32 instead of 24, but mask >> i isn't correct for i == 31 - for s in cutlass.range_constexpr(cute.ceil_div(ncol, 24)): - # Don't need to clamp to 32 since the shr.u32 instruction does that already - col_limit_right_s = max(col_limit_transformed - s * 24, 0) - # 0 -> 0b00...00, 1 -> 0b00...01, ..., 31 -> 0b01...11, 32 -> 0b11...11 - mask = (1 << col_limit_right_s) - 1 - # This needs to be range_constexpr, o/w the compiler can't generate the R2P instruction - for i in cutlass.range_constexpr(min(24, ncol - s * 24)): - in_bound = cutlass.Boolean(mask & (1 << i)) - c = s * 24 + i - if const_expr(rank1): - X[c] = X[c] if in_bound else -Float32.inf - # This is the equivalent of: - # X[s * 24 + i] = X[s * 24 + i] if col_limit_right_s <= i else -Float32.inf - else: - for r in cutlass.range_constexpr(cute.size(X.shape[0])): - X[r, c] = X[r, c] if in_bound else -Float32.inf - - -@cute.jit -def mask_r2p_transposed(X: cute.Tensor, row_limit_top: Int32, num_rep: int) -> None: - # Bit manipulation, compiles down to the R2P instruction - # For sm100: we know that tScS_t2r[i][0] has the form 0, 1, ..., 31, 64, ..., 127 - # or 0, 1, ..., 15, 32, ..., 47, 64, ... - # We compare a transformed version of limit to 0, 1, 2, 3, 4, 5, ... - # Here we hardcode for the case of 2 warp groups. - num_wg = 2 - row_limit_top_transformed = row_limit_top // (num_rep * num_wg) * num_rep + min( - row_limit_top % (num_rep * num_wg), num_rep - ) - ncol = cute.size(X.shape) - # Ideally we'd move by 32 instead of 24, but mask >> i isn't correct for i == 31 - for s in cutlass.range_constexpr(cute.ceil_div(ncol, 24)): - row_limit_top_s = max(row_limit_top_transformed - s * 24, 0) - # 0 -> 0b00...00, 1 -> 0b00...01, ..., 31 -> 0b01...11, 32 -> 0b11...11 - mask = (1 << row_limit_top_s) - 1 - # This needs to be range_constexpr, o/w the compiler can't generate the R2P instruction - for i in cutlass.range_constexpr(min(24, ncol - s * 24)): - out_bound = cutlass.Boolean(mask & (1 << i)) - c = s * 24 + i - X[c] = -Float32.inf if out_bound else X[c] - # tidx = cute.arch.thread_idx()[0] % 256 - # if tidx == 128: - # cute.printf("tidx = {}, s = {}, i = {}, row_limit_top = {}, row_limit_top_s = {}, mask = {}, out_bound = {}", tidx, s, i, row_limit_top, row_limit_top_s, mask, out_bound) - - -@dataclass(frozen=True) -class AttentionMask: - tile_m: cutlass.Constexpr[int] - tile_n: cutlass.Constexpr[int] - seqlen_info: SeqlenInfoQK - window_size_left: Optional[Int32] = None - window_size_right: Optional[Int32] = None - qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1 # only pass in if we're doing PackGQA - swap_AB: cutlass.Constexpr[bool] = False - - @property - def seqlen_q(self) -> Int32: - return self.seqlen_info.seqlen_q - - @property - def seqlen_k(self) -> Int32: - return self.seqlen_info.seqlen_k - - @cute.jit - def apply_mask( - self, - acc_S: cute.Tensor, - batch_idx: cutlass.Int32, - head_idx: cutlass.Int32, - m_block: cutlass.Int32, - n_block: cutlass.Int32, - thr_mma: cute.TiledMma, - mask_seqlen: cutlass.Constexpr[bool], - mask_causal: cutlass.Constexpr[bool], - mask_local: cutlass.Constexpr[bool] = False, - mask_mod: cutlass.Constexpr[Optional[Callable]] = None, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - ) -> None: - assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True" - acc_S_mn = utils.make_acc_tensor_mn_view(acc_S, transpose=self.swap_AB) - acc_shape = (self.tile_m, self.tile_n) - cS = cute.make_identity_tensor(acc_shape if not self.swap_AB else acc_shape[::-1]) - tScS_mn = utils.make_acc_tensor_mn_view(thr_mma.partition_C(cS), transpose=self.swap_AB) - # We use t0ScS as these indices are known at compile time. We then must subtract the - # column limit by the thread column offset. - t0ScS_mn = utils.make_acc_tensor_mn_view( - thr_mma.get_slice(0).partition_C(cS), transpose=self.swap_AB - ) - ROW = 0 if const_expr(not self.swap_AB) else 1 - COL = 1 if const_expr(not self.swap_AB) else 0 - thr_col_offset = tScS_mn[0][COL] - # To handle edge cases of completely masked out rows where n_block_max = 0, - # we treat negative n_blocks as 0th n_block - # TODO: find more transparent solution - if n_block < 0: - n_block = 0 - seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n - thr_col_offset - if const_expr(not mask_causal and not mask_local and mask_mod is None): - if const_expr(mask_seqlen): - # The compiler now choses not to use R2P - r2p = const_expr(False and not self.swap_AB) - if const_expr(not r2p): - # traverse column index. - for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True): - oob = t0ScS_mn[0, c][COL] >= seqlenk_col_limit - for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True): - acc_S_mn[r, c] = -Float32.inf if oob else acc_S_mn[r, c] - else: - mask_r2p(acc_S_mn, seqlenk_col_limit, arch=90) - - elif const_expr( - not mask_causal and not mask_local and mask_mod is not None - ): # FlexAttention mask mod - nrow = const_expr(cute.size(tScS_mn.shape[0])) - ncol = const_expr(cute.size(tScS_mn.shape[1])) - has_fastdiv = const_expr( - fastdiv_mods is not None - and fastdiv_mods[0] is not None - and fastdiv_mods[1] is not None - ) - wrap_aux_indices = const_expr( - has_fastdiv and mask_seqlen and const_expr(aux_tensors is not None) - ) - - for r in cutlass.range_constexpr(nrow): - # Respect swap_AB: ROW/COL determine which coordinate component corresponds to Q/KV. - local_row = tScS_mn[r, 0][ROW] - global_row_idx = local_row + m_block * self.tile_m - row_for_mod = global_row_idx - head_idx_for_mod = head_idx - if const_expr(self.qhead_per_kvhead_packgqa != 1): - head_offset = global_row_idx % self.qhead_per_kvhead_packgqa - head_idx_for_mod = head_idx * self.qhead_per_kvhead_packgqa + head_offset - row_for_mod = global_row_idx // self.qhead_per_kvhead_packgqa - row_for_seqlen = row_for_mod - if const_expr(wrap_aux_indices): - _, row_for_mod = divmod(row_for_mod, fastdiv_mods[0]) - - for col in cutlass.range_constexpr(ncol): - col_idx_local = t0ScS_mn[0, col][COL] - # Convert to absolute column index - global_col_idx = thr_col_offset + col_idx_local + n_block * self.tile_n - col_for_mod = global_col_idx - if const_expr(wrap_aux_indices): - _, col_for_mod = divmod(global_col_idx, fastdiv_mods[1]) - - batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32) - head_idx_ssa = utils.scalar_to_ssa(head_idx_for_mod, cutlass.Int32) - q_idx_ssa = utils.scalar_to_ssa(row_for_mod, cutlass.Int32) - kv_idx_ssa = utils.scalar_to_ssa(col_for_mod, cutlass.Int32) - mask_value = mask_mod( - batch_idx_ssa, - head_idx_ssa, - q_idx_ssa, - kv_idx_ssa, - self.seqlen_info, - aux_tensors, - ) - cond = cutlass.Boolean(utils.ssa_to_scalar(mask_value)) - if const_expr(mask_seqlen): - out_of_bounds = (row_for_seqlen >= self.seqlen_q) or ( - global_col_idx >= self.seqlen_k - ) - if out_of_bounds: - acc_S_mn[r, col] = -cutlass.Float32.inf - else: - acc_S_mn[r, col] = acc_S_mn[r, col] if cond else -cutlass.Float32.inf - else: - acc_S_mn[r, col] = acc_S_mn[r, col] if cond else -cutlass.Float32.inf - - else: # Causal or local - if const_expr(not self.swap_AB): - # If PackGQA, we split the work of compute divmod among threads in the same row - threads_per_row = thr_mma.tv_layout_C.shape[0][0] - mma_m_idx = None - if const_expr(self.qhead_per_kvhead_packgqa != 1): - assert not self.swap_AB, "swap_AB with PackGQA not supported yet" - assert cute.arch.WARP_SIZE % threads_per_row == 0, ( - "threads_per_row must divide WARP_SIZE" - ) - assert cute.size(acc_S_mn.shape[0]) <= threads_per_row - tidx = thr_mma.thr_idx - mma_m_idx = ( - m_block * self.tile_m + tScS_mn[tidx % threads_per_row, 0][0] - ) // self.qhead_per_kvhead_packgqa - causal_row_offset = ( - 1 + self.seqlen_k - n_block * self.tile_n - self.seqlen_q - thr_col_offset - ) - if const_expr(mask_causal): - r2p = const_expr(not self.swap_AB) # R2P trick, see apply_mask_sm100 - for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True): - # get the column index limit based on current row. Only consider the row index, so the column index sets to 0. - if const_expr(self.qhead_per_kvhead_packgqa == 1): - row_idx = tScS_mn[r, 0][0] + m_block * self.tile_m - else: - row_idx = utils.shuffle_sync( - mma_m_idx, r % threads_per_row, width=threads_per_row - ) - col_limit_right = row_idx + causal_row_offset - if const_expr(mask_seqlen): - col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit) - if const_expr(not r2p): - # traverse column index. - for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True): - acc_S_mn[r, c] = ( - -Float32.inf - if t0ScS_mn[0, c][1] >= col_limit_right - else acc_S_mn[r, c] - ) - else: - mask_r2p(acc_S_mn[r, None], col_limit_right, arch=90, rank1=True) - else: # Local - local_row_offset_right = ( - causal_row_offset + self.window_size_right - if const_expr(self.window_size_right is not None) - else None - ) - local_row_offset_left = ( - causal_row_offset - 1 - self.window_size_left - if const_expr(self.window_size_left is not None) - else None - ) - for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True): - if const_expr(self.qhead_per_kvhead_packgqa == 1): - row_idx = tScS_mn[r, 0][0] + m_block * self.tile_m - else: - row_idx = utils.shuffle_sync( - mma_m_idx, r % threads_per_row, width=threads_per_row - ) - if const_expr(self.window_size_right is not None): - col_limit_right = row_idx + local_row_offset_right - else: - col_limit_right = self.tile_n - if const_expr(mask_seqlen): - col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit) - col_limit_left = ( - row_idx + local_row_offset_left - if const_expr(self.window_size_left is not None) - else 0 - ) - # if cute.arch.thread_idx()[0] == 128: cute.printf("n_block = {}, r = {}, row_idx = {}, causal_row_offset = {}, col_limit_right = {}, col_limit_left = {}", n_block, r, row_idx, causal_row_offset, col_limit_right, col_limit_left) - # traverse column index. - for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True): - col_idx = t0ScS_mn[0, c][1] - # only consider the column index, so the row index sets to 0. - if col_idx >= col_limit_right or col_idx < col_limit_left: - acc_S_mn[r, c] = -Float32.inf - else: # swap_AB - assert self.qhead_per_kvhead_packgqa == 1 - thr_row_offset = tScS_mn[0][ROW] - causal_row_offset = ( - seqlenk_col_limit - self.seqlen_q + m_block * self.tile_m + thr_row_offset - ) - if const_expr(mask_causal): - for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True): - col0 = t0ScS_mn[0, c][COL] - # If col0 is beyond the column limit, we want to mask out the entire - # column, by setting row limit to be self.tile_m. - row_limit_top = ( - self.tile_m - if col0 >= seqlenk_col_limit and mask_seqlen - else col0 - causal_row_offset - ) - for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True): - acc_S_mn[r, c] = ( - -Float32.inf - if t0ScS_mn[r, 0][ROW] < row_limit_top - else acc_S_mn[r, c] - ) - else: - for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True): - col0 = t0ScS_mn[0, c][COL] - # If col0 is beyond the column limit, we want to mask out the entire - # column, by setting row limit to be self.tile_m. - row_limit_top = ( - self.tile_m - if col0 >= seqlenk_col_limit - else col0 - causal_row_offset - self.window_size_right - ) - # TODO: do we need col_limit_sink? - row_limit_bot = col0 - causal_row_offset + self.window_size_left - for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True): - row_idx = t0ScS_mn[r, 0][ROW] - acc_S_mn[r, c] = ( - -Float32.inf - if row_idx < row_limit_top or row_idx > row_limit_bot - else acc_S_mn[r, c] - ) - - @cute.jit - def apply_mask_sm100( - self, - acc_S: cute.Tensor, - m_block: Int32, - n_block: Int32, - thr_mma: cute.TiledMma, - thr_tmem_load: cute.TiledCopy, - mask_seqlen: cutlass.Constexpr[bool], - mask_causal: cutlass.Constexpr[bool], - mask_local: cutlass.Constexpr[bool] = False, - mask_mod: cutlass.Constexpr[Optional[Callable]] = None, - batch_idx: Int32 = None, - head_idx: Int32 = None, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - check_q_boundary: bool = False, - ) -> None: - assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True" - acc_shape = (self.tile_m, self.tile_n) - cS = cute.make_identity_tensor(acc_shape if not self.swap_AB else acc_shape[::-1]) - tScS = thr_mma.partition_C(cS) - tScS_t2r = thr_tmem_load.partition_D(tScS) - # To handle edge cases of completely masked out rows where n_block_max = 0, - # we treat negative n_blocks as 0th n_block - # TODO: find more transparent solution - if n_block < 0: - n_block = 0 - seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n - r2p = True - if const_expr(not mask_causal and not mask_local and mask_mod is None): - if const_expr(mask_seqlen): - if const_expr(not r2p): - for i in cutlass.range(cute.size(tScS_t2r.shape), unroll_full=True): - # if tScS_t2r[i][1] >= seqlenk_col_limit: - # acc_S[i] = -Float32.inf - # For some reason the 2 lines above generate really bad SASS - acc_S[i] = -Float32.inf if tScS_t2r[i][1] >= seqlenk_col_limit else acc_S[i] - else: - mask_r2p(acc_S, seqlenk_col_limit, arch=100, rank1=True) - - elif const_expr(not mask_causal and not mask_local and mask_mod is not None): - # Block sparse case w/ mask_mod - has_fastdiv = const_expr( - fastdiv_mods is not None - and fastdiv_mods[0] is not None - and fastdiv_mods[1] is not None - ) - batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32) - - ncol = const_expr(cute.size(tScS_t2r.shape)) - for i in cutlass.range_constexpr(ncol): - row_coord = tScS_t2r[i][0] if not self.swap_AB else tScS_t2r[i][1] - col_coord = tScS_t2r[i][1] if not self.swap_AB else tScS_t2r[i][0] - global_row = row_coord + m_block * self.tile_m - global_col = col_coord + n_block * self.tile_n - - if const_expr(self.qhead_per_kvhead_packgqa != 1): - head_offset = global_row % self.qhead_per_kvhead_packgqa - head_idx_for_mod = head_idx * self.qhead_per_kvhead_packgqa + head_offset - mask_row = global_row // self.qhead_per_kvhead_packgqa - else: - head_idx_for_mod = head_idx - mask_row = global_row - - mask_row_for_mod = mask_row - if const_expr(has_fastdiv and aux_tensors is not None): - if check_q_boundary: - _, mask_row_for_mod = divmod(mask_row, fastdiv_mods[0]) - global_col_for_mod = global_col - if const_expr(has_fastdiv and mask_seqlen and aux_tensors is not None): - _, global_col_for_mod = divmod(global_col, fastdiv_mods[1]) - - head_idx_ssa = utils.scalar_to_ssa(head_idx_for_mod, cutlass.Int32) - mask_row_ssa = utils.scalar_to_ssa(mask_row_for_mod, cutlass.Int32) - kv_idx_ssa = utils.scalar_to_ssa(global_col_for_mod, cutlass.Int32) - mask_value = mask_mod( - batch_idx_ssa, - head_idx_ssa, - mask_row_ssa, - kv_idx_ssa, - self.seqlen_info, - aux_tensors, - ) - cond = cutlass.Boolean(utils.ssa_to_scalar(mask_value)) - acc_S[i] = acc_S[i] if cond else -Float32.inf - if const_expr(mask_seqlen): - acc_S[i] = -Float32.inf if global_col >= self.seqlen_k else acc_S[i] - if check_q_boundary: - acc_S[i] = -Float32.inf if mask_row >= self.seqlen_q else acc_S[i] - - else: # Causal or local - causal_row_offset = 1 + self.seqlen_k - n_block * self.tile_n - self.seqlen_q - row_idx = tScS_t2r[0][0] + m_block * self.tile_m - if const_expr(self.qhead_per_kvhead_packgqa != 1): - row_idx = row_idx // self.qhead_per_kvhead_packgqa - if const_expr(mask_causal): - col_limit_right = row_idx + causal_row_offset - if const_expr(mask_seqlen): - col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit) - # if cute.arch.thread_idx()[0] % 32 == 0: - # cute.printf("tidx = %d, tidx tmem = %d, row_idx = %d, col_limit_right = %d, causal_row_offset = %d\n", cute.arch.thread_idx()[0], thr_tmem_load.thr_idx, row_idx, col_limit_right, causal_row_offset) - ncol = const_expr(cute.size(tScS_t2r.shape)) - if const_expr(not r2p): - for i in cutlass.range(ncol, unroll_full=True): - acc_S[i] = -Float32.inf if tScS_t2r[i][1] >= col_limit_right else acc_S[i] - else: - mask_r2p(acc_S, col_limit_right, arch=100, rank1=True) - else: - local_row_offset_right = ( - causal_row_offset + self.window_size_right - if const_expr(self.window_size_right is not None) - else None - ) - local_row_offset_left = ( - causal_row_offset - 1 - self.window_size_left - if const_expr(self.window_size_left is not None) - else None - ) - if const_expr(self.window_size_right is not None): - col_limit_right = row_idx + local_row_offset_right - else: - col_limit_right = self.tile_n - if const_expr(mask_seqlen): - col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit) - col_limit_left = ( - row_idx + local_row_offset_left - if const_expr(self.window_size_left is not None) - else 0 - ) - # if cute.arch.thread_idx()[0] == 0 or cute.arch.thread_idx()[0] == 128: cute.printf("m_block = {}, n_block = {}, row_idx = {}, causal_row_offset = {}, col_limit_right = {}, col_limit_left = {}", m_block, n_block, row_idx, causal_row_offset, col_limit_right, col_limit_left) - for i in cutlass.range(cute.size(tScS_t2r.shape), unroll_full=True): - col_idx = tScS_t2r[i][1] - acc_S[i] = ( - -Float32.inf - if col_idx >= col_limit_right or col_idx < col_limit_left - else acc_S[i] - ) - - @cute.jit - def apply_mask_sm100_transposed( - self, - acc_S: cute.Tensor, - tScS_t2r: cute.Tensor, - t0ScS_t2r: cute.Tensor, - m_block: cutlass.Int32, - n_block: cutlass.Int32, - mask_seqlen: cutlass.Constexpr, - mask_causal: cutlass.Constexpr, - mask_local: cutlass.Constexpr, - mask_mod: cutlass.Constexpr[Optional[Callable]] = None, - batch_idx: Int32 = None, - head_idx: Int32 = None, - aux_tensors: Optional[list] = None, - fastdiv_mods=(None, None), - is_full_block: bool = False, - check_m_boundary: bool = True, - ) -> None: - """ - Backward pass: mask S = K @ Q.T where n_block tiles seqlen_k and m_block tiles seqlen_q. - - Coordinate conventio: - - ROW corresponds to Q (m_block) - - COL corresponds to KV (n_block) - - is_full_block: If True, skip mask_mod (all elements valid). Only apply seqlen masking. - check_m_boundary: If False, skip seqlen_q boundary check (optimization for non-boundary m_blocks). - When iterating m_blocks in forward order, only the last m_block may be partial. - """ - assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True" - ROW = 0 if const_expr(not self.swap_AB) else 1 - COL = 1 if const_expr(not self.swap_AB) else 0 - assert t0ScS_t2r[0][COL] == 0, "col0 == 0" - thr_col_offset = tScS_t2r[0][COL] - seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n - thr_col_offset - - if const_expr(not mask_causal and not mask_local and mask_mod is not None): - # Block sparse case with mask_mod (backward) - # - # Coordinate convention: ROW → Q (m_block), COL → KV (n_block). - # These already account for swap_AB. - # - # FULL blocks: mask_mod returns True for all elements, so skip it. - # Still need seqlen bounds check (elements may be OOB on last m_block). - # PARTIAL blocks: apply mask_mod element-wise, then seqlen bounds. - if is_full_block: - if const_expr(mask_seqlen): - if seqlenk_col_limit <= 0: - # Entire tile is OOB for K - for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True): - acc_S[i] = -cutlass.Float32.inf - elif check_m_boundary: - # Last m_block: check Q and K boundaries - ncol = const_expr(cute.size(tScS_t2r.shape)) - for i in cutlass.range_constexpr(ncol): - row_coord = tScS_t2r[i][ROW] - col_coord = tScS_t2r[i][COL] - global_q = row_coord + m_block * self.tile_m - global_kv = col_coord + n_block * self.tile_n - q_out_of_bounds = global_q >= self.seqlen_q - kv_out_of_bounds = global_kv >= self.seqlen_k - out_of_bounds = q_out_of_bounds or kv_out_of_bounds - acc_S[i] = -cutlass.Float32.inf if out_of_bounds else acc_S[i] - else: - # Partial block - has_fastdiv = const_expr( - fastdiv_mods is not None - and fastdiv_mods[0] is not None - and fastdiv_mods[1] is not None - ) - wrap_aux_indices = const_expr( - has_fastdiv and mask_seqlen and const_expr(aux_tensors is not None) - ) - batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32) - head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32) - - ncol = const_expr(cute.size(tScS_t2r.shape)) - for i in cutlass.range_constexpr(ncol): - row_coord = tScS_t2r[i][ROW] - col_coord = tScS_t2r[i][COL] - global_q = row_coord + m_block * self.tile_m - global_kv = col_coord + n_block * self.tile_n - - q_idx_for_mod = global_q - kv_idx_for_mod = global_kv - if const_expr(wrap_aux_indices): - _, q_idx_for_mod = divmod(global_q, fastdiv_mods[0]) - _, kv_idx_for_mod = divmod(global_kv, fastdiv_mods[1]) - - q_idx_ssa = utils.scalar_to_ssa(q_idx_for_mod, cutlass.Int32) - kv_idx_ssa = utils.scalar_to_ssa(kv_idx_for_mod, cutlass.Int32) - - mask_value = mask_mod( - batch_idx_ssa, - head_idx_ssa, - q_idx_ssa, - kv_idx_ssa, - self.seqlen_info, - aux_tensors, - ) - cond = cutlass.Boolean(utils.ssa_to_scalar(mask_value)) - acc_S[i] = acc_S[i] if cond else -cutlass.Float32.inf - - if const_expr(mask_seqlen): - # check_m_boundary=False skips q check for non-boundary m_blocks - q_out_of_bounds = check_m_boundary and (global_q >= self.seqlen_q) - kv_out_of_bounds = global_kv >= self.seqlen_k - out_of_bounds = q_out_of_bounds or kv_out_of_bounds - acc_S[i] = -cutlass.Float32.inf if out_of_bounds else acc_S[i] - - elif const_expr(not mask_causal and not mask_local): - if const_expr(mask_seqlen): - if seqlenk_col_limit <= 0: - for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True): - acc_S[i] = -cutlass.Float32.inf - else: # Causal or local - thr_row_offset = tScS_t2r[0][ROW] - seqlenq_row_limit = self.seqlen_q - m_block * self.tile_m - thr_row_offset - causal_offset = seqlenq_row_limit - seqlenk_col_limit - if const_expr(mask_causal): - # tidx = cute.arch.thread_idx()[0] % 256 - # if tidx < 32: - # cute.printf("tidx = {}, {} {}, {} {}", tidx, tScS_t2r[0][0], tScS_t2r[0][1], tScS_t2r[1][0], tScS_t2r[1][1]) - row_limit_top = causal_offset - if const_expr(mask_seqlen): - # If col is beyond the column limit, we want to mask out the entire - # column, by setting row limit to be self.tile_m. - if seqlenk_col_limit <= 0: - row_limit_top = self.tile_m - r2p = True - if const_expr(not r2p): - for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True): - acc_S[i] = ( - -cutlass.Float32.inf if t0ScS_t2r[i][ROW] < row_limit_top else acc_S[i] - ) - else: - num_rep = cute.size(tScS_t2r, mode=[0]) # 16 or 32 - mask_r2p_transposed(acc_S, row_limit_top, num_rep) - else: - if const_expr(self.window_size_right is not None): - row_limit_top = causal_offset - self.window_size_right - else: - row_limit_top = 0 - if const_expr(self.window_size_left is not None): - row_limit_bot = causal_offset + self.window_size_left - if const_expr(mask_seqlen): - if seqlenk_col_limit <= 0: - row_limit_top = self.tile_m - for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True): - row_idx = t0ScS_t2r[i][ROW] - local_mask = row_idx < row_limit_top - if const_expr(self.window_size_left is not None): - local_mask |= row_idx > row_limit_bot - acc_S[i] = -cutlass.Float32.inf if local_mask else acc_S[i] diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mma_sm100_desc.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mma_sm100_desc.py deleted file mode 100644 index 16336c34686b..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/mma_sm100_desc.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright (c) 2025, Tri Dao. -# Ported Cutlass code from C++ to Python: -# https://github.com/NVIDIA/cutlass/blob/main/include/cute/arch/mma_sm100_desc.hpp -# https://github.com/NVIDIA/cutlass/blob/main/include/cute/atom/mma_traits_sm100.hpp - -from enum import IntEnum - -import cutlass -import cutlass.cute as cute - -# --------------------------------------------------------------------------- -# Enumerations that match the HW encodings (values MUST stay identical) -# --------------------------------------------------------------------------- - - -class Major(IntEnum): # matrix “layout” in the ISA docs - K = 0 - MN = 1 - - -class ScaleIn(IntEnum): # negate flags - One = 0 - Neg = 1 - - -class Saturate(IntEnum): - False_ = 0 - True_ = 1 - - -class CFormat(IntEnum): # 2-bit field (bits 4-5) - F16 = 0 - F32 = 1 - S32 = 2 - - -class F16F32Format(IntEnum): # 3-bit field (A/B element type) - F16 = 0 - BF16 = 1 - TF32 = 2 - - -class S8Format(IntEnum): - UINT8 = 0 - INT8 = 1 - - -class MXF8F6F4Format(IntEnum): - E4M3 = 0 - E5M2 = 1 - E2M3 = 3 - E3M2 = 4 - E2M1 = 5 - - -class MaxShift(IntEnum): - NoShift = 0 - MaxShift8 = 1 - MaxShift16 = 2 - MaxShift32 = 3 - - -# --------------------------------------------------------------------------- -# CUTLASS-type → encoding helpers -# --------------------------------------------------------------------------- - - -def to_UMMA_format(cutlass_type) -> int: - """ - Map a CUTLASS scalar class to the 3-bit encoding for Matrix A/B. - """ - if cutlass_type is cutlass.Int8: - return S8Format.INT8 - # Unsigned 8-bit (if available in your CUTLASS build) - if cutlass_type is cutlass.Uint8: - return S8Format.UINT8 - # FP-16 / BF-16 - if cutlass_type is cutlass.Float16: - return F16F32Format.F16 - if cutlass_type is cutlass.BFloat16: - return F16F32Format.BF16 - # TensorFloat-32 (8-bit exponent, 10-bit mantissa packed in 19 bits) - if cutlass_type is cutlass.TFloat32: - return F16F32Format.TF32 - # Float-8 / Float-6 / Float-4 – add whenever CUTLASS exposes them - if cutlass_type is cutlass.FloatE4M3FN: - return MXF8F6F4Format.E4M3 - if cutlass_type is cutlass.FloatE5M2: - return MXF8F6F4Format.E5M2 - raise TypeError(f"Unsupported CUTLASS scalar type for A/B: {cutlass_type!r}") - - -def to_C_format(cutlass_type) -> int: - """ - Map a CUTLASS scalar class to the 2-bit accumulator encoding. - """ - if cutlass_type is cutlass.Float16: - return CFormat.F16 - if cutlass_type is cutlass.Float32: - return CFormat.F32 - if cutlass_type is cutlass.Int32: - return CFormat.S32 - raise TypeError(f"Unsupported CUTLASS scalar type for accumulator: {cutlass_type!r}") - - -# --------------------------------------------------------------------------- -# The constructor – accepts only CUTLASS scalar classes -# --------------------------------------------------------------------------- - - -def make_instr_desc( - a_type, # CUTLASS scalar class, e.g. cutlass.Int8 - b_type, - c_type, - M: int, # 64, 128 or 256 - N: int, # 8 … 256 (multiple of 8) - a_major: Major, - b_major: Major, - a_neg: ScaleIn = ScaleIn.One, - b_neg: ScaleIn = ScaleIn.One, - c_sat: Saturate = Saturate.False_, - is_sparse: bool = False, - max_shift: MaxShift = MaxShift.NoShift, -) -> int: - """ - Build the 32-bit instruction descriptor for Blackwell MMA. - All matrix/accumulator **types must be CUTLASS scalar classes** – - passing integers is forbidden. - """ - # --- encode element formats ------------------------------------------------- - a_fmt = int(to_UMMA_format(a_type)) - b_fmt = int(to_UMMA_format(b_type)) - c_fmt = int(to_C_format(c_type)) - - # --- range checks on M/N ----------------------------------------------------- - if M not in (64, 128, 256): - raise ValueError("M must be 64, 128 or 256") - if N < 8 or N > 256 or (N & 7): - raise ValueError("N must be a multiple of 8 in the range 8…256") - - m_dim = M >> 4 # 5-bit field - n_dim = N >> 3 # 6-bit field - - # fmt: off - # --- pack the bit-fields ----------------------------------------------------- - desc = 0 - desc |= (0 & 0x3) << 0 # sparse_id2 (always 0 here) - desc |= (int(is_sparse) & 0x1) << 2 # sparse_flag - desc |= (int(c_sat) & 0x1) << 3 # saturate - desc |= (c_fmt & 0x3) << 4 # c_format - desc |= (a_fmt & 0x7) << 7 # a_format - desc |= (b_fmt & 0x7) << 10 # b_format - desc |= (int(a_neg) & 0x1) << 13 # a_negate - desc |= (int(b_neg) & 0x1) << 14 # b_negate - desc |= (int(a_major) & 0x1) << 15 # a_major - desc |= (int(b_major) & 0x1) << 16 # b_major - desc |= (n_dim & 0x3F) << 17 # n_dim (6 bits) - desc |= (m_dim & 0x1F) << 24 # m_dim (5 bits) - desc |= (int(max_shift) & 0x3) << 30 # max_shift (2 bits) - # fmt: on - - return desc & 0xFFFF_FFFF # ensure 32-bit result - - -def mma_op_to_idesc(op: cute.nvgpu.tcgen05.mma.MmaOp): - return make_instr_desc( - op.a_dtype, - op.b_dtype, - op.acc_dtype, - op.shape_mnk[0], - op.shape_mnk[1], - Major.K if op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN, - Major.K if op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN, - ) - - -class LayoutType(IntEnum): # occupies the top-3 bits [61:64) - SWIZZLE_NONE = 0 # (a.k.a. “INTERLEAVE” in older docs) - SWIZZLE_128B_BASE32B = 1 - SWIZZLE_128B = 2 - SWIZZLE_64B = 4 - SWIZZLE_32B = 6 - # values 3,5,7 are reserved / illegal for UMMA - - -# --------------------------------------------------------------------------- -# Helpers – figure out the SWIZZLE_* family from the tensor layout -# --------------------------------------------------------------------------- - - -def _layout_type(swizzle: cute.Swizzle) -> LayoutType: - # No idea what the right way to get B, M, S is – so we're just parsing it from the __str__ - # Swizzle string has the form "S" - swz_str = str(swizzle) - inside = swz_str[swz_str.index("<") + 1 : swz_str.index(">")] # '3,4,3' - B, M, S = [int(x) for x in inside.split(",")] # [3, 4, 3] - - if M == 4: # Swizzle<*,4,3> - if S != 3: - raise ValueError("Unexpected swizzle shift – want S==3 for M==4") - return { - 0: LayoutType.SWIZZLE_NONE, - 1: LayoutType.SWIZZLE_32B, - 2: LayoutType.SWIZZLE_64B, - 3: LayoutType.SWIZZLE_128B, - }[B] # KeyError ⇒ invalid B→ raise - if M == 5: # Swizzle<2,5,2> (the only legal triple for M==5) - if (B, S) != (2, 2): - raise ValueError("Only Swizzle<2,5,2> supported for 128B_BASE32B") - return LayoutType.SWIZZLE_128B_BASE32B - - # Any other (M,B,S) triple is not a UMMA-legal shared-memory layout - raise ValueError("Unsupported swizzle triple for UMMA smem descriptor") - - -def make_smem_desc_base(layout: cute.Layout, swizzle: cute.Swizzle, major: Major) -> int: - """ - Convert a 2-D *shared-memory* Cute layout into the Blackwell 64-bit - smem-descriptor, without the smem start address. - layout must correspond to layout of an uint128 tensor. - """ - # ------------------------------------------------------------------ meta - layout_type = _layout_type(swizzle) # resolve SWIZZLE_* family - - VERSION = 1 # bits 46–47 - LBO_MODE = 0 # bit 52 - BASE_OFFSET = 0 # bits 49–51 (CUTLASS always 0) - - # ---------------------------------------------------------- strides (units: uint128_t = 16 B) - swizzle_atom_mn_size = { - LayoutType.SWIZZLE_NONE: 1, - LayoutType.SWIZZLE_32B: 2, - LayoutType.SWIZZLE_64B: 4, - LayoutType.SWIZZLE_128B: 8, - LayoutType.SWIZZLE_128B_BASE32B: 8, - }[layout_type] - - if major is Major.MN: - swizzle_atom_k_size = 4 if layout_type is LayoutType.SWIZZLE_128B_BASE32B else 8 - canonical_layout = cute.logical_divide(layout, (swizzle_atom_mn_size, swizzle_atom_k_size)) - if not cute.is_congruent(canonical_layout, ((1, 1), (1, 1))): - raise ValueError("Not a canonical UMMA_MN Layout: Expected profile failure.") - stride_00 = canonical_layout.stride[0][0] - if layout_type is not LayoutType.SWIZZLE_NONE and stride_00 != 1: - raise ValueError("Not a canonical UMMA_MN Layout: Expected stride failure.") - stride_10 = canonical_layout.stride[1][0] - if stride_10 != swizzle_atom_mn_size: - raise ValueError("Not a canonical UMMA_MN Layout: Expected stride failure.") - stride_01, stride_11 = canonical_layout.stride[0][1], canonical_layout.stride[1][1] - if layout_type is LayoutType.SWIZZLE_NONE: - stride_byte_offset, leading_byte_offset = stride_01, stride_11 - else: - stride_byte_offset, leading_byte_offset = stride_11, stride_01 - else: - if layout_type == LayoutType.SWIZZLE_128B_BASE32B: - raise ValueError("SWIZZLE_128B_BASE32B is invalid for Major-K") - if not cute.size(layout.shape[0]) % 8 == 0: - raise ValueError("Not a canonical UMMA_K Layout: Expected MN-size multiple of 8.") - canonical_layout = cute.logical_divide(layout, (8, 2)) - if not cute.is_congruent(canonical_layout, ((1, 1), (1, 1))): - raise ValueError("Not a canonical UMMA_K Layout: Expected profile failure.") - stride_00 = canonical_layout.stride[0][0] - if stride_00 != swizzle_atom_mn_size: - raise ValueError("Not a canonical UMMA_K Layout: Expected stride failure.") - stride_10 = canonical_layout.stride[1][0] - if layout_type is not LayoutType.SWIZZLE_NONE and stride_10 != 1: - raise ValueError("Not a canonical UMMA_K Layout: Expected stride failure.") - stride_01 = canonical_layout.stride[0][1] - stride_byte_offset, leading_byte_offset = stride_01, stride_10 - - # ------------------------------------------------------------------ pack - desc = 0 - # leading_byte_offset_ [16:30) - desc |= (leading_byte_offset & 0x3FFF) << 16 - # stride_byte_offset_ [32:46) - desc |= (stride_byte_offset & 0x3FFF) << 32 - # version_ [46:48) - desc |= (VERSION & 0x3) << 46 - # base_offset_ [49:52) - desc |= (BASE_OFFSET & 0x7) << 49 - # lbo_mode_ [52:53) - desc |= (LBO_MODE & 0x1) << 52 - # layout_type_ [61:64) - desc |= (int(layout_type) & 0x7) << 61 - - return desc & 0xFFFF_FFFF_FFFF_FFFF # force 64-bit width - - -def make_smem_desc_start_addr(start_addr: cute.Pointer) -> cutlass.Int32: - # 14 bits, remove 4 LSB (bits 0-13 in desc) - return (start_addr.toint() & 0x3FFFF) >> 4 diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/named_barrier.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/named_barrier.py deleted file mode 100644 index 777c44079a04..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/named_barrier.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. - -import enum - - -class NamedBarrierFwd(enum.IntEnum): - Epilogue = enum.auto() # starts from 1 as barrier 0 is reserved for sync_threads() - WarpSchedulerWG1 = enum.auto() - WarpSchedulerWG2 = enum.auto() - WarpSchedulerWG3 = enum.auto() - PFull = enum.auto() - PEmpty = enum.auto() - - -class NamedBarrierBwd(enum.IntEnum): - Epilogue = enum.auto() - WarpSchedulerWG1 = enum.auto() - WarpSchedulerWG2 = enum.auto() - WarpSchedulerWG3 = enum.auto() - PdS = enum.auto() - dQFullWG0 = enum.auto() - dQFullWG1 = enum.auto() - dQEmptyWG0 = enum.auto() - dQEmptyWG1 = enum.auto() - - -class NamedBarrierBwdSm100(enum.IntEnum): - EpilogueWG1 = enum.auto() - EpilogueWG2 = enum.auto() - Compute = enum.auto() - dQaccReduce = enum.auto() diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pack_gqa.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pack_gqa.py deleted file mode 100644 index 3a6fcde61298..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pack_gqa.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - - -import cutlass -import cutlass.cute as cute - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils - - -class PackGQA: - def __init__( - self, - m_block_size: cutlass.Constexpr[int], - head_dim_padded: cutlass.Constexpr[int], - check_hdim_oob: cutlass.Constexpr[bool], - qhead_per_kvhead: cutlass.Constexpr[bool], - ): - self.m_block_size = m_block_size - self.head_dim_padded = head_dim_padded - self.check_hdim_oob = check_hdim_oob - self.qhead_per_kvhead = qhead_per_kvhead - - @cute.jit - def compute_ptr( - self, - tensor: cute.Tensor, - cRows: cute.Tensor, - tidx: cutlass.Int32, - block: cutlass.Int32, - threads_per_row: cutlass.Constexpr[int], - num_threads: cutlass.Constexpr[int], - ): - num_ptr_per_thread = cute.ceil_div(cute.size(cRows), threads_per_row) - tPrPtr = cute.make_fragment(num_ptr_per_thread, cutlass.Int64) - for i in cutlass.range_constexpr(num_ptr_per_thread): - row = i * num_threads + cRows[tidx % threads_per_row][0] - idx = block * self.m_block_size + row - m_idx = idx // self.qhead_per_kvhead - h_idx = idx - m_idx * self.qhead_per_kvhead - tPrPtr[i] = utils.elem_pointer(tensor, ((h_idx, m_idx),)).toint() - return tPrPtr - - @cute.jit - def load_Q( - self, - mQ: cute.Tensor, # ((qhead_per_kvhead, seqlen_q), headdim) - sQ: cute.Tensor, # (m_block_size, head_dim_padded) - gmem_tiled_copy: cute.TiledCopy, - tidx: cutlass.Int32, - block: cutlass.Int32, - seqlen: cutlass.Int32, - ): - gmem_thr_copy = gmem_tiled_copy.get_slice(tidx) - cQ = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded)) - tQsQ = gmem_thr_copy.partition_D(sQ) - tQcQ = gmem_thr_copy.partition_S(cQ) - t0QcQ = gmem_thr_copy.get_slice(0).partition_S(cQ) - tQpQ = utils.predicate_k(tQcQ, limit=mQ.shape[1]) - tQcQ_row = tQcQ[0, None, 0] - threads_per_row = gmem_tiled_copy.layout_tv_tiled.shape[0][0] - assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE" - num_threads = gmem_tiled_copy.size - tPrQPtr = self.compute_ptr(mQ[None, 0], tQcQ_row, tidx, block, threads_per_row, num_threads) - for m in cutlass.range_constexpr(cute.size(tQsQ.shape[1])): - q_ptr_i64 = utils.shuffle_sync( - tPrQPtr[m // threads_per_row], m % threads_per_row, width=threads_per_row - ) - q_gmem_ptr = cute.make_ptr( - mQ.element_type, q_ptr_i64, cute.AddressSpace.gmem, assumed_align=16 - ) - if ( - t0QcQ[0, m, 0][0] - < seqlen * self.qhead_per_kvhead - block * self.m_block_size - tQcQ_row[0][0] - ): - mQ_cur = cute.make_tensor(q_gmem_ptr, (self.head_dim_padded,)) - elems_per_load = cute.size(tQsQ.shape[0][0]) - mQ_cur_copy = cute.tiled_divide(mQ_cur, (elems_per_load,)) - for k in cutlass.range_constexpr(cute.size(tQsQ.shape[2])): - ki = tQcQ[0, 0, k][1] // elems_per_load - cute.copy( - gmem_thr_copy, - mQ_cur_copy[None, ki], - tQsQ[None, m, k], - pred=tQpQ[None, m, k] if cutlass.const_expr(self.check_hdim_oob) else None, - ) - # We don't need to clear the sQ smem tiles since we'll only write out the valid outputs - - @cute.jit - def store_LSE( - self, - mLSE: cute.Tensor, # (qhead_per_kvhead, seqlen_q) - tLSErLSE: cute.Tensor, # (m_block_size, head_dim_padded) - tiled_mma: cute.TiledMma, - tidx: cutlass.Int32, - block: cutlass.Int32, - seqlen: cutlass.Int32, - ): - thr_mma = tiled_mma.get_slice(tidx) - caccO = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded)) - taccOcO = thr_mma.partition_C(caccO) - taccOcO_row = utils.make_acc_tensor_mn_view(taccOcO)[None, 0] - assert cute.size(tLSErLSE) == cute.size(taccOcO_row) - threads_per_row = tiled_mma.tv_layout_C.shape[0][0] - assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE" - assert cute.size(tLSErLSE) <= threads_per_row - num_threads = tiled_mma.size - tPrLSEPtr = self.compute_ptr(mLSE, taccOcO_row, tidx, block, threads_per_row, num_threads) - for m in cutlass.range_constexpr(cute.size(tLSErLSE)): - lse_ptr_i64 = utils.shuffle_sync( - tPrLSEPtr[m // threads_per_row], - m % threads_per_row, - width=threads_per_row, - ) - lse_gmem_ptr = cute.make_ptr( - mLSE.element_type, lse_ptr_i64, cute.AddressSpace.gmem, assumed_align=4 - ) - row = block * self.m_block_size + taccOcO_row[m][0] - # Only the thread corresponding to column 0 writes out the lse to gmem - if taccOcO[0][1] == 0 and row < seqlen * self.qhead_per_kvhead: - mLSE_copy = cute.make_tensor(lse_gmem_ptr, (1,)) - mLSE_copy[0] = tLSErLSE[m] - - @cute.jit - def store_O( - self, - mO: cute.Tensor, # ((qhead_per_kvhead, seqlen_q), headdim) - tOrO: cute.Tensor, # (m_block_size, head_dim_padded) split across threads according to gmem_tiled_copy - gmem_tiled_copy: cute.TiledCopy, - tidx: cutlass.Int32, - block: cutlass.Int32, - seqlen: cutlass.Int32, - ): - gmem_thr_copy = gmem_tiled_copy.get_slice(tidx) - cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded)) - tOcO = gmem_thr_copy.partition_S(cO) - t0OcO = gmem_thr_copy.get_slice(0).partition_S(cO) - tOpO = utils.predicate_k(tOcO, limit=mO.shape[1]) - tOcO_row = tOcO[0, None, 0] - threads_per_row = gmem_tiled_copy.layout_tv_tiled.shape[0][0] - assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE" - num_threads = gmem_tiled_copy.size - tPrOPtr = self.compute_ptr(mO[None, 0], tOcO_row, tidx, block, threads_per_row, num_threads) - for m in cutlass.range_constexpr(cute.size(tOrO.shape[1])): - o_ptr_i64 = utils.shuffle_sync( - tPrOPtr[m // threads_per_row], m % threads_per_row, width=threads_per_row - ) - o_gmem_ptr = cute.make_ptr( - mO.element_type, o_ptr_i64, cute.AddressSpace.gmem, assumed_align=16 - ) - if ( - t0OcO[0, m, 0][0] - < seqlen * self.qhead_per_kvhead - block * self.m_block_size - tOcO_row[0][0] - ): - mO_cur = cute.make_tensor(o_gmem_ptr, (self.head_dim_padded,)) - elems_per_load = cute.size(tOrO.shape[0][0]) - mO_cur_copy = cute.tiled_divide(mO_cur, (elems_per_load,)) - for k in cutlass.range_constexpr(cute.size(tOrO.shape[2])): - ki = tOcO[0, 0, k][1] // elems_per_load - cute.copy( - gmem_thr_copy, - tOrO[None, m, k], - mO_cur_copy[None, ki], - pred=tOpO[None, m, k] if cutlass.const_expr(self.check_hdim_oob) else None, - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/paged_kv.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/paged_kv.py deleted file mode 100644 index 4d44dac7a0a8..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/paged_kv.py +++ /dev/null @@ -1,188 +0,0 @@ -from typing import Type -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass.cute.nvgpu import cpasync -from cutlass import Int32, const_expr - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .cute_dsl_utils import ParamsBase -from cutlass.cute import FastDivmodDivisor - - -@dataclass -class PagedKVManager(ParamsBase): - mPageTable: cute.Tensor - mK_paged: cute.Tensor - mV_paged: cute.Tensor - thread_idx: Int32 - - page_size_divmod: FastDivmodDivisor - seqlen_k: Int32 - leftpad_k: Int32 - n_block_size: Int32 - num_threads: cutlass.Constexpr[Int32] - head_dim_padded: cutlass.Constexpr[Int32] - head_dim_v_padded: cutlass.Constexpr[Int32] - - gmem_threads_per_row: cutlass.Constexpr[Int32] - page_entry_per_thread: Int32 - async_copy_elems: Int32 - - gmem_tiled_copy_KV: cute.TiledCopy - gmem_thr_copy_KV: cute.TiledCopy - tPrPage: cute.Tensor - tPrPageOffset: cute.Tensor - tKpK: cute.Tensor - tVpV: cute.Tensor - - @staticmethod - def create( - mPageTable: cute.Tensor, - mK_paged: cute.Tensor, - mV_paged: cute.Tensor, - page_size_divmod: FastDivmodDivisor, - bidb: Int32, - bidh: Int32, - thread_idx: Int32, - seqlen_k: Int32, - leftpad_k: Int32, - n_block_size: cutlass.Constexpr[Int32], - head_dim_padded: cutlass.Constexpr[Int32], - head_dim_v_padded: cutlass.Constexpr[Int32], - num_threads: cutlass.Constexpr[Int32], - dtype: Type[cutlass.Numeric], - ): - universal_copy_bits = 128 - gmem_threads_per_row = 8 # 8 threads loading 128 bits = 128 bytes = 1 cache line - async_copy_elems = universal_copy_bits // dtype.width - atom_async_copy = cute.make_copy_atom( - cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), - dtype, - num_bits_per_copy=universal_copy_bits, - ) - thr_layout = cute.make_ordered_layout( - (num_threads // gmem_threads_per_row, gmem_threads_per_row), - order=(1, 0), - ) - val_layout = cute.make_layout((1, async_copy_elems)) - gmem_tiled_copy_KV = cute.make_tiled_copy_tv(atom_async_copy, thr_layout, val_layout) - gmem_thr_copy_KV = gmem_tiled_copy_KV.get_slice(thread_idx) - page_entry_per_thread = n_block_size * gmem_threads_per_row // num_threads - - tPrPage = cute.make_rmem_tensor((page_entry_per_thread,), Int32) - tPrPageOffset = cute.make_rmem_tensor((page_entry_per_thread,), Int32) - - mPageTable = mPageTable[bidb, None] - mK_paged = mK_paged[None, None, bidh, None] - mV_paged = mV_paged[None, None, bidh, None] - - cK = cute.make_identity_tensor((n_block_size, head_dim_padded)) - tKcK = gmem_thr_copy_KV.partition_S(cK) - tKpK = utils.predicate_k(tKcK, limit=mK_paged.shape[1]) - - if const_expr(head_dim_padded == head_dim_v_padded): - tVpV = tKpK - else: - cV = cute.make_identity_tensor((n_block_size, head_dim_v_padded)) - tVcV = gmem_thr_copy_KV.partition_S(cV) - tVpV = utils.predicate_k(tVcV, limit=mV_paged.shape[0]) - - return PagedKVManager( - mPageTable, - mK_paged, - mV_paged, - thread_idx, - page_size_divmod, - seqlen_k, - leftpad_k, - n_block_size, - num_threads, - head_dim_padded, - head_dim_v_padded, - gmem_threads_per_row, - page_entry_per_thread, - async_copy_elems, - gmem_tiled_copy_KV, - gmem_thr_copy_KV, - tPrPage, - tPrPageOffset, - tKpK, - tVpV, - ) - - @cute.jit - def load_page_table(self, n_block: Int32): - for i in cutlass.range(self.page_entry_per_thread, unroll=1): - row = (i * self.num_threads + self.thread_idx) // self.gmem_threads_per_row - row_idx = n_block * self.n_block_size + row - - page_idx, page_offset = divmod(row_idx + self.leftpad_k, self.page_size_divmod) - - is_valid = ( - (i + 1) * self.num_threads <= self.n_block_size or row < self.n_block_size - ) and row_idx < self.seqlen_k - page = self.mPageTable[page_idx] if is_valid else 0 - - self.tPrPage[i] = page - self.tPrPageOffset[i] = page_offset - - @cute.jit - def load_KV(self, n_block: Int32, sX: cute.Tensor, K_or_V: str): - assert K_or_V in ("K", "V") - - # Finesse sX layout to be (M, N). - sX_pi = cute.make_tensor( - sX.iterator, - cute.make_layout( - (sX.shape[0][0], (sX.shape[0][1], sX.shape[2])), - stride=(sX.stride[0][0], (sX.stride[0][1], sX.stride[2])), - ), - ) - - if const_expr(K_or_V == "V"): - # Need to transpose V - sX_pi = cute.make_tensor(sX_pi.iterator, cute.select(sX_pi.layout, mode=[1, 0])) - - head_dim = self.head_dim_v_padded if const_expr(K_or_V == "V") else self.head_dim_padded - cX = cute.make_identity_tensor((self.n_block_size, head_dim)) - tXsX = self.gmem_thr_copy_KV.partition_D(sX_pi) - tXcX = self.gmem_thr_copy_KV.partition_S(cX) - - seqlenk_row_limit = self.seqlen_k - n_block * self.n_block_size if n_block >= 0 else 0 - for m in cutlass.range(cute.size(tXsX, mode=[1]), unroll=1): - should_load = tXcX[0, m, 0][0] < seqlenk_row_limit - - page = self.tPrPage[m] - page_offset = self.tPrPageOffset[m] - mX_paged_cur = ( - self.mK_paged[page_offset, None, page] - if const_expr(K_or_V == "K") - else self.mV_paged[None, page_offset, page] - ) - mX_paged_cur_copy = cute.tiled_divide(mX_paged_cur, (self.async_copy_elems,)) - - if should_load: - for k in cutlass.range(cute.size(tXsX, mode=[2]), unroll=1): - ki = tXcX[0, 0, k][1] // self.async_copy_elems - cute.copy( - self.gmem_tiled_copy_KV, - mX_paged_cur_copy[None, ki], - tXsX[None, m, k], - ) - elif const_expr(K_or_V == "V"): - # Don't need to clear out the rest of the smem for K since we'll mask out the scores anyway. - fill_swizzled(tXsX[None, m, None], 0) - - -@cutlass.dsl_user_op -def fill_swizzled(tensor, value: cutlass.Numeric, *, loc=None, ip=None) -> None: - """Fill tensor with a constant value. - - Fills all elements of the tensor with the specified value, assuming static size - and supported memory space. - """ - rTmp = cute.make_rmem_tensor_like(tensor, tensor.element_type) - rTmp.fill(value) - cute.autovec_copy(rTmp, tensor) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pipeline.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pipeline.py deleted file mode 100644 index 54981bca1274..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pipeline.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -# import math -from typing import Optional -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass import Boolean, Int32, const_expr -from cutlass.cutlass_dsl import if_generate -from cutlass.pipeline import PipelineAsync, PipelineState, Agent, CooperativeGroup -from cutlass.pipeline import PipelineUserType, PipelineOp -from cutlass.pipeline import PipelineTmaAsync as PipelineTmaAsyncOg -from cutlass.pipeline import PipelineTmaUmma as PipelineTmaUmmaOg - - -# We deviate from cute-dsl implementation to use cute.arch.cluster_arrive_relaxed -def pipeline_init_wait(cta_layout_vmnk: Optional[cute.Layout] = None): - """ - Fences the mbarrier init and syncs the threadblock or cluster - """ - cute.arch.mbarrier_init_fence() - - if cta_layout_vmnk is None or cute.size(cta_layout_vmnk) == 1: - # If not using clusters, sync the threadblock - _sync(Agent.ThreadBlock) - else: - # If using clusters, sync the cluster - _sync(Agent.ThreadBlockCluster) - - -def _sync(group: Agent): - """ - Syncs all threads within an agent. - """ - if group is Agent.Thread: - raise NotImplementedError("Error: Not supported.") - elif group is Agent.ThreadBlock: - cute.arch.sync_threads() - elif group is Agent.ThreadBlockCluster: - cute.arch.cluster_arrive_relaxed() - cute.arch.cluster_wait() - else: - assert False, ( - "Error: No explicit sync instruction exists. Please use barriers (named / mbarrier) instead." - ) - - -class PipelineStateSimple: - """ - Pipeline state contains an index and phase bit corresponding to the current position in the circular buffer. - Use a single Int32 to store both the index and phase bit, then we use divmod to get the - index and phase. If stages is a power of 2, divmod turns into bit twiddling. - """ - - def __init__(self, stages: int, phase_index: Int32): - # assert stages < 2**16 - # self._log_stages = int(math.log2(stages)) - # assert 1 << self._log_stages == stages, "Number of stages must be a power of 2." - self._stages = stages - self._phase_index = phase_index - - def clone(self) -> "PipelineStateSimple": - return PipelineStateSimple(self.stages, self._phase_index) - - @property - def stages(self) -> int: - # return 1 << self._log_stages - return self._stages - - @property - def index(self) -> Int32: - # return self._phase_index & 0xFFFF - # return self._phase_index & ((1 << self._log_stages) - 1) - if const_expr(self._stages == 1): - return Int32(0) - else: - return self._phase_index % self._stages - - @property - def phase(self) -> Int32: - # return self._phase_index >> 16 - # PTX docs say that the phase parity needs to be 0 or 1, so by right we need to - # take modulo 2. But in practice just passing the phase in without modulo works fine. - # return (self._phase_index >> self._log_stages) % 2 - # return self._phase_index >> self._log_stages - if const_expr(self._stages == 1): - return self._phase_index - else: - return self._phase_index // self._stages - - def advance(self): - if const_expr(self._stages == 1): - self._phase_index ^= 1 - else: - self._phase_index += 1 - - # def then_body(phase_index): - # # XOR the phase bit and set the index to 0 - # return (phase_index & 0xFFFF0000) ^ (1 << 16) - - # def else_body(phase_index): - # return phase_index - - # self._phase_index = if_generate( - # (self._phase_index & 0xFFFF) == self.stages, - # then_body, - # else_body, - # [self._phase_index], - # [Int32], - # ) - - def __extract_mlir_values__(self): - phase_index = self._phase_index - return [phase_index.ir_value()] - - def __new_from_mlir_values__(self, values): - return PipelineStateSimple(self.stages, Int32(values[0])) - - -def make_pipeline_state(type: PipelineUserType, stages: int): - """ - Creates a pipeline state. Producers are assumed to start with an empty buffer and have a flipped phase bit of 1. - """ - if type is PipelineUserType.Producer: - # return PipelineStateSimple(stages, Int32(1 << 16)) - return PipelineStateSimple(stages, Int32(stages)) - elif type is PipelineUserType.Consumer: - return PipelineStateSimple(stages, Int32(0)) - else: - assert False, "Error: invalid PipelineUserType specified for make_pipeline_state." - - -@dataclass(frozen=True) -class PipelineTmaAsync(PipelineTmaAsyncOg): - """ - Override producer_acquire to take in extra_tx_count parameter. - """ - - @staticmethod - def create(*args, **kwargs): - obj = PipelineTmaAsyncOg.create(*args, **kwargs) - # Can't assign to __class__ directly since the dataclass is frozen - # obj.__class__ = PipelineTmaAsync - object.__setattr__(obj, "__class__", PipelineTmaAsync) - return obj - - def producer_acquire( - self, - state: PipelineState, - try_acquire_token: Optional[Boolean] = None, - extra_tx_count: int = 0, - ): - """ - TMA producer commit conditionally waits on buffer empty and sets the transaction barrier for leader threadblocks. - """ - if_generate( - try_acquire_token is None or try_acquire_token == 0, - lambda: self.sync_object_empty.wait(state.index, state.phase), - ) - if const_expr(extra_tx_count == 0): - self.sync_object_full.arrive(state.index, self.producer_mask) - else: - tx_count = self.sync_object_full.tx_count + extra_tx_count - self.sync_object_full.arrive_and_expect_tx(state.index, tx_count) - - -@dataclass(frozen=True) -class PipelineTmaUmma(PipelineTmaUmmaOg): - @staticmethod - def create( - *, - num_stages: int, - producer_group: CooperativeGroup, - consumer_group: CooperativeGroup, - tx_count: int, - barrier_storage: cute.Pointer = None, - cta_layout_vmnk: Optional[cute.Layout] = None, - mcast_mode_mn: tuple[int, int] = (1, 1), - init_wait: cutlass.Constexpr[bool] = True, - ): - """ - This helper function computes any necessary attributes and returns an instance of PipelineTmaUmma. - :param barrier_storage: Pointer to the smem address for this pipeline's mbarriers - :type barrier_storage: cute.Pointer - :param num_stages: Number of buffer stages for this pipeline - :type num_stages: Int32 - :param producer_group: `CooperativeGroup` for the producer agent - :type producer_group: CooperativeGroup - :param consumer_group: `CooperativeGroup` for the consumer agent - :type consumer_group: CooperativeGroup - :param tx_count: Number of bytes expected to be written to the transaction barrier for one stage - :type tx_count: int - :param cta_layout_vmnk: Layout of the cluster shape - :type cta_layout_vmnk: cute.Layout | None - :param mcast_mode_mn: Tuple of two integers, specifying whether mcast is enabled for the m and n modes. At least one of the two integers must be 1. - :type mcast_mode_mn: tuple[int, int] - """ - if not isinstance(barrier_storage, cute.Pointer): - raise ValueError( - f"Expected barrier_storage to be a cute.Pointer, but got {type(barrier_storage)}" - ) - - producer_type = PipelineOp.TmaLoad - consumer_type = PipelineOp.TCGen05Mma - - producer = (producer_type, producer_group) - consumer = (consumer_type, consumer_group) - - sync_object_full = PipelineAsync._make_sync_object( - barrier_storage.align(min_align=8), num_stages, producer, tx_count - ) - sync_object_empty = PipelineAsync._make_sync_object( - barrier_storage.align(min_align=8) + num_stages, num_stages, consumer - ) - - if cta_layout_vmnk is None or cute.size(cta_layout_vmnk) == 1: - # No mcast mask if not using clusters - producer_mask = None - # All threadblocks are leaders if not using clusters - is_leader_cta = True - else: - producer_mask = PipelineTmaUmma._compute_mcast_arrival_mask( - cta_layout_vmnk, mcast_mode_mn - ) - is_leader_cta = PipelineTmaUmma._compute_is_leader_cta(cta_layout_vmnk) - - cta_group = ( - cute.nvgpu.tcgen05.CtaGroup.ONE - if cta_layout_vmnk is None or cute.size(cta_layout_vmnk, mode=[0]) == 1 - else cute.nvgpu.tcgen05.CtaGroup.TWO - ) - - consumer_mask = producer_mask - - if const_expr(init_wait): - pipeline_init_wait(cta_layout_vmnk) - - return PipelineTmaUmma( - sync_object_full, - sync_object_empty, - num_stages, - producer_mask, - consumer_mask, - is_leader_cta, - cta_group, - ) - - def producer_acquire( - self, - state: PipelineState, - try_acquire_token: Optional[Boolean] = None, - extra_tx_count: int = 0, - ): - """ - TMA producer commit conditionally waits on buffer empty and sets the transaction barrier for leader threadblocks. - """ - if_generate( - try_acquire_token is None or try_acquire_token == 0, - lambda: self.sync_object_empty.wait(state.index, state.phase), - ) - if const_expr(extra_tx_count == 0): - if_generate( - self.is_leader_cta, - lambda: self.sync_object_full.arrive(state.index, self.producer_mask), - ) - else: - tx_count = self.sync_object_full.tx_count + extra_tx_count - if_generate( - self.is_leader_cta, - lambda: self.sync_object_full.arrive_and_expect_tx(state.index, tx_count), - ) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pyproject.toml b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pyproject.toml deleted file mode 100644 index 7aa3f6768ed0..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/pyproject.toml +++ /dev/null @@ -1,56 +0,0 @@ -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "flash-attn-cute" -version = "0.1.0" -description = "Flash Attention CUTE (CUDA Template Engine) implementation" -readme = "README.md" -requires-python = ">=3.10" -license = {text = "BSD 3-Clause License"} -authors = [ - {name = "Tri Dao"}, -] -classifiers = [ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] - -dependencies = [ - "nvidia-cutlass-dsl>=4.4.2,<4.5.0", - "torch", - "einops", - "typing_extensions", - "apache-tvm-ffi>=0.1.5,<0.2", - "torch-c-dlpack-ext", - "quack-kernels==0.2.4", -] - -[project.optional-dependencies] -dev = [ - "pytest", - "ruff", -] - -[project.urls] -Homepage = "https://github.com/Dao-AILab/flash-attention" -Repository = "https://github.com/Dao-AILab/flash-attention" - -[tool.setuptools] -packages = ["flash_attn.cute"] -package-dir = {"flash_attn.cute" = "."} - -[tool.ruff] -line-length = 100 - -[tool.ruff.lint] -ignore = [ - "E731", # do not assign a lambda expression, use a def - "E741", # Do not use variables named 'I', 'O', or 'l' - "F841", # local variable is assigned to but never used -] diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/seqlen_info.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/seqlen_info.py deleted file mode 100644 index 6d8c6feb279b..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/seqlen_info.py +++ /dev/null @@ -1,138 +0,0 @@ -from typing import Optional -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass import Int32, const_expr - -""" -This consolidates all the info related to sequence length. This is so that we can do all -the gmem reads once at the beginning of each tile, rather than having to repeat these reads -to compute various things like n_block_min, n_block_max, etc. -""" - - -@dataclass(frozen=True) -class SeqlenInfo: - offset: cutlass.Int32 - seqlen: cutlass.Int32 - - @staticmethod - def create( - batch_idx: cutlass.Int32, - seqlen_static: cutlass.Int32, - cu_seqlens: Optional[cute.Tensor] = None, - seqused: Optional[cute.Tensor] = None, - ): - offset = 0 if const_expr(cu_seqlens is None) else cu_seqlens[batch_idx] - if const_expr(seqused is not None): - seqlen = seqused[batch_idx] - elif const_expr(cu_seqlens is not None): - seqlen = cu_seqlens[batch_idx + 1] - cu_seqlens[batch_idx] - else: - seqlen = seqlen_static - return SeqlenInfo(offset, seqlen) - - -@dataclass(frozen=True) -class SeqlenInfoQK: - offset_q: cutlass.Int32 - offset_k: cutlass.Int32 - padded_offset_q: cutlass.Int32 - padded_offset_k: cutlass.Int32 - seqlen_q: cutlass.Int32 - seqlen_k: cutlass.Int32 - has_cu_seqlens_q: cutlass.Constexpr[bool] - has_cu_seqlens_k: cutlass.Constexpr[bool] - has_seqused_q: cutlass.Constexpr[bool] - has_seqused_k: cutlass.Constexpr[bool] - - @staticmethod - def create( - batch_idx: cutlass.Int32, - seqlen_q_static: cutlass.Int32, - seqlen_k_static: cutlass.Int32, - mCuSeqlensQ: Optional[cute.Tensor] = None, - mCuSeqlensK: Optional[cute.Tensor] = None, - mSeqUsedQ: Optional[cute.Tensor] = None, - mSeqUsedK: Optional[cute.Tensor] = None, - tile_m: cutlass.Constexpr[cutlass.Int32] = 128, - tile_n: cutlass.Constexpr[cutlass.Int32] = 128, - ): - offset_q = 0 if const_expr(mCuSeqlensQ is None) else mCuSeqlensQ[batch_idx] - offset_k = 0 if const_expr(mCuSeqlensK is None) else mCuSeqlensK[batch_idx] - padded_offset_q = ( - 0 - if const_expr(mCuSeqlensQ is None) - else (offset_q + batch_idx * tile_m) // tile_m * tile_m - ) - padded_offset_k = ( - 0 - if const_expr(mCuSeqlensK is None) - else (offset_k + batch_idx * tile_n) // tile_n * tile_n - ) - if const_expr(mSeqUsedQ is not None): - seqlen_q = mSeqUsedQ[batch_idx] - else: - seqlen_q = ( - seqlen_q_static - if const_expr(mCuSeqlensQ is None) - else mCuSeqlensQ[batch_idx + 1] - offset_q - ) - if const_expr(mSeqUsedK is not None): - seqlen_k = mSeqUsedK[batch_idx] - else: - seqlen_k = ( - seqlen_k_static - if const_expr(mCuSeqlensK is None) - else mCuSeqlensK[batch_idx + 1] - offset_k - ) - has_cu_seqlens_q: int = mCuSeqlensQ is not None - has_cu_seqlens_k: int = mCuSeqlensK is not None - has_seqused_q: int = mSeqUsedQ is not None - has_seqused_k: int = mSeqUsedK is not None - return SeqlenInfoQK( - offset_q, - offset_k, - padded_offset_q, - padded_offset_k, - seqlen_q, - seqlen_k, - has_cu_seqlens_q, - has_cu_seqlens_k, - has_seqused_q, - has_seqused_k, - ) - - def offset_batch_Q( - self, - mQ: cute.Tensor, - batch_idx: Int32, - dim: int, - padded: cutlass.Constexpr[bool] = False, - ) -> cute.Tensor: - """Seqlen must be the first dimension of mQ""" - if const_expr(not self.has_cu_seqlens_q): - idx = (None,) * dim + (batch_idx,) + (None,) * (cute.rank(mQ) - 1 - dim) - return mQ[idx] - else: - offset_q = self.offset_q if const_expr(not padded) else self.padded_offset_q - offset = offset_q if const_expr(cute.rank(mQ.shape[0]) == 1) else (0, offset_q) - idx = (offset,) + (0,) * (cute.rank(mQ) - 1) - return cute.domain_offset(idx, mQ) - - def offset_batch_K( - self, - mK: cute.Tensor, - batch_idx: Int32, - dim: int, - padded: cutlass.Constexpr[bool] = False, - ) -> cute.Tensor: - """Seqlen must be the first dimension of mK""" - if const_expr(not self.has_cu_seqlens_k): - idx = (None,) * dim + (batch_idx,) + (None,) * (cute.rank(mK) - 1 - dim) - return mK[idx] - else: - offset_k = self.offset_k if const_expr(not padded) else self.padded_offset_k - idx = (offset_k,) + (0,) * (cute.rank(mK) - 1) - return cute.domain_offset(idx, mK) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/softmax.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/softmax.py deleted file mode 100644 index 55a9bc11960c..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/softmax.py +++ /dev/null @@ -1,580 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -import math -import operator -from typing import Tuple -from dataclasses import dataclass - -import cutlass -import cutlass.cute as cute -from cutlass import Float32 - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .cute_dsl_utils import ParamsBase -from .seqlen_info import SeqlenInfoQK - - -@dataclass -class Softmax(ParamsBase): - scale_log2: Float32 - num_rows: cutlass.Constexpr[int] - row_max: cute.Tensor - row_sum: cute.Tensor - arch: cutlass.Constexpr[int] = 80 - softmax_scale: Float32 | None = None - - @staticmethod - def create( - scale_log2: Float32, - num_rows: cutlass.Constexpr[int], - arch: cutlass.Constexpr[int] = 80, - softmax_scale: Float32 | None = None, - ): - row_max = cute.make_rmem_tensor(num_rows, Float32) - row_sum = cute.make_rmem_tensor(num_rows, Float32) - return Softmax(scale_log2, num_rows, row_max, row_sum, arch, softmax_scale) - - def reset(self) -> None: - self.row_max.fill(-Float32.inf) - self.row_sum.fill(0.0) - - def _compute_row_max( - self, acc_S_row: cute.TensorSSA, init_val: float | Float32 | None = None - ) -> Float32: - return utils.fmax_reduce(acc_S_row, init_val, arch=self.arch) - - def _compute_row_sum( - self, acc_S_row_exp: cute.TensorSSA, init_val: float | Float32 | None = None - ) -> Float32: - return utils.fadd_reduce(acc_S_row_exp, init_val, arch=self.arch) - - @cute.jit - def online_softmax( - self, - acc_S: cute.Tensor, - is_first: cutlass.Constexpr[bool] = False, - check_inf: cutlass.Constexpr[bool] = True, - ) -> cute.Tensor: - """Apply online softmax and return the row_scale to rescale O. - - :param acc_S: acc_S tensor - :type acc_S: cute.Tensor - :param is_first: is first n_block - :type is_first: cutlass.Constexpr - """ - # Change acc_S to M,N layout view. - acc_S_mn = utils.make_acc_tensor_mn_view(acc_S) - row_scale = cute.make_fragment_like(self.row_max, Float32) - - row_max = self.row_max - row_sum = self.row_sum - scale_log2 = self.scale_log2 - arch = self.arch - - # Each iteration processes one row of acc_S - for r in cutlass.range(cute.size(row_max), unroll_full=True): - acc_S_row = acc_S_mn[r, None].load() # (n_block_size) - - row_max_cur = utils.fmax_reduce( - acc_S_row, - init_val=row_max[r] if cutlass.const_expr(not is_first) else None, - arch=arch, - ) - - row_max_cur = utils.warp_reduce(row_max_cur, cute.arch.fmax, width=4) - if cutlass.const_expr(check_inf): - row_max_cur = 0.0 if row_max_cur == -Float32.inf else row_max_cur - - if cutlass.const_expr(is_first): - row_max_cur_scaled = row_max_cur * scale_log2 - acc_S_row_exp = utils.exp2f(acc_S_row * scale_log2 - row_max_cur_scaled) - - acc_S_row_sum = utils.fadd_reduce(acc_S_row_exp, init_val=None, arch=arch) - row_scale[r] = 1.0 - else: - row_max_prev = row_max[r] - row_max_cur_scaled = row_max_cur * scale_log2 - acc_S_row_exp = utils.exp2f(acc_S_row * scale_log2 - row_max_cur_scaled) - # row_scale[r] = utils.exp2f(row_max_prev * self.scale_log2 - row_max_cur_scaled) - row_scale[r] = utils.exp2f((row_max_prev - row_max_cur) * scale_log2) - - acc_S_row_sum = utils.fadd_reduce( - acc_S_row_exp, init_val=row_sum[r] * row_scale[r], arch=arch - ) - - row_max[r] = row_max_cur - row_sum[r] = acc_S_row_sum - acc_S_mn[r, None].store(acc_S_row_exp) - - return row_scale - - @cute.jit - def finalize( - self, final_scale: Float32 = 1.0, sink_val: Float32 | cute.Tensor | None = None - ) -> cute.Tensor: - """Finalize the online softmax by computing the scale and logsumexp.""" - if cutlass.const_expr(sink_val is not None and isinstance(sink_val, cute.Tensor)): - assert cute.size(sink_val) == cute.size(self.row_sum) - row_sum = self.row_sum - row_max = self.row_max - scale_log2 = self.scale_log2 - - # quad reduction for row_sum as we didn't do it during each iteration of online softmax - row_sum.store(utils.warp_reduce(row_sum.load(), operator.add, width=4)) - row_scale = cute.make_fragment_like(row_max, Float32) - - for r in cutlass.range(cute.size(row_sum), unroll_full=True): - if cutlass.const_expr(sink_val is not None): - sink_val_cur = sink_val if not isinstance(sink_val, cute.Tensor) else sink_val[r] - LOG2_E = math.log2(math.e) - row_sum[r] += utils.exp2f(sink_val_cur * LOG2_E - row_max[r] * scale_log2) - - # if row_sum is zero or nan, set acc_O_mn_row to 1.0 - acc_O_mn_row_is_zero_or_nan = row_sum[r] == 0.0 or row_sum[r] != row_sum[r] - row_scale[r] = ( - cute.arch.rcp_approx(row_sum[r] if not acc_O_mn_row_is_zero_or_nan else 1.0) - ) * final_scale - row_sum_cur = row_sum[r] - LN2 = math.log(2.0) - row_sum[r] = ( - (row_max[r] * scale_log2 + utils.log2f(row_sum_cur)) * LN2 - if not acc_O_mn_row_is_zero_or_nan - else -Float32.inf - ) - return row_scale - - @cute.jit - def rescale_O(self, acc_O: cute.Tensor, row_scale: cute.Tensor) -> None: - """Scale each row of acc_O by the given scale tensor. - :param acc_O: input tensor - :type acc_O: cute.Tensor - :param row_scale: row_scale tensor - :type row_scale: cute.Tensor - """ - acc_O_mn = utils.make_acc_tensor_mn_view(acc_O) - assert cute.size(row_scale) == cute.size(acc_O_mn, mode=[0]) - for r in cutlass.range(cute.size(row_scale), unroll_full=True): - acc_O_mn[r, None].store(acc_O_mn[r, None].load() * row_scale[r]) - - -@dataclass -class SoftmaxSm100(Softmax): - rescale_threshold: cutlass.Constexpr[float] = 0.0 - - @staticmethod - def create( - scale_log2: Float32, - rescale_threshold: cutlass.Constexpr[float] = 0.0, - softmax_scale: Float32 | None = None, - ): - num_rows = 1 - arch = 100 - row_max = cute.make_rmem_tensor(num_rows, Float32) - row_sum = cute.make_rmem_tensor(num_rows, Float32) - return SoftmaxSm100( - scale_log2, - num_rows, - row_max, - row_sum, - arch, - softmax_scale, - rescale_threshold=rescale_threshold, - ) - - @cute.jit - def update_row_max(self, acc_S_row: cute.TensorSSA, is_first: int) -> Tuple[Float32, Float32]: - if cutlass.const_expr(is_first): - row_max_new = self._compute_row_max(acc_S_row) - row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0 - acc_scale = 0.0 - else: - row_max_old = self.row_max[0] - row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old) - row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0 - acc_scale_ = (row_max_old - row_max_safe) * self.scale_log2 - acc_scale = utils.exp2f(acc_scale_) - if cutlass.const_expr(self.rescale_threshold > 0.0): - if acc_scale_ >= -self.rescale_threshold: - row_max_new = row_max_old - row_max_safe = row_max_old - acc_scale = 1.0 - self.row_max[0] = row_max_new - return row_max_safe, acc_scale - - def update_row_sum( - self, acc_S_row_exp: cute.TensorSSA, row_scale: Float32, is_first: int = False - ) -> None: - init_val = self.row_sum[0] * row_scale if cutlass.const_expr(not is_first) else None - # self.row_sum[0] = self._compute_row_sum(acc_S_row_exp, init_val=self.row_sum[0] * row_scale) - self.row_sum[0] = self._compute_row_sum(acc_S_row_exp, init_val=init_val) - # tmp = self._compute_row_sum(acc_S_row_exp) - # self.row_sum[0] = self.row_sum[0] * row_scale + tmp - - @cute.jit - def scale_subtract_rowmax( - self, - acc_S_row: cute.Tensor, - row_max: Float32, - ): - assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements" - row_max_scaled = row_max * self.scale_log2 - for i in cutlass.range(0, cute.size(acc_S_row.shape), 2, unroll_full=True): - acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2( - (acc_S_row[i], acc_S_row[i + 1]), - (self.scale_log2, self.scale_log2), - (-row_max_scaled, -row_max_scaled), - ) - - @cute.jit - def apply_exp2_convert( - self, - acc_S_row: cute.Tensor, - acc_S_row_converted: cute.Tensor, - e2e: cutlass.Constexpr[bool] = False, - e2e_freq: cutlass.Constexpr[int] = 16, - e2e_res: cutlass.Constexpr[int] = 4, - e2e_frg_limit: cutlass.Constexpr[int] = 1, - ): - assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements" - frg_tile = 32 - assert frg_tile % 2 == 0 - frg_cnt = cute.size(acc_S_row) // frg_tile - assert cute.size(acc_S_row) % frg_tile == 0 - acc_S_row_frg = cute.logical_divide(acc_S_row, cute.make_layout(frg_tile)) - acc_S_row_converted_frg = cute.logical_divide( - acc_S_row_converted, cute.make_layout(frg_tile) - ) - for j in cutlass.range_constexpr(frg_cnt): - for k in cutlass.range_constexpr(0, cute.size(acc_S_row_frg, mode=[0]), 2): - # acc_S_row_frg[k, j] = utils.exp2f(acc_S_row_frg[k, j]) - # acc_S_row_frg[k + 1, j] = utils.exp2f(acc_S_row_frg[k + 1, j]) - if cutlass.const_expr(not e2e): - acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j]) - acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j]) - else: - if cutlass.const_expr( - k % e2e_freq < e2e_freq - e2e_res or j >= frg_cnt - e2e_frg_limit - ): - acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j]) - acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j]) - else: - # acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = utils.e2e_asm2(acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j]) - acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = utils.ex2_emulation_2( - acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] - ) - acc_S_row_converted_frg[None, j].store( - acc_S_row_frg[None, j].load().to(acc_S_row_converted.element_type) - ) - - @cute.jit - def scale_apply_exp2_convert( - self, - acc_S_row: cute.Tensor, - row_max: Float32, - acc_S_row_converted: cute.Tensor, - ): - assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements" - minus_row_max_scaled = -row_max * self.scale_log2 - for i in cutlass.range_constexpr(0, cute.size(acc_S_row.shape), 2): - acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2( - (acc_S_row[i], acc_S_row[i + 1]), - (self.scale_log2, self.scale_log2), - (minus_row_max_scaled, minus_row_max_scaled), - ) - - # for i in cutlass.range_constexpr(0, cute.size(acc_S_row.shape), 2): - # acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2( - # (acc_S_row[i], acc_S_row[i + 1]), - # (self.scale_log2, self.scale_log2), - # (minus_row_max_scaled, minus_row_max_scaled), - # ) - # acc_S_row[i] = cute.arch.exp2(acc_S_row[i]) - # acc_S_row[i + 1] = cute.arch.exp2(acc_S_row[i + 1]) - - frg_tile = 32 - assert frg_tile % 2 == 0 - frg_cnt = cute.size(acc_S_row) // frg_tile - assert cute.size(acc_S_row) % frg_tile == 0 - acc_S_row_frg = cute.logical_divide(acc_S_row, cute.make_layout(frg_tile)) - acc_S_row_converted_frg = cute.logical_divide( - acc_S_row_converted, cute.make_layout(frg_tile) - ) - for j in cutlass.range_constexpr(frg_cnt): - for k in cutlass.range_constexpr(0, cute.size(acc_S_row_frg, mode=[0]), 2): - # acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = ( - # utils.fma_packed_f32x2( - # (acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j]), - # (self.scale_log2, self.scale_log2), - # (minus_row_max_scaled, minus_row_max_scaled), - # ) - # ) - # acc_S_row_frg[k, j] = utils.exp2f(acc_S_row_frg[k, j]) - # acc_S_row_frg[k + 1, j] = utils.exp2f(acc_S_row_frg[k + 1, j]) - acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j]) - acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j]) - acc_S_row_converted_frg[None, j].store( - acc_S_row_frg[None, j].load().to(acc_S_row_converted.element_type) - ) - - -@cute.jit -def floor_if_packed( - q_idx, - qhead_per_kvhead: cutlass.Constexpr[int], -) -> cute.Tensor: - """Convert q_idx to packed format for Pack-GQA.""" - if cutlass.const_expr(qhead_per_kvhead == 1): - return q_idx - return q_idx // qhead_per_kvhead - - -@cute.jit -def apply_score_mod_inner( - score_tensor, - index_tensor, - score_mod: cutlass.Constexpr, - batch_idx, - head_idx, - softmax_scale, - vec_size: cutlass.Constexpr, - qk_acc_dtype: cutlass.Constexpr, - aux_tensors, - fastdiv_mods, - seqlen_info: SeqlenInfoQK, - constant_q_idx: cutlass.Constexpr, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, - transpose_indices: cutlass.Constexpr[bool] = False, -): - """Shared implementation for applying score modification. - - Args: - score_tensor: The scores to modify (acc_S for flash_fwd, tSrS_t2r for sm100) - index_tensor: Index positions (tScS for flash_fwd, tScS_t2r for sm100) - score_mod: The score modification function to apply - batch_idx: Batch index - head_idx: Head index - softmax_scale: Scale to apply - vec_size: Vector size for processing elements - qk_acc_dtype: Data type for accumulator - aux_tensors: Optional aux_tensors for FlexAttention - fastdiv_mods: Tuple of (seqlen_q_divmod, seqlen_k_divmod) for wrapping - seqlen_info: Sequence length info - constant_q_idx: If provided, use this constant for all q_idx values - If None, compute q_idx per-element - qhead_per_kvhead_packgqa: Pack-GQA replication factor. Divide q_idx by this - when greater than 1 so score mods see logical heads. - transpose_indices: If True, swap q_idx/kv_idx in index_tensor (for bwd kernel where S is transposed) - """ - # Index positions in the index_tensor tuple - # Forward: index_tensor[...][0] = q_idx, index_tensor[...][1] = kv_idx - # Backward (transposed): index_tensor[...][0] = kv_idx, index_tensor[...][1] = q_idx - if cutlass.const_expr(transpose_indices): - q_idx_pos = cutlass.const_expr(1) - kv_idx_pos = cutlass.const_expr(0) - else: - q_idx_pos = cutlass.const_expr(0) - kv_idx_pos = cutlass.const_expr(1) - - n_vals = cutlass.const_expr(cute.size(score_tensor.shape)) - score_vec = cute.make_rmem_tensor(vec_size, qk_acc_dtype) - kv_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32) - - # SSA values for batch (constant across all elements) - batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32).broadcast_to((vec_size,)) - - # Handle q_idx based on whether it's constant - q_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32) - - # For Pack-GQA with non-constant q_idx, we need per-element head indices - # since a thread my process multiple query head indices - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - head_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32) - - for i in cutlass.range(0, n_vals, vec_size, unroll_full=True): - for j in cutlass.range(vec_size, unroll_full=True): - score_vec[j] = score_tensor[i + j] * softmax_scale - - # Extract head offset from packed q_idx for Pack-GQA - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - q_idx_packed = index_tensor[i + j][q_idx_pos] - # Building up the logical q_head idx: final_q_head = kv_head * qhead_per_kvhead + (q_physical % qhead_per_kvhead) - q_idx_logical = q_idx_packed // qhead_per_kvhead - head_offset = q_idx_packed - q_idx_logical * qhead_per_kvhead - head_idx_vec[j] = head_idx * qhead_per_kvhead + head_offset - - # If we will do loads we mod, in order to not read OOB - if cutlass.const_expr(aux_tensors is not None and fastdiv_mods is not None): - if cutlass.const_expr(constant_q_idx is None): - seqlen_q_divmod, seqlen_k_divmod = fastdiv_mods - q_idx_floored = floor_if_packed( - index_tensor[i + j][q_idx_pos], qhead_per_kvhead - ) - _, q_idx_wrapped = divmod(q_idx_floored, seqlen_q_divmod) - q_idx_vec[j] = q_idx_wrapped - else: - _, seqlen_k_divmod = fastdiv_mods - - _, kv_idx_wrapped = divmod(index_tensor[i + j][kv_idx_pos], seqlen_k_divmod) - kv_idx_vec[j] = kv_idx_wrapped - else: - # No bounds checking - direct indexing - if constant_q_idx is None: - q_idx_vec[j] = floor_if_packed(index_tensor[i + j][q_idx_pos], qhead_per_kvhead) - kv_idx_vec[j] = index_tensor[i + j][kv_idx_pos] - - # Convert to SSA for score_mod call - score_ssa = score_vec.load() - kv_idx_ssa = kv_idx_vec.load() - if cutlass.const_expr(constant_q_idx is None): - q_idx_ssa = q_idx_vec.load() - else: - # NB we do not apply Pack-GQA division here, as constant_q_idx is assumed to already be logical - q_idx_const = constant_q_idx - q_idx_ssa = utils.scalar_to_ssa(q_idx_const, cutlass.Int32).broadcast_to((vec_size,)) - - # Compute head_idx_ssa: per-element for Pack-GQA with non-constant q_idx, constant otherwise - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - head_idx_ssa = head_idx_vec.load() - else: - head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32).broadcast_to((vec_size,)) - - aux_args = [] - if cutlass.const_expr(aux_tensors is not None): - aux_args = aux_tensors - - post_mod_scores = score_mod( - score_ssa, - batch_idx_ssa, - head_idx_ssa, - q_idx=q_idx_ssa, - kv_idx=kv_idx_ssa, - seqlen_info=seqlen_info, - aux_tensors=aux_args, - ) - - # Write back modified scores - score_vec.store(post_mod_scores) - for j in cutlass.range(vec_size, unroll_full=True): - score_tensor[i + j] = score_vec[j] - - -@cute.jit -def apply_score_mod_bwd_inner( - grad_tensor, - score_tensor, - index_tensor, - score_mod_bwd: cutlass.Constexpr, - batch_idx, - head_idx, - softmax_scale, - vec_size: cutlass.Constexpr, - qk_acc_dtype: cutlass.Constexpr, - aux_tensors, - fastdiv_mods, - seqlen_info, - constant_q_idx: cutlass.Constexpr, - qhead_per_kvhead: cutlass.Constexpr[int] = 1, - transpose_indices: cutlass.Constexpr[bool] = False, -): - """Apply backward score modification (joint graph). - - Args: - grad_tensor: in/out: dlogits rewritten in-place with d(scaled_scores) - score_tensor: pre-mod scores (unscaled QK tile), scaled by softmax_scale internally - index_tensor: Index positions (same as forward) - score_mod_bwd: The backward score modification function (joint graph) - batch_idx: Batch index - head_idx: Head index - softmax_scale: Scale to apply to score_tensor - vec_size: Vector size for processing elements - qk_acc_dtype: Data type for accumulator - aux_tensors: Optional aux_tensors for FlexAttention - fastdiv_mods: Tuple of (seqlen_q_divmod, seqlen_k_divmod) for wrapping - seqlen_info: Sequence length info - constant_q_idx: If provided, use this constant for all q_idx values - qhead_per_kvhead: Pack-GQA replication factor - transpose_indices: If True, swap q_idx/kv_idx in index_tensor - """ - # Index positions in the index_tensor tuple - # Forward: index_tensor[...][0] = q_idx, index_tensor[...][1] = kv_idx - # Backward (transposed): index_tensor[...][0] = kv_idx, index_tensor[...][1] = q_idx - if cutlass.const_expr(transpose_indices): - q_idx_pos = cutlass.const_expr(1) - kv_idx_pos = cutlass.const_expr(0) - else: - q_idx_pos = cutlass.const_expr(0) - kv_idx_pos = cutlass.const_expr(1) - n_vals = cutlass.const_expr(cute.size(grad_tensor.shape)) - grad_vec = cute.make_fragment(vec_size, qk_acc_dtype) - score_vec = cute.make_fragment(vec_size, qk_acc_dtype) - kv_idx_vec = cute.make_fragment(vec_size, cutlass.Int32) - batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32).broadcast_to((vec_size,)) - q_idx_vec = cute.make_fragment(vec_size, cutlass.Int32) - - # For Pack-GQA with non-constant q_idx, we need per-element head indices - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - head_idx_vec = cute.make_fragment(vec_size, cutlass.Int32) - - for i in cutlass.range(0, n_vals, vec_size, unroll_full=True): - for j in cutlass.range(vec_size, unroll_full=True): - grad_vec[j] = grad_tensor[i + j] - # Scale score so joint graph sees same value as forward score_mod - score_vec[j] = score_tensor[i + j] * softmax_scale - - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - q_idx_packed = index_tensor[i + j][q_idx_pos] - q_idx_logical = q_idx_packed // qhead_per_kvhead - head_offset = q_idx_packed - q_idx_logical * qhead_per_kvhead - head_idx_vec[j] = head_idx * qhead_per_kvhead + head_offset - - if cutlass.const_expr(aux_tensors is not None and fastdiv_mods is not None): - if cutlass.const_expr(constant_q_idx is None): - seqlen_q_divmod, seqlen_k_divmod = fastdiv_mods - q_idx_floored = floor_if_packed( - index_tensor[i + j][q_idx_pos], qhead_per_kvhead - ) - _, q_idx_wrapped = divmod(q_idx_floored, seqlen_q_divmod) - q_idx_vec[j] = q_idx_wrapped - else: - _, seqlen_k_divmod = fastdiv_mods - - _, kv_idx_wrapped = divmod(index_tensor[i + j][kv_idx_pos], seqlen_k_divmod) - kv_idx_vec[j] = kv_idx_wrapped - else: - # No bounds checking - direct indexing - if constant_q_idx is None: - q_idx_vec[j] = floor_if_packed(index_tensor[i + j][q_idx_pos], qhead_per_kvhead) - kv_idx_vec[j] = index_tensor[i + j][kv_idx_pos] - - grad_ssa = grad_vec.load() - score_ssa = score_vec.load() - kv_idx_ssa = kv_idx_vec.load() - - if cutlass.const_expr(constant_q_idx is None): - q_idx_ssa = q_idx_vec.load() - else: - q_idx_ssa = utils.scalar_to_ssa(constant_q_idx, cutlass.Int32).broadcast_to((vec_size,)) - - if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None): - head_idx_ssa = head_idx_vec.load() - else: - head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32).broadcast_to((vec_size,)) - - aux_args = [] - if cutlass.const_expr(aux_tensors is not None): - aux_args = aux_tensors - - grad_out_ssa = score_mod_bwd( - grad_ssa, - score_ssa, - batch_idx_ssa, - head_idx_ssa, - q_idx=q_idx_ssa, - kv_idx=kv_idx_ssa, - seqlen_info=seqlen_info, - aux_tensors=aux_args, - ) - - grad_vec.store(grad_out_ssa) - for j in cutlass.range(vec_size, unroll_full=True): - grad_tensor[i + j] = grad_vec[j] diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/testing.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/testing.py deleted file mode 100644 index 2897e64fc3dc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/testing.py +++ /dev/null @@ -1,423 +0,0 @@ -import math -from typing import Optional - -import torch -import torch.nn.functional as F -from einops import rearrange, repeat - - -class IndexFirstAxis(torch.autograd.Function): - @staticmethod - def forward(ctx, input, indices): - ctx.save_for_backward(indices) - assert input.ndim >= 2 - ctx.first_axis_dim, other_shape = input.shape[0], input.shape[1:] - second_dim = other_shape.numel() - return torch.gather( - rearrange(input, "b ... -> b (...)"), - 0, - repeat(indices, "z -> z d", d=second_dim), - ).reshape(-1, *other_shape) - - @staticmethod - def backward(ctx, grad_output): - (indices,) = ctx.saved_tensors - assert grad_output.ndim >= 2 - other_shape = grad_output.shape[1:] - grad_output = rearrange(grad_output, "b ... -> b (...)") - grad_input = torch.zeros( - [ctx.first_axis_dim, grad_output.shape[1]], - device=grad_output.device, - dtype=grad_output.dtype, - ) - grad_input.scatter_(0, repeat(indices, "z -> z d", d=grad_output.shape[1]), grad_output) - return grad_input.reshape(ctx.first_axis_dim, *other_shape), None - - -index_first_axis = IndexFirstAxis.apply - - -class IndexPutFirstAxis(torch.autograd.Function): - @staticmethod - def forward(ctx, values, indices, first_axis_dim): - ctx.save_for_backward(indices) - assert indices.ndim == 1 - assert values.ndim >= 2 - output = torch.zeros( - first_axis_dim, *values.shape[1:], device=values.device, dtype=values.dtype - ) - output[indices] = values - return output - - @staticmethod - def backward(ctx, grad_output): - (indices,) = ctx.saved_tensors - grad_values = grad_output[indices] - return grad_values, None, None - - -index_put_first_axis = IndexPutFirstAxis.apply - - -def unpad_input(hidden_states, attention_mask, unused_mask=None): - all_masks = (attention_mask + unused_mask) if unused_mask is not None else attention_mask - seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) - used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) - indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() - max_seqlen_in_batch = seqlens_in_batch.max().item() - cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) - return ( - index_first_axis(rearrange(hidden_states, "b s ... -> (b s) ..."), indices), - indices, - cu_seqlens, - max_seqlen_in_batch, - used_seqlens_in_batch, - ) - - -def pad_input(hidden_states, indices, batch, seqlen): - output = index_put_first_axis(hidden_states, indices, batch * seqlen) - return rearrange(output, "(b s) ... -> b s ...", b=batch) - - -def generate_random_padding_mask(max_seqlen, batch_size, device, mode="random", zero_lengths=False): - assert mode in ["full", "random", "third"] - if mode == "full": - lengths = torch.full((batch_size, 1), max_seqlen, device=device, dtype=torch.int32) - elif mode == "random": - lengths = torch.randint( - max(0 if zero_lengths else 1, max_seqlen - 20), - max_seqlen + 1, - (batch_size, 1), - device=device, - ) - else: - lengths = torch.randint( - max(0 if zero_lengths else 1, max_seqlen // 3), - max_seqlen + 1, - (batch_size, 1), - device=device, - ) - - if zero_lengths: - for i in range(batch_size): - if i % 5 == 0: - lengths[i] = 0 - lengths[-1] = 0 - padding_mask = ( - repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) < lengths - ) - return padding_mask - - -def generate_qkv( - q, - k, - v, - query_padding_mask=None, - key_padding_mask=None, - qv=None, - kvpacked=False, - qkvpacked=False, - query_unused_mask=None, - key_unused_mask=None, -): - assert not (kvpacked and qkvpacked) - batch_size, seqlen_q, nheads, d = q.shape - d_v = v.shape[-1] - _, seqlen_k, nheads_k, _ = k.shape - assert k.shape == (batch_size, seqlen_k, nheads_k, d) - assert v.shape == (batch_size, seqlen_k, nheads_k, d_v) - if query_unused_mask is not None or key_unused_mask is not None: - assert not kvpacked - assert not qkvpacked - - if query_padding_mask is not None: - q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( - q, query_padding_mask, query_unused_mask - ) - output_pad_fn = lambda output_unpad: pad_input( - output_unpad, indices_q, batch_size, seqlen_q - ) - qv_unpad = rearrange(qv, "b s ... -> (b s) ...")[indices_q] if qv is not None else None - else: - q_unpad = rearrange(q, "b s h d -> (b s) h d") - cu_seqlens_q = torch.arange( - 0, (batch_size + 1) * seqlen_q, step=seqlen_q, dtype=torch.int32, device=q_unpad.device - ) - seqused_q = None - max_seqlen_q = seqlen_q - output_pad_fn = lambda output_unpad: rearrange( - output_unpad, "(b s) h d -> b s h d", b=batch_size - ) - qv_unpad = rearrange(qv, "b s ... -> (b s) ...") if qv is not None else None - - if key_padding_mask is not None: - k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( - k, key_padding_mask, key_unused_mask - ) - v_unpad, *_ = unpad_input(v, key_padding_mask, key_unused_mask) - else: - k_unpad = rearrange(k, "b s h d -> (b s) h d") - v_unpad = rearrange(v, "b s h d -> (b s) h d") - cu_seqlens_k = torch.arange( - 0, (batch_size + 1) * seqlen_k, step=seqlen_k, dtype=torch.int32, device=k_unpad.device - ) - seqused_k = None - max_seqlen_k = seqlen_k - - if qkvpacked: - assert (query_padding_mask == key_padding_mask).all() - assert nheads == nheads_k - qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) - qkv = torch.stack([q, k, v], dim=2) - if query_padding_mask is not None: - dqkv_pad_fn = lambda dqkv_unpad: pad_input(dqkv_unpad, indices_q, batch_size, seqlen_q) - else: - dqkv_pad_fn = lambda dqkv_unpad: rearrange( - dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size - ) - return ( - qkv_unpad.detach().requires_grad_(), - cu_seqlens_q, - max_seqlen_q, - qkv.detach().requires_grad_(), - output_pad_fn, - dqkv_pad_fn, - ) - elif kvpacked: - kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) - kv = torch.stack([k, v], dim=2) - dq_pad_fn = output_pad_fn - if key_padding_mask is not None: - dkv_pad_fn = lambda dkv_unpad: pad_input(dkv_unpad, indices_k, batch_size, seqlen_k) - else: - dkv_pad_fn = lambda dkv_unpad: rearrange( - dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size - ) - return ( - q_unpad.detach().requires_grad_(), - kv_unpad.detach().requires_grad_(), - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - q.detach().requires_grad_(), - kv.detach().requires_grad_(), - output_pad_fn, - dq_pad_fn, - dkv_pad_fn, - ) - else: - dq_pad_fn = output_pad_fn - if key_padding_mask is not None: - dk_pad_fn = lambda dk_unpad: pad_input(dk_unpad, indices_k, batch_size, seqlen_k) - else: - dk_pad_fn = lambda dk_unpad: rearrange(dk_unpad, "(b s) h d -> b s h d", b=batch_size) - return ( - q_unpad.detach().requires_grad_(), - k_unpad.detach().requires_grad_(), - v_unpad.detach().requires_grad_(), - qv_unpad.detach() if qv is not None else None, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - max_seqlen_q, - max_seqlen_k, - q.detach().requires_grad_(), - k.detach().requires_grad_(), - v.detach().requires_grad_(), - qv.detach() if qv is not None else None, - output_pad_fn, - dq_pad_fn, - dk_pad_fn, - ) - - -def construct_local_mask( - seqlen_q, - seqlen_k, - window_size=(None, None), - sink_token_length=0, - query_padding_mask=None, - key_padding_mask=None, - key_leftpad=None, - device=None, -): - row_idx = rearrange(torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1") - col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) - if key_leftpad is not None: - key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") - col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) - col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) - sk = ( - seqlen_k - if key_padding_mask is None - else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sq = ( - seqlen_q - if query_padding_mask is None - else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") - ) - if window_size[0] is None: - return col_idx > row_idx + sk - sq + window_size[1] - else: - sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk - if window_size[1] is None: - local_mask_left = col_idx > sk - else: - local_mask_left = col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk) - return torch.logical_or( - local_mask_left, - torch.logical_and( - col_idx < row_idx + sk - sq - window_size[0], col_idx >= sink_token_length - ), - ) - - -def construct_chunk_mask( - seqlen_q, - seqlen_k, - attention_chunk, - query_padding_mask=None, - key_padding_mask=None, - key_leftpad=None, - device=None, -): - row_idx = rearrange(torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1") - col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) - if key_leftpad is not None: - key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") - col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) - col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) - sk = ( - seqlen_k - if key_padding_mask is None - else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sq = ( - seqlen_q - if query_padding_mask is None - else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk - col_limit_left_chunk = row_idx + sk - sq - (row_idx + sk - sq) % attention_chunk - return torch.logical_or( - col_idx < col_limit_left_chunk, col_idx >= col_limit_left_chunk + attention_chunk - ) - - -def attention_ref( - q, - k, - v, - query_padding_mask=None, - key_padding_mask=None, - key_leftpad=None, - attn_bias=None, - dropout_p=0.0, - dropout_mask=None, - causal=False, - qv=None, - q_descale=None, - k_descale=None, - v_descale=None, - window_size=(None, None), - attention_chunk=0, - sink_token_length=0, - learnable_sink: Optional[torch.Tensor] = None, - softcap=0.0, - upcast=True, - reorder_ops=False, - intermediate_dtype=None, -): - if causal: - window_size = (window_size[0], 0) - dtype_og = q.dtype - if upcast: - q, k, v = q.float(), k.float(), v.float() - qv = qv.float() if qv is not None else None - if q_descale is not None: - q_descale = repeat(q_descale, "b h -> b 1 (h g) 1", g=q.shape[2] // k.shape[2]) - q = (q.float() * q_descale).to(q.dtype) - qv = (qv.float() * q_descale).to(qv.dtype) if qv is not None else None - if k_descale is not None: - k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) - if v_descale is not None: - v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) - seqlen_q, seqlen_k = q.shape[1], k.shape[1] - k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) - v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) - d = q.shape[-1] - dv = v.shape[-1] - softmax_scale = 1.0 / math.sqrt(d if qv is None else d + dv) - if not reorder_ops: - scores = torch.einsum("bthd,bshd->bhts", q * softmax_scale, k) - else: - scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale) - if qv is not None: - scores = scores + torch.einsum("bthd,bshd->bhts", qv * softmax_scale, v) - if softcap > 0: - scores = torch.tanh(scores / softcap) * softcap - if key_padding_mask is not None: - scores.masked_fill_(rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) - local_mask = None - if window_size[0] is not None or window_size[1] is not None: - local_mask = construct_local_mask( - seqlen_q, - seqlen_k, - window_size, - sink_token_length, - query_padding_mask, - key_padding_mask, - key_leftpad=key_leftpad, - device=q.device, - ) - if attention_chunk > 0: - chunk_mask = construct_chunk_mask( - seqlen_q, - seqlen_k, - attention_chunk, - query_padding_mask, - key_padding_mask, - key_leftpad=key_leftpad, - device=q.device, - ) - local_mask = ( - torch.logical_or(local_mask, chunk_mask) if local_mask is not None else chunk_mask - ) - if local_mask is not None: - scores.masked_fill_(local_mask, float("-inf")) - if attn_bias is not None: - scores = scores + attn_bias - if learnable_sink is None: - attention = torch.softmax(scores, dim=-1).to(v.dtype) - else: - scores_fp32 = scores.to(torch.float32) - logits_max = torch.amax(scores_fp32, dim=-1, keepdim=True) - learnable_sink = rearrange(learnable_sink, "h -> h 1 1") - logits_or_sinks_max = torch.maximum(learnable_sink, logits_max) - unnormalized_scores = torch.exp(scores_fp32 - logits_or_sinks_max) - normalizer = unnormalized_scores.sum(dim=-1, keepdim=True) + torch.exp( - learnable_sink - logits_or_sinks_max - ) - attention = (unnormalized_scores / normalizer).to(v.dtype) - if query_padding_mask is not None: - attention = attention.masked_fill(rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0) - if key_padding_mask is not None: - attention = attention.masked_fill(rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0) - if local_mask is not None: - attention = attention.masked_fill(torch.all(local_mask, dim=-1, keepdim=True), 0.0) - dropout_scaling = 1.0 / (1 - dropout_p) - if dropout_mask is not None: - attention_drop = attention.masked_fill(~dropout_mask, 0.0) - else: - attention_drop = attention - if intermediate_dtype is not None: - attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) - output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) - if query_padding_mask is not None: - output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) - return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/tile_scheduler.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/tile_scheduler.py deleted file mode 100644 index d849c50407e0..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/tile_scheduler.py +++ /dev/null @@ -1,719 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -from typing import Optional, Tuple -from dataclasses import dataclass, fields - -try: - from typing import override -except ImportError: # Python < 3.12 - from typing_extensions import override - -import cutlass -from cutlass._mlir import ir -import cutlass.cute as cute -from cutlass import Int32, const_expr - -import tensorrt_llm._torch.visual_gen.jit_kernels.flash_attention.cute.utils as utils -from .fast_math import clz -from cutlass.cute import FastDivmodDivisor - - -class WorkTileInfo(cutlass.utils.WorkTileInfo): - """Altered WorkTileInfo which includes four axes: (block, head, batch, split)""" - - @override - def __new_from_mlir_values__(self, values: list[ir.Value]) -> "WorkTileInfo": - assert len(values) == 5 - new_tile_idx = cutlass.new_from_mlir_values(self._tile_idx, values[:-1]) - new_is_valid_tile = cutlass.new_from_mlir_values(self._is_valid_tile, [values[-1]]) - return WorkTileInfo(new_tile_idx, new_is_valid_tile) - - -@dataclass -class ParamsBase: - def __extract_mlir_values__(self): - all_fields = [getattr(self, field.name) for field in fields(self)] - non_constexpr_fields = [f for f in all_fields if not isinstance(f, cutlass.Constexpr)] - values, self._values_pos = [], [] - for obj in non_constexpr_fields: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - all_fields = {field.name: getattr(self, field.name) for field in fields(self)} - constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, cutlass.Constexpr)} - non_constexpr_fields = { - n: f for n, f in all_fields.items() if not isinstance(f, cutlass.Constexpr) - } - for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos): - non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items]) - values = values[n_items:] - return self.__class__(**non_constexpr_fields, **constexpr_fields) - - -@dataclass -class TileSchedulerArguments(ParamsBase): - num_block: Int32 - num_head: Int32 - num_batch: Int32 - num_splits: Int32 - seqlen_k: Int32 - headdim: Int32 - headdim_v: Int32 - total_q: Int32 - tile_shape_mn: cutlass.Constexpr[Tuple[int, int]] - cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1) - mCuSeqlensQ: Optional[cute.Tensor] = None - mSeqUsedQ: Optional[cute.Tensor] = None - qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1 - element_size: cutlass.Constexpr[int] = 2 - is_persistent: cutlass.Constexpr[bool] = False - lpt: cutlass.Constexpr[bool] = False - is_split_kv: cutlass.Constexpr[bool] = False - head_swizzle: cutlass.Constexpr[bool] = False - - -class SingleTileScheduler: - @dataclass - class Params(ParamsBase): - num_block: Int32 - num_head: Int32 - num_batch: Int32 - num_splits: Int32 - num_splits_divmod: FastDivmodDivisor - is_split_kv: cutlass.Constexpr[bool] = False - cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1) - - @staticmethod - def create( - args: TileSchedulerArguments, *, loc=None, ip=None - ) -> "SingleTileScheduler.Params": - return SingleTileScheduler.Params( - args.num_block, - args.num_head, - args.num_batch, - args.num_splits, - FastDivmodDivisor(args.num_splits), - args.is_split_kv, - args.cluster_shape_mn, - ) - - def __init__(self, params: Params, blk_coord: cute.Coord, *, loc=None, ip=None): - self.params = params - self._blk_coord = blk_coord - self._is_first_block = True - self._loc = loc - self._ip = ip - - @staticmethod - def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params: - return SingleTileScheduler.Params.create(args, loc=loc, ip=ip) - - @staticmethod - def create(params: Params, *, loc=None, ip=None) -> "SingleTileScheduler": - blk_coord = cute.arch.block_idx() - return SingleTileScheduler(params, blk_coord, loc=loc, ip=ip) - - # called by host - @staticmethod - def get_grid_shape( - params: Params, - *, - loc=None, - ip=None, - ) -> Tuple[Int32, Int32, Int32]: - # TODO: this hard-codes the fact that we only use cluster = (1, 1) or (2, 1) - assert params.cluster_shape_mn[1] == 1, "Only cluster_shape_mn[1] == 1 is supported" - return ( - cute.round_up(params.num_block, params.cluster_shape_mn[0]), - params.num_head * params.num_splits, - params.num_batch, - ) - - def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: - block_idx, head_idx, batch_idx = self._blk_coord - if const_expr(self.params.is_split_kv): - head_idx, split_idx = divmod(head_idx, self.params.num_splits_divmod) - else: - split_idx = Int32(0) - return WorkTileInfo( - (block_idx, head_idx, batch_idx, split_idx), - self._is_first_block, - ) - - def initial_work_tile_info(self, *, loc=None, ip=None): - return self.get_current_work(loc=loc, ip=ip) - - def prefetch_next_work(self, *, loc=None, ip=None): - pass - - def advance_to_next_work(self, *, loc=None, ip=None): - self._is_first_block = False - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.params, self._blk_coord]: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip([self.params, self._blk_coord], self._values_pos): - obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return SingleTileScheduler(*(tuple(obj_list)), loc=self._loc) - - -class StaticPersistentTileScheduler: - @dataclass - class Params(ParamsBase): - num_block_divmod: FastDivmodDivisor - num_head_divmod: FastDivmodDivisor - total_blocks: Int32 - - @staticmethod - def create( - args: TileSchedulerArguments, *, loc=None, ip=None - ) -> "StaticPersistentTileScheduler.Params": - total_blocks = args.num_block * args.num_head * args.num_batch - return StaticPersistentTileScheduler.Params( - FastDivmodDivisor(args.num_block), FastDivmodDivisor(args.num_head), total_blocks - ) - - def __init__(self, params: Params, tile_idx: Int32, *, loc=None, ip=None): - self.params = params - self._tile_idx = tile_idx - self._loc = loc - self._ip = ip - - @staticmethod - def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params: - return StaticPersistentTileScheduler.Params.create(args, loc=loc, ip=ip) - - @staticmethod - def create(params: Params, *, loc=None, ip=None) -> "StaticPersistentTileScheduler": - tile_idx = cute.arch.block_idx()[0] - return StaticPersistentTileScheduler(params, tile_idx, loc=loc, ip=ip) - - # called by host - @staticmethod - def get_grid_shape( - params: Params, - *, - loc=None, - ip=None, - ) -> Tuple[Int32, Int32, Int32]: - hardware_info = cutlass.utils.HardwareInfo() - sm_count = hardware_info.get_device_multiprocessor_count() - return (cutlass.min(sm_count, params.total_blocks), Int32(1), Int32(1)) - - # @cute.jit - def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: - hn_idx, block_idx = divmod(self._tile_idx, self.params.num_block_divmod) - batch_idx, head_idx = divmod(hn_idx, self.params.num_head_divmod) - is_valid = self._tile_idx < self.params.total_blocks - # if cute.arch.thread_idx()[0] == 0: - # cute.printf("TileScheduler: tile_idx=%d, hn_idx=%d, block_idx=%d, batch_idx=%d, head_idx=%d, is_valid=%d", self._tile_idx, hn_idx, block_idx, batch_idx, head_idx, is_valid) - return WorkTileInfo( - (Int32(block_idx), Int32(head_idx), Int32(batch_idx), Int32(0)), is_valid - ) - - def initial_work_tile_info(self, *, loc=None, ip=None): - return self.get_current_work(loc=loc, ip=ip) - - def prefetch_next_work(self, *, loc=None, ip=None): - pass - - def advance_to_next_work(self, *, loc=None, ip=None): - self._tile_idx += cute.arch.grid_dim()[0] - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.params, self._tile_idx]: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip( - [self.params, self._tile_idx], - self._values_pos, - ): - obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return StaticPersistentTileScheduler(*(tuple(obj_list)), loc=self._loc) - - -class SingleTileLPTScheduler: - @dataclass - class Params(ParamsBase): - total_blocks: Int32 - num_splits: Int32 - num_block: Int32 - l2_minor: Int32 - num_block_divmod: FastDivmodDivisor - num_head_divmod: FastDivmodDivisor - l2_minor_divmod: FastDivmodDivisor - l2_major_divmod: FastDivmodDivisor - l2_minor_residual_divmod: FastDivmodDivisor - num_hb_quotient: Int32 - is_split_kv: cutlass.Constexpr[bool] = False - - @staticmethod - @cute.jit - def create( - args: TileSchedulerArguments, *, loc=None, ip=None - ) -> "SingleTileLPTScheduler.Params": - # cute.printf(args.num_block, args.num_head, args.num_batch, args.seqlen_k, args.headdim, args.headdim_v, args.total_q, args.tile_shape_mn, args.qhead_per_kvhead_packgqa, args.element_size) - size_one_kv_head = args.seqlen_k * (args.headdim + args.headdim_v) * args.element_size - size_one_head = size_one_kv_head - size_l2 = 50 * 1024 * 1024 # 40 MB for K & V - # Swizzle is the size of each "section". Round swizzle to a power of 2 - # Need to be careful about the case where only one head will fit - # swizzle is how many heads can fit in L2 - # swizzle = 1 if size_l2 < size_one_head else (size_l2 // size_one_head) - # Seems faster if swizzle if a power of 2 - log2_floor = lambda n: 31 - clz(n) - swizzle = 1 if size_l2 < size_one_head else (1 << log2_floor(size_l2 // size_one_head)) - # swizzle = 1 if size_l2 < size_one_head else (size_l2 // size_one_head) - # If we're in the last section (called residual), we don't want to divide by - # swizzle. Instead we want to divide by the remainder. - num_hb_quotient = (args.num_head * args.num_batch) // swizzle - num_hb_remainder = (args.num_head * args.num_batch) % swizzle - return SingleTileLPTScheduler.Params( - total_blocks=args.num_block * args.num_head * args.num_batch, - num_block=args.num_block, - l2_minor=Int32(swizzle), - num_block_divmod=FastDivmodDivisor(args.num_block), - num_head_divmod=FastDivmodDivisor(args.num_head), - l2_minor_divmod=FastDivmodDivisor(swizzle), - l2_major_divmod=FastDivmodDivisor(swizzle * args.num_block), - l2_minor_residual_divmod=FastDivmodDivisor( - max(num_hb_remainder, 1) - ), # don't divide by 0 - num_hb_quotient=Int32(num_hb_quotient), - num_splits=args.num_splits, - is_split_kv=args.is_split_kv, - ) - - def __init__(self, params: Params, tile_idx: Int32, split_idx: Int32, *, loc=None, ip=None): - self.params = params - self._tile_idx = tile_idx - self._split_idx = split_idx - self._loc = loc - self._ip = ip - - @staticmethod - def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params: - return SingleTileLPTScheduler.Params.create(args, loc=loc, ip=ip) - - @staticmethod - @cute.jit - def create(params: Params, *, loc=None, ip=None) -> "SingleTileLPTScheduler": - tile_idx, split_idx, _ = cute.arch.block_idx() - return SingleTileLPTScheduler(params, tile_idx, split_idx, loc=loc, ip=ip) - - # called by host - @staticmethod - def get_grid_shape( - params: Params, - *, - loc=None, - ip=None, - ) -> Tuple[Int32, Int32, Int32]: - return (params.total_blocks, params.num_splits, Int32(1)) - - @cute.jit - def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: - params = self.params - # Implement LPT scheduling coordinate calculation - bidhb, l2_mod = divmod(self._tile_idx, params.l2_major_divmod) - # If we're in the last section (called residual), we don't want to divide by - # swizzle. Instead we want to divide by the remainder. - block, bidhb_residual = 0, 0 - if bidhb < params.num_hb_quotient: - block, bidhb_residual = divmod(l2_mod, params.l2_minor_divmod) - else: - block, bidhb_residual = divmod(l2_mod, params.l2_minor_residual_divmod) - bidhb_actual = bidhb * params.l2_minor + bidhb_residual - batch_idx, head_idx = divmod(bidhb_actual, params.num_head_divmod) - # Longest-processing-time-first - block = params.num_block - 1 - block - is_valid = self._tile_idx < params.total_blocks - return WorkTileInfo( - (Int32(block), Int32(head_idx), Int32(batch_idx), Int32(self._split_idx)), is_valid - ) - - def initial_work_tile_info(self, *, loc=None, ip=None): - return self.get_current_work(loc=loc, ip=ip) - - def prefetch_next_work(self, *, loc=None, ip=None): - pass - - def advance_to_next_work(self, *, loc=None, ip=None): - # Single tile scheduler - set to invalid tile_idx to indicate no more work - self._tile_idx = self.params.total_blocks - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.params, self._tile_idx, self._split_idx]: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip([self.params, self._tile_idx, self._split_idx], self._values_pos): - obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return self.__class__(*(tuple(obj_list)), loc=self._loc) - - -class SingleTileLPTBwdScheduler: - @dataclass - class Params(ParamsBase): - total_blocks: Int32 - num_block: Int32 - l2_minor: Int32 - num_head_divmod: FastDivmodDivisor - l2_minor_divmod: FastDivmodDivisor - l2_major_divmod: FastDivmodDivisor - l2_minor_residual_divmod: FastDivmodDivisor - num_hb_quotient: Int32 - cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1) - spt: cutlass.Constexpr[bool] = True - - @staticmethod - @cute.jit - def create( - args: TileSchedulerArguments, *, loc=None, ip=None - ) -> "SingleTileLPTBwdScheduler.Params": - size_l2 = 50 * 1024 * 1024 - size_one_qdo_head = args.seqlen_k * (args.headdim + args.headdim_v) * args.element_size - # size_one_dqaccum_head = args.seqlen_k * (args.headdim) * 4 - size_one_dqaccum_head = 0 - size_one_head = size_one_qdo_head + size_one_dqaccum_head - log2_floor = lambda n: 31 - clz(n) - swizzle = 1 if size_l2 < size_one_head else (1 << log2_floor(size_l2 // size_one_head)) - # swizzle = 8 - # If we're in the last section (called residual), we don't want to divide by - # swizzle. Instead we want to divide by the remainder. - num_hb_quotient = (args.num_head * args.num_batch) // swizzle - num_hb_remainder = (args.num_head * args.num_batch) % swizzle - num_block = cute.ceil_div(args.num_block, args.cluster_shape_mn[0]) - return SingleTileLPTBwdScheduler.Params( - total_blocks=(num_block * args.cluster_shape_mn[0]) - * args.num_head - * args.num_batch, - num_block=num_block, - l2_minor=Int32(swizzle), - num_head_divmod=FastDivmodDivisor(args.num_head), - l2_minor_divmod=FastDivmodDivisor(swizzle), - l2_major_divmod=FastDivmodDivisor(swizzle * num_block), - l2_minor_residual_divmod=FastDivmodDivisor( - max(num_hb_remainder, 1) - ), # don't divide by 0 - num_hb_quotient=Int32(num_hb_quotient), - cluster_shape_mn=args.cluster_shape_mn, - spt=args.lpt, - ) - - def __init__(self, params: Params, tile_idx: Int32, *, loc=None, ip=None): - self.params = params - self._tile_idx = tile_idx - self._loc = loc - self._ip = ip - - @staticmethod - def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params: - return SingleTileLPTBwdScheduler.Params.create(args, loc=loc, ip=ip) - - @staticmethod - @cute.jit - def create(params: Params, *, loc=None, ip=None) -> "SingleTileLPTBwdScheduler": - tile_idx = cute.arch.block_idx()[0] - return SingleTileLPTBwdScheduler(params, tile_idx, loc=loc, ip=ip) - - # called by host - @staticmethod - def get_grid_shape( - params: Params, - *, - loc=None, - ip=None, - ) -> Tuple[Int32, Int32, Int32]: - return (params.total_blocks, Int32(1), Int32(1)) - - @cute.jit - def get_current_work(self, *, loc=None, ip=None) -> cutlass.utils.WorkTileInfo: - cluster_idx = self._tile_idx // self.params.cluster_shape_mn[0] - params = self.params - # Implement LPT scheduling coordinate calculation - bidhb, l2_mod = divmod(cluster_idx, params.l2_major_divmod) - # If we're in the last section (called residual), we don't want to divide by - # swizzle. Instead we want to divide by the remainder. - block, bidhb_residual = 0, 0 - if bidhb < params.num_hb_quotient: - block, bidhb_residual = divmod(l2_mod, params.l2_minor_divmod) - else: - block, bidhb_residual = divmod(l2_mod, params.l2_minor_residual_divmod) - bidhb_actual = bidhb * params.l2_minor + bidhb_residual - batch_idx, head_idx = divmod(bidhb_actual, params.num_head_divmod) - is_valid = self._tile_idx < params.total_blocks - bidx_in_cluster = cute.arch.block_in_cluster_idx() - block = block * params.cluster_shape_mn[0] + bidx_in_cluster[0] - if cutlass.const_expr(params.spt): - block = params.num_block - 1 - block - return WorkTileInfo((Int32(block), Int32(head_idx), Int32(batch_idx), Int32(0)), is_valid) - - def initial_work_tile_info(self, *, loc=None, ip=None): - return self.get_current_work(loc=loc, ip=ip) - - def prefetch_next_work(self, *, loc=None, ip=None): - pass - - def advance_to_next_work(self, *, loc=None, ip=None): - # Single tile scheduler - set to invalid tile_idx to indicate no more work - self._tile_idx = self.params.total_blocks - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.params, self._tile_idx]: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip([self.params, self._tile_idx], self._values_pos): - obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return self.__class__(*(tuple(obj_list)), loc=self._loc) - - -class SingleTileVarlenScheduler: - @dataclass - class Params(ParamsBase): - num_head: Int32 - num_batch: Int32 - total_q: Int32 - num_splits: Int32 - max_kvblock_in_l2: Int32 - tile_shape_mn: cutlass.Constexpr[Tuple[int, int]] - mCuSeqlensQ: Optional[cute.Tensor] = None - mSeqUsedQ: Optional[cute.Tensor] = None - qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1 - lpt: cutlass.Constexpr[bool] = False - is_split_kv: cutlass.Constexpr[bool] = False - head_swizzle: cutlass.Constexpr[bool] = False - - @staticmethod - @cute.jit - def create( - args: TileSchedulerArguments, *, loc=None, ip=None - ) -> "SingleTileVarlenScheduler.Params": - size_l2 = 50 * 1024 * 1024 # 50 MB for K & V - max_kvblock_in_l2 = size_l2 // ( - (args.headdim + args.headdim_v) * args.element_size * args.tile_shape_mn[1] - ) - assert args.mCuSeqlensQ is not None or args.mSeqUsedQ is not None, ( - "At least one of mCuSeqlensQ or mSeqUsedQ must be provided" - ) - return SingleTileVarlenScheduler.Params( - num_head=args.num_head, - num_batch=args.num_batch, - total_q=args.total_q, - num_splits=args.num_splits, - max_kvblock_in_l2=max_kvblock_in_l2, - tile_shape_mn=args.tile_shape_mn, - mCuSeqlensQ=args.mCuSeqlensQ, - mSeqUsedQ=args.mSeqUsedQ, - qhead_per_kvhead_packgqa=args.qhead_per_kvhead_packgqa, - lpt=args.lpt, - is_split_kv=args.is_split_kv, - head_swizzle=args.head_swizzle, - ) - - def __init__(self, params: Params, tile_idx: Int32, split_idx: Int32, *, loc=None, ip=None): - self.params = params - self._tile_idx = tile_idx - self._split_idx = split_idx - self._is_first_block = True - self._loc = loc - self._ip = ip - - @staticmethod - def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params: - return SingleTileVarlenScheduler.Params.create(args, loc=loc, ip=ip) - - @staticmethod - def create(params: Params, *, loc=None, ip=None) -> "SingleTileVarlenScheduler": - tile_idx, split_idx, _ = cute.arch.block_idx() - return SingleTileVarlenScheduler(params, tile_idx, split_idx, loc=loc, ip=ip) - - # called by host - @staticmethod - def get_grid_shape( - params: Params, - *, - loc=None, - ip=None, - ) -> Tuple[Int32, Int32, Int32]: - total_blocks_max = ( - params.total_q + params.num_batch * (params.tile_shape_mn[0] - 1) - ) // params.tile_shape_mn[0] - return (total_blocks_max * params.num_head, params.num_splits, Int32(1)) - - @cute.jit - def _get_num_m_blocks(self, lane: Int32, bidb_start: Int32) -> Int32: - params = self.params - batch_idx = lane + bidb_start - if cutlass.const_expr(params.mSeqUsedQ is not None): - seqlen = Int32(0) - if batch_idx < params.num_batch: - seqlen = params.mSeqUsedQ[batch_idx] - else: - assert params.mCuSeqlensQ is not None - cur_cu_seqlen = Int32(0) - if batch_idx <= params.num_batch: - cur_cu_seqlen = params.mCuSeqlensQ[batch_idx] - next_cu_seqlen = cute.arch.shuffle_sync_down(cur_cu_seqlen, offset=1) - seqlen = next_cu_seqlen - cur_cu_seqlen - if cutlass.const_expr(params.qhead_per_kvhead_packgqa > 1): - seqlen *= params.qhead_per_kvhead_packgqa - return ( - cute.ceil_div(seqlen, params.tile_shape_mn[0]) - if batch_idx < params.num_batch and lane < cute.arch.WARP_SIZE - 1 - else Int32(0) - ) - - @cute.jit - def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: - params = self.params - lane_idx = cute.arch.lane_idx() - num_m_blocks = self._get_num_m_blocks(lane_idx, bidb_start=0) - num_m_blocks_cumulative = utils.warp_prefix_sum(num_m_blocks, lane_idx) - # Total number of blocks for the next 31 batches - m_blocks_in_group = cute.arch.shuffle_sync(num_m_blocks_cumulative, cute.arch.WARP_SIZE - 1) - # Same for all lanes - group_end_tile = m_blocks_in_group * params.num_head - # if cute.arch.thread_idx()[0] == 128 + 31: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, group_end_tile = %d, num_m_blocks=%d, num_m_blocks_cumulative = %d, m_blocks_in_group = %d", self._tile_idx, group_end_tile, num_m_blocks, num_m_blocks_cumulative, m_blocks_in_group) - block, head_idx, batch_idx = Int32(0), Int32(0), Int32(0) - next_tile_idx = self._tile_idx - while group_end_tile <= next_tile_idx: - batch_idx += cute.arch.WARP_SIZE - 1 - if batch_idx >= params.num_batch: - batch_idx = Int32(params.num_batch) - group_end_tile = next_tile_idx + 1 - else: - num_m_blocks = self._get_num_m_blocks(lane_idx, bidb_start=batch_idx) - num_m_blocks_cumulative = utils.warp_prefix_sum(num_m_blocks, lane_idx) - m_blocks_in_group = cute.arch.shuffle_sync( - num_m_blocks_cumulative, cute.arch.WARP_SIZE - 1 - ) - group_end_tile += m_blocks_in_group * params.num_head - is_valid = False - if batch_idx >= params.num_batch: - block, head_idx, batch_idx = Int32(0), Int32(0), Int32(params.num_batch) - else: - group_start_tile = group_end_tile - m_blocks_in_group * params.num_head - # if cute.arch.thread_idx()[0] == 128 + 31: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, group_end_tile = %d, num_m_blocks=%d, batch_idx = %d", self._tile_idx, group_end_tile, num_m_blocks, batch_idx) - # The next problem to process is the first one that does not have ending tile position - # that is greater than or equal to tile index. - batch_idx_in_group = cute.arch.popc( - cute.arch.vote_ballot_sync( - group_start_tile + num_m_blocks_cumulative * params.num_head <= next_tile_idx - ) - ) - batch_idx += batch_idx_in_group - num_m_blocks_prev_lane = ( - 0 - if batch_idx_in_group == 0 - else cute.arch.shuffle_sync(num_m_blocks_cumulative, batch_idx_in_group - 1) - ) - num_m_blocks = cute.arch.shuffle_sync(num_m_blocks, batch_idx_in_group) - mh_block = next_tile_idx - group_start_tile - num_m_blocks_prev_lane * params.num_head - if cutlass.const_expr(params.lpt or params.head_swizzle): - # This is a version of the SingleTileLPTScheduler, complicated by the fact that - # the seqlen can vary per batch. - # TODO: is there any case where num_m_blocks is 0? - # TODO: by right we should read the seqlen_kv but we're assuming seqlen_q == seqlen_k here - num_n_blocks = ( - num_m_blocks - * params.tile_shape_mn[0] - // params.qhead_per_kvhead_packgqa - // params.tile_shape_mn[1] - ) - # nheads_in_l2 = min(max(self.max_kvblock_in_l2 // num_n_blocks, 1), self.num_head) - # Seems faster to have this be a power of 2 - nheads_in_l2 = ( - 16 - if num_n_blocks * 16 <= params.max_kvblock_in_l2 - else ( - 8 - if num_n_blocks * 8 <= params.max_kvblock_in_l2 - else ( - 4 - if num_n_blocks * 4 <= params.max_kvblock_in_l2 - else (2 if num_n_blocks * 2 <= params.max_kvblock_in_l2 else 1) - ) - ) - ) - nheads_in_l2 = min(nheads_in_l2, params.num_head) - mh_in_l2 = nheads_in_l2 * num_m_blocks - section_idx = mh_block // mh_in_l2 - l2_mod = mh_block - section_idx * mh_in_l2 - # Deal with tail section - nheads_in_this_section = ( - nheads_in_l2 - if nheads_in_l2 * (section_idx + 1) <= params.num_head - else params.num_head - section_idx * nheads_in_l2 - ) - block = l2_mod // nheads_in_this_section - head_idx_residual = l2_mod - block * nheads_in_this_section - head_idx = section_idx * nheads_in_l2 + head_idx_residual - if cutlass.const_expr(params.lpt): - block = num_m_blocks - 1 - block - else: - head_idx = mh_block // num_m_blocks - block = mh_block - head_idx * num_m_blocks - is_valid = self._is_first_block and batch_idx < params.num_batch - # if cute.arch.thread_idx()[0] == 128: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, batch_idx=%d, head_idx=%d, block=%d, is_valid = %d", self._tile_idx, batch_idx, head_idx, block, is_valid) - split_idx = self._split_idx if const_expr(params.is_split_kv) else Int32(0) - return WorkTileInfo((Int32(block), Int32(head_idx), Int32(batch_idx), split_idx), is_valid) - - def initial_work_tile_info(self, *, loc=None, ip=None): - return self.get_current_work(loc=loc, ip=ip) - - def prefetch_next_work(self, *, loc=None, ip=None): - pass - - def advance_to_next_work(self, *, loc=None, ip=None): - # Single tile scheduler - set to invalid tile_idx to indicate no more work - self._is_first_block = False - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.params, self._tile_idx, self._split_idx]: - obj_values = cutlass.extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip( - [self.params, self._tile_idx, self._split_idx], - self._values_pos, - ): - obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return SingleTileVarlenScheduler(*(tuple(obj_list)), loc=self._loc) diff --git a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/utils.py b/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/utils.py deleted file mode 100644 index 74b0064f8d28..000000000000 --- a/tensorrt_llm/_torch/visual_gen/jit_kernels/flash_attention/cute/utils.py +++ /dev/null @@ -1,852 +0,0 @@ -# Copyright (c) 2025, Tri Dao. - -import math -import hashlib -import inspect -import re -from typing import Type, Callable, Optional, Tuple, overload -from functools import partial - -import cutlass -import cutlass.cute as cute - -from cutlass import Float32, const_expr -from cutlass.cutlass_dsl import T, dsl_user_op -from cutlass._mlir.dialects import nvvm, llvm -from cutlass.cute.runtime import from_dlpack - - -# cute.arch.{fma,mul,add}_packed_f32x2 uses RZ rounding mode by default -# CuTe DSL CUDA 13 validates rounding modes as string literals. The string -# form is also accepted by older wrappers, so keep it version-independent. -_RND_RN = "rn" -fma_packed_f32x2 = partial(cute.arch.fma_packed_f32x2, rnd=_RND_RN) -mul_packed_f32x2 = partial(cute.arch.mul_packed_f32x2, rnd=_RND_RN) -add_packed_f32x2 = partial(cute.arch.add_packed_f32x2, rnd=_RND_RN) -sub_packed_f32x2 = partial( - cute.arch.calc_packed_f32x2_op, - src_c=None, - calc_func=nvvm.sub_packed_f32x2, - rnd=_RND_RN, -) - - -def hash_callable(func: Callable) -> str: - """Hash a callable based on the source code or bytecode and closure values. - - Fast-path: if the callable (or its __wrapped__ base) has a ``__cute_hash__`` - attribute, that value is returned immediately. Code-generation backends such - as Inductor can set this attribute to avoid expensive runtime hashing. - """ - if hasattr(func, "__cute_hash__"): - return func.__cute_hash__ - - # Unwrap decorated functions (e.g., cute.jit wrappers). - if hasattr(func, "__wrapped__"): - base_func = func.__wrapped__ - if hasattr(base_func, "__cute_hash__"): - return base_func.__cute_hash__ - func = base_func - - try: - data = inspect.getsource(func).encode() - except (OSError, TypeError): - if hasattr(func, "__code__") and func.__code__ is not None: - data = func.__code__.co_code - else: - data = repr(func).encode() - - hasher = hashlib.sha256(data) - - if hasattr(func, "__closure__") and func.__closure__ is not None: - for idx, cell in enumerate(func.__closure__): - cell_value = cell.cell_contents - hasher.update(repr(cell_value).encode()) - - return hasher.hexdigest() - - -def create_softcap_scoremod(softcap_val): - inv_softcap = 1.0 / softcap_val - - @cute.jit - def scoremod_premask_fn(acc_S_SSA, batch_idx, head_idx, q_idx, kv_idx, aux_tensors): - scores = acc_S_SSA * inv_softcap - return scores * cute.math.tanh(scores, fastmath=True) - - return scoremod_premask_fn - - -def convert_from_dlpack(x, leading_dim, alignment=16, divisibility=1) -> cute.Tensor: - return ( - from_dlpack(x, assumed_align=alignment) - .mark_layout_dynamic(leading_dim=leading_dim) - .mark_compact_shape_dynamic( - mode=leading_dim, stride_order=x.dim_order(), divisibility=divisibility - ) - ) - - -def convert_from_dlpack_leading_static( - x, leading_dim, alignment=16, static_modes=None, stride_order=None -) -> cute.Tensor: - if stride_order is None: - stride_order = x.dim_order() - x_ = from_dlpack(x, assumed_align=alignment) - for i in range(x.ndim): - if i != leading_dim and (static_modes is None or i not in static_modes): - x_ = x_.mark_compact_shape_dynamic(mode=i, stride_order=stride_order) - return x_ - - -def make_tiled_copy_A( - copy_atom: cute.CopyAtom, tiled_mma: cute.TiledMma, swapAB: cutlass.Constexpr[bool] = False -) -> cute.TiledCopy: - if const_expr(swapAB): - return cute.make_tiled_copy_B(copy_atom, tiled_mma) - else: - return cute.make_tiled_copy_A(copy_atom, tiled_mma) - - -def make_tiled_copy_B( - copy_atom: cute.CopyAtom, tiled_mma: cute.TiledMma, swapAB: cutlass.Constexpr[bool] = False -) -> cute.TiledCopy: - if const_expr(swapAB): - return cute.make_tiled_copy_A(copy_atom, tiled_mma) - else: - return cute.make_tiled_copy_B(copy_atom, tiled_mma) - - -def mma_make_fragment_A( - smem: cute.Tensor, thr_mma: cute.core.ThrMma, swapAB: cutlass.Constexpr[bool] = False -) -> cute.Tensor: - if const_expr(swapAB): - return mma_make_fragment_B(smem, thr_mma) - else: - return thr_mma.make_fragment_A(thr_mma.partition_A(smem)) - - -def mma_make_fragment_B( - smem: cute.Tensor, thr_mma: cute.core.ThrMma, swapAB: cutlass.Constexpr[bool] = False -) -> cute.Tensor: - if const_expr(swapAB): - return mma_make_fragment_A(smem, thr_mma) - else: - return thr_mma.make_fragment_B(thr_mma.partition_B(smem)) - - -def get_smem_store_atom( - arch: cutlass.Constexpr[int], element_type: Type[cute.Numeric], transpose: bool = False -) -> cute.CopyAtom: - if const_expr(arch < 90 or element_type.width != 16): - return cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - element_type, - num_bits_per_copy=2 * element_type.width, - ) - else: - return cute.make_copy_atom( - cute.nvgpu.warp.StMatrix8x8x16bOp(transpose=transpose, num_matrices=4), - element_type, - ) - - -@cute.jit -def warp_reduce( - val: cute.TensorSSA | cute.Numeric, - op: Callable, - width: cutlass.Constexpr[int] = cute.arch.WARP_SIZE, -) -> cute.TensorSSA | cute.Numeric: - if const_expr(isinstance(val, cute.TensorSSA)): - res = cute.make_fragment(val.shape, val.dtype) - res.store(val) - for i in cutlass.range_constexpr(cute.size(val.shape)): - res[i] = warp_reduce(res[i], op, width) - return res.load() - else: - for i in cutlass.range_constexpr(int(math.log2(width))): - val = op(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i)) - return val - - -def convert_layout_acc_mn(acc_layout: cute.Layout, transpose: bool = False) -> cute.Layout: - """ - For Sm80, convert ((2, 2), MMA_M, MMA_N, ...) to ((2, MMA_M), (2, MMA_N), ...). - For Sm90, convert ((2, 2, V), MMA_M, MMA_N, ...) to ((2, MMA_M), (2, V, MMA_N), ...). - """ - acc_layout_col_major = cute.make_layout(acc_layout.shape) - shape = ( - (acc_layout_col_major.shape[0][1], acc_layout_col_major.shape[1]), # MMA_M - ( - acc_layout_col_major.shape[0][0], - *acc_layout_col_major.shape[0][2:], - acc_layout_col_major.shape[2], - ), # MMA_N - *acc_layout_col_major.shape[3:], - ) - stride = ( - (acc_layout_col_major.stride[0][1], acc_layout_col_major.stride[1]), # MMA_M - ( - acc_layout_col_major.stride[0][0], - *acc_layout_col_major.stride[0][2:], - acc_layout_col_major.stride[2], - ), # MMA_N - *acc_layout_col_major.stride[3:], - ) - if const_expr(transpose): - shape = (shape[1], shape[0], *shape[2:]) - stride = (stride[1], stride[0], *stride[2:]) - acc_layout_mn = cute.make_layout(shape, stride=stride) - return cute.composition(acc_layout, acc_layout_mn) - - -def make_acc_tensor_mn_view(acc: cute.Tensor, transpose: bool = False) -> cute.Tensor: - return cute.make_tensor(acc.iterator, convert_layout_acc_mn(acc.layout, transpose=transpose)) - - -@cute.jit -def convert_layout_acc_frgA(acc_layout: cute.Layout) -> cute.Layout: - # For back to back gemm, convert layout of acc0 to gemm 1 accept layout. - # For Sm80, as the mma instruction shape is 16x8x16, we need to convert from (4, MMA_M, MMA_N) to ((4, 2), MMA_M, MMA_N / 2) - # For Sm90, FP16/BF16, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to ((2, 2, 2), MMA_M, (N / 16, MMA_N)) - # TODO: Sm90 FP8 - if const_expr(cute.rank(acc_layout.shape[0]) == 3): # Sm90 - l = cute.logical_divide( - acc_layout, ((None, None, 2), None, None) - ) # ((2, 2, (2, N / 16)), MMA_M, MMA_N) - rA_mma_view = cute.make_layout( - ( - (l.shape[0][0], l.shape[0][1], l.shape[0][2][0]), - l.shape[1], - (l.shape[0][2][1], l.shape[2]), - ), - stride=( - (l.stride[0][0], l.stride[0][1], l.stride[0][2][0]), - l.stride[1], - (l.stride[0][2][1], l.stride[2]), - ), - ) - else: # Sm80 - # (4, MMA_M, MMA_N) -> (4, MMA_M, (2, MMA_N / 2)) - l = cute.logical_divide(acc_layout, (None, None, 2)) - rA_mma_view = cute.make_layout( - ( - (l.shape[0], l.shape[2][0]), - l.shape[1], - l.shape[2][1], - ), - stride=( - (l.stride[0], l.stride[2][0]), - l.stride[1], - l.stride[2][1], - ), - ) - return rA_mma_view - - -def make_acc_tensor_frgA_view(acc: cute.Tensor) -> cute.Tensor: - return cute.make_tensor(acc.iterator, convert_layout_acc_frgA(acc.layout)) - - -def select(a: cute.Tensor, mode: list[int]) -> cute.Tensor: - return cute.make_tensor(a.iterator, cute.select(a.layout, mode)) - - -def transpose_view(a: cute.Tensor) -> cute.Tensor: - """Transpose the first two dimensions of a tensor on smem.""" - shape = (a.shape[1], a.shape[0], *a.shape[2:]) - order = (1, 0, *range(2, cute.rank(a))) - return cute.composition(a, cute.make_ordered_layout(shape, order=order)) - # stride = (a.layout.stride[1], a.layout.stride[0], *a.layout.stride[2:]) - # return cute.make_tensor(a.iterator, cute.make_layout(shape, stride=stride)) - - -def parse_swizzle_from_pointer(ptr: cute.Pointer) -> cute.Swizzle: - """Extract swizzle parameters from a pointer's swizzle_type. - - The swizzle_type string has the form '!cute.swizzle<"S">' where - b, m, s are the swizzle parameters (bits, base, shift). - - Returns: - A cute.Swizzle object constructed from the extracted parameters - - Raises: - ValueError: If the swizzle_type string cannot be parsed - """ - # Ideally there should be a better API to get swizzle parameters, but we'll just parse - # the string here. - swizzle_str = str(ptr.type.swizzle_type) - # Extract the inner part "S" - match = re.search(r"S<(\d+),(\d+),(\d+)>", swizzle_str) - if match: - b, m, s = int(match.group(1)), int(match.group(2)), int(match.group(3)) - return cute.make_swizzle(b, m, s) - else: - raise ValueError(f"Could not parse swizzle_type: {swizzle_str}") - - -@cute.jit -def exp2f(x: cute.TensorSSA | Float32) -> cute.TensorSSA | Float32: - """exp2f calculation for both vector and scalar. - :param x: input value - :type x: cute.TensorSSA or Float32 - :return: exp2 value - :rtype: cute.TensorSSA or Float32 - """ - if const_expr(isinstance(x, cute.TensorSSA)): - res = cute.make_fragment(x.shape, Float32) - res.store(x) - for i in cutlass.range_constexpr(cute.size(x.shape)): - res[i] = cute.arch.exp2(res[i]) - return res.load() - else: - return cute.arch.exp2(x) - - -@dsl_user_op -def log2f(a: float | Float32, *, loc=None, ip=None) -> Float32: - return Float32( - llvm.inline_asm( - T.f32(), - [Float32(a).ir_value(loc=loc, ip=ip)], - "lg2.approx.ftz.f32 $0, $1;", - "=f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@dsl_user_op -def logf(a: float | Float32, *, loc=None, ip=None) -> Float32: - return log2f(a, loc=loc, ip=ip) * math.log(2.0) - - -@dsl_user_op -def fmax( - a: float | Float32, b: float | Float32, c: float | Float32 | None = None, *, loc=None, ip=None -) -> Float32: - return Float32( - nvvm.fmax( - Float32(a).ir_value(loc=loc, ip=ip), - Float32(b).ir_value(loc=loc, ip=ip), - c=Float32(c).ir_value(loc=loc, ip=ip) if c is not None else None, - loc=loc, - ip=ip, - ) - ) - - -@cute.jit -def fmax_reduce( - x: cute.TensorSSA, init_val: float | Float32 | None = None, arch: cutlass.Constexpr[int] = 80 -) -> Float32: - if const_expr(arch < 100 or cute.size(x.shape) % 8 != 0): - # if const_expr(init_val is None): - # init_val = -cutlass.Float32.if - # return x.reduce(cute.ReductionOp.MAX, init_val, 0) - res = cute.make_fragment(x.shape, Float32) - res.store(x) - # local_max = [res[0], res[1]] - # for i in cutlass.range_constexpr(2, cute.size(x.shape), 2): - # local_max[0] = fmax(local_max[0], res[i + 0]) - # local_max[1] = fmax(local_max[1], res[i + 1]) - # local_max[0] = fmax(local_max[0], local_max[1]) - # return local_max[0] if const_expr(init_val is None) else fmax(local_max[0], init_val) - local_max = [res[0], res[1], res[2], res[3]] - for i in cutlass.range_constexpr(4, cute.size(x.shape), 4): - local_max[0] = fmax(local_max[0], res[i + 0]) - local_max[1] = fmax(local_max[1], res[i + 1]) - local_max[2] = fmax(local_max[2], res[i + 2]) - local_max[3] = fmax(local_max[3], res[i + 3]) - local_max[0] = fmax(local_max[0], local_max[1]) - local_max[2] = fmax(local_max[2], local_max[3]) - local_max[0] = fmax(local_max[0], local_max[2]) - return local_max[0] if const_expr(init_val is None) else fmax(local_max[0], init_val) - else: - # [2025-06-15] x.reduce only seems to use 50% 3-input max and 50% 2-input max - # We instead force the 3-input max. - res = cute.make_fragment(x.shape, Float32) - res.store(x) - local_max_0 = ( - fmax(init_val, res[0], res[1]) - if const_expr(init_val is not None) - else fmax(res[0], res[1]) - ) - local_max = [ - local_max_0, - fmax(res[2], res[3]), - fmax(res[4], res[5]), - fmax(res[6], res[7]), - ] - for i in cutlass.range_constexpr(8, cute.size(x.shape), 8): - local_max[0] = fmax(local_max[0], res[i], res[i + 1]) - local_max[1] = fmax(local_max[1], res[i + 2], res[i + 3]) - local_max[2] = fmax(local_max[2], res[i + 4], res[i + 5]) - local_max[3] = fmax(local_max[3], res[i + 6], res[i + 7]) - local_max[0] = fmax(local_max[0], local_max[1]) - return fmax(local_max[0], local_max[2], local_max[3]) - - -@cute.jit -def fadd_reduce( - x: cute.TensorSSA, init_val: float | Float32 | None = None, arch: cutlass.Constexpr[int] = 80 -) -> Float32: - if const_expr(arch < 100 or cute.size(x.shape) % 8 != 0): - if const_expr(init_val is None): - init_val = Float32.zero - return x.reduce(cute.ReductionOp.ADD, init_val, 0) - # res = cute.make_fragment(x.shape, Float32) - # res.store(x) - # local_sum = [res[0], res[1], res[2], res[3]] - # for i in cutlass.range_constexpr(4, cute.size(x.shape), 4): - # local_sum[0] += res[i + 0] - # local_sum[1] += res[i + 1] - # local_sum[2] += res[i + 2] - # local_sum[3] += res[i + 3] - # local_sum[0] += local_sum[1] - # local_sum[2] += local_sum[3] - # local_sum[0] += local_sum[2] - # return local_sum[0] if const_expr(init_val is None) else local_sum[0] + init_val - else: - res = cute.make_fragment(x.shape, Float32) - res.store(x) - local_sum_0 = ( - add_packed_f32x2((init_val, 0.0), (res[0], res[1])) - # add_packed_f32x2((init_val / 2, init_val / 2), (res[0], res[1])) - if const_expr(init_val is not None) - else (res[0], res[1]) - ) - local_sum = [local_sum_0, (res[2], res[3]), (res[4], res[5]), (res[6], res[7])] - for i in cutlass.range_constexpr(8, cute.size(x.shape), 8): - local_sum[0] = add_packed_f32x2(local_sum[0], (res[i + 0], res[i + 1])) - local_sum[1] = add_packed_f32x2(local_sum[1], (res[i + 2], res[i + 3])) - local_sum[2] = add_packed_f32x2(local_sum[2], (res[i + 4], res[i + 5])) - local_sum[3] = add_packed_f32x2(local_sum[3], (res[i + 6], res[i + 7])) - local_sum[0] = add_packed_f32x2(local_sum[0], local_sum[1]) - local_sum[2] = add_packed_f32x2(local_sum[2], local_sum[3]) - local_sum[0] = add_packed_f32x2(local_sum[0], local_sum[2]) - return local_sum[0][0] + local_sum[0][1] - - -@dsl_user_op -def atomic_add_fp32(a: float | Float32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> None: - # gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value() - # # cache_hint = cutlass.Int64(0x12F0000000000000) - # llvm.inline_asm( - # None, - # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip)], - # # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip), cache_hint.ir_value()], - # "red.global.add.f32 [$0], $1;", - # # "red.global.add.L2::cache_hint.f32 [$0], $1, 0x12F0000000000000;", - # # "red.global.add.L2::cache_hint.f32 [$0], $1, $2;", - # "l,f", - # # "l,f,l", - # has_side_effects=True, - # is_align_stack=False, - # asm_dialect=llvm.AsmDialect.AD_ATT, - # ) - nvvm.atomicrmw( - res=T.f32(), op=nvvm.AtomicOpKind.FADD, ptr=gmem_ptr.llvm_ptr, a=Float32(a).ir_value() - ) - - -@dsl_user_op -def elem_pointer(x: cute.Tensor, coord: cute.Coord, *, loc=None, ip=None) -> cute.Pointer: - return x.iterator + cute.crd2idx(coord, x.layout, loc=loc, ip=ip) - - -@dsl_user_op -def elem_pointer_i64(x: cute.Tensor, coord: cute.Coord, *, loc=None, ip=None) -> cute.Pointer: - flat_coord_i64 = tuple(cutlass.Int64(c) for c in cute.flatten(coord)) - flat_stride = cute.flatten_to_tuple(x.stride) - assert len(flat_coord_i64) == len(flat_stride), ( - "Coordinate and stride must have the same length" - ) - offset = sum(c * s for c, s in zip(flat_coord_i64, flat_stride)) - # HACK: we assume that applying the offset does not change the pointer alignment - byte_offset = offset * x.element_type.width // 8 - return cute.make_ptr( - x.element_type, - x.iterator.toint() + byte_offset, - x.memspace, - assumed_align=x.iterator.alignment, - ) - - -@cute.jit -def predicate_k(tAcA: cute.Tensor, limit: cutlass.Int32) -> cute.Tensor: - # Only compute predicates for the "k" dimension. For the mn dimension, we will use "if" - tApA = cute.make_fragment( - cute.make_layout( - (cute.size(tAcA, mode=[0, 1]), cute.size(tAcA, mode=[1]), cute.size(tAcA, mode=[2])), - stride=(cute.size(tAcA, mode=[2]), 0, 1), - ), - cutlass.Boolean, - ) - for rest_v in cutlass.range_constexpr(tApA.shape[0]): - for rest_k in cutlass.range_constexpr(tApA.shape[2]): - tApA[rest_v, 0, rest_k] = cute.elem_less(tAcA[(0, rest_v), 0, rest_k][1], limit) - return tApA - - -def canonical_warp_group_idx(sync: bool = True) -> cutlass.Int32: - warp_group_idx = cute.arch.thread_idx()[0] // 128 - if const_expr(sync): - warp_group_idx = cute.arch.make_warp_uniform(warp_group_idx) - return warp_group_idx - - -# @dsl_user_op -# def warp_vote_any_lt(a: float | Float32, b: float | Float32, *, loc=None, ip=None) -> cutlass.Boolean: -# mask = cutlass.Int32(-1) -# return cutlass.Boolean( -# llvm.inline_asm( -# T.i32(), -# [Float32(a).ir_value(loc=loc, ip=ip), Float32(b).ir_value(loc=loc, ip=ip), mask.ir_value(loc=loc, ip=ip)], -# ".pred p1, p2;\n" -# "setp.lt.f32 p1, $1, $2;\n" -# "vote.sync.any.pred p2, p1, $3;\n" -# "selp.u32 $0, 1, 0, p2;", -# # "selp.u32 $0, 1, 0, p1;", -# "=r,f,f,r", -# has_side_effects=False, -# is_align_stack=False, -# asm_dialect=llvm.AsmDialect.AD_ATT, -# ) -# ) - - -@cute.jit -def shuffle_sync( - value: cute.Numeric, - offset: cute.typing.Int, - width: cutlass.Constexpr[int] = cute.arch.WARP_SIZE, -) -> cute.Numeric: - assert value.width % 32 == 0, "value type must be a multiple of 32 bits" - # 1 -> 0b11111, 2 -> 0b11110, 4 -> 0b11100, 8 -> 0b11000, 16 -> 0b10000, 32 -> 0b00000 - mask = cute.arch.WARP_SIZE - width - clamp = cute.arch.WARP_SIZE - 1 - mask_and_clamp = mask << 8 | clamp - # important: need stride 1 and not 0 for recast_tensor to work - val = cute.make_rmem_tensor(cute.make_layout((1,), stride=(1,)), type(value)) - val[0] = value - val_i32 = cute.recast_tensor(val, cutlass.Int32) - for i in cutlass.range_constexpr(cute.size(val_i32)): - val_i32[i] = cute.arch.shuffle_sync(val_i32[i], offset, mask_and_clamp=mask_and_clamp) - return val[0] - - -@dsl_user_op -def shr_u32(val: cutlass.Uint32, shift: cutlass.Uint32, *, loc=None, ip=None) -> cutlass.Uint32: - return cutlass.Uint32( - llvm.inline_asm( - T.i32(), - [ - cutlass.Uint32(val).ir_value(loc=loc, ip=ip), - cutlass.Uint32(shift).ir_value(loc=loc, ip=ip), - ], - "shr.s32 $0, $1, $2;", - "=r,r,r", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@cute.jit -def warp_prefix_sum(val: cutlass.Int32, lane: Optional[cutlass.Int32] = None) -> cutlass.Int32: - if const_expr(lane is None): - lane = cute.arch.lane_idx() - # if cute.arch.thread_idx()[0] >= 128 and cute.arch.thread_idx()[0] < 128 + 32 and cute.arch.block_idx()[0] == 0: cute.printf("tidx = %d, val = %d", cute.arch.thread_idx()[0] % 32, val) - for i in cutlass.range_constexpr(int(math.log2(cute.arch.WARP_SIZE))): - offset = 1 << i - # Very important that we set mask_and_clamp to 0 - partial_sum = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0) - if lane >= offset: - val += partial_sum - # if cute.arch.thread_idx()[0] >= 128 and cute.arch.thread_idx()[0] < 128 + 32 and cute.arch.block_idx()[0] == 0: cute.printf("tidx = %d, partial_sum = %d, val = %d", cute.arch.thread_idx()[0] % 32, partial_sum, val) - return val - - -@dsl_user_op -def cvt_f16x2_f32( - a: float | Float32, b: float | Float32, to_dtype: Type, *, loc=None, ip=None -) -> cutlass.Int32: - assert to_dtype in [cutlass.BFloat16, cutlass.Float16], "to_dtype must be BFloat16 or Float16" - return cutlass.Int32( - llvm.inline_asm( - T.i32(), - [Float32(a).ir_value(loc=loc, ip=ip), Float32(b).ir_value(loc=loc, ip=ip)], - f"cvt.rn.{'bf16x2' if to_dtype is cutlass.BFloat16 else 'f16x2'}.f32 $0, $2, $1;", - "=r,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@overload -def cvt_f16(src: cute.Tensor, dst: cute.Tensor) -> None: ... - - -@overload -def cvt_f16(src: cute.Tensor, dtype: Type[cute.Numeric]) -> cute.Tensor: ... - - -@cute.jit -def cvt_f16(src: cute.Tensor, dst_or_dtype): - """Convert Float32 tensor to Float16/BFloat16. - - Args: - src: Source tensor with Float32 element type - dst_or_dtype: Either a destination tensor or a dtype (Float16/BFloat16) - - Returns: - None if dst is a tensor, or a new tensor if dtype is provided - """ - if const_expr(isinstance(dst_or_dtype, type)): - # dtype variant: create new tensor and call the tensor variant - dtype = dst_or_dtype - dst = cute.make_fragment(src.shape, dtype) - cvt_f16(src, dst) - return dst - else: - # tensor variant: write to dst - dst = dst_or_dtype - assert cute.size(dst.shape) == cute.size(src.shape), "dst and src must have the same size" - assert cute.size(src.shape) % 2 == 0, "src must have an even number of elements" - assert dst.element_type in [cutlass.BFloat16, cutlass.Float16], ( - "dst must be BFloat16 or Float16" - ) - assert src.element_type is Float32, "src must be Float32" - dst_i32 = cute.recast_tensor(dst, cutlass.Int32) - assert cute.size(dst_i32.shape) * 2 == cute.size(src.shape) - for i in cutlass.range_constexpr(cute.size(dst_i32)): - dst_i32[i] = cvt_f16x2_f32(src[2 * i], src[2 * i + 1], dst.element_type) - - -@dsl_user_op -@cute.jit -def evaluate_polynomial(x: Float32, poly: Tuple[Float32, ...], *, loc=None, ip=None) -> Float32: - deg = len(poly) - 1 - out = poly[deg] - for i in cutlass.range_constexpr(deg - 1, -1, -1): - out = out * x + poly[i] - return out - - -@dsl_user_op -@cute.jit -def evaluate_polynomial_2( - x: Float32, y: Float32, poly: Tuple[Float32, ...], *, loc=None, ip=None -) -> Tuple[Float32, Float32]: - deg = len(poly) - 1 - out = (poly[deg], poly[deg]) - for i in cutlass.range_constexpr(deg - 1, -1, -1): - out = fma_packed_f32x2(out, (x, y), (poly[i], poly[i])) - return out - - -@dsl_user_op -def add_round_down(x: float | Float32, y: float | Float32, *, loc=None, ip=None) -> Float32: - # There's probably a way to call llvm or nvvm to do this instead of ptx - return cutlass.Float32( - llvm.inline_asm( - T.f32(), - [Float32(x).ir_value(loc=loc, ip=ip), Float32(y).ir_value(loc=loc, ip=ip)], - "add.rm.ftz.f32 $0, $1, $2;", - "=f,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@dsl_user_op -def combine_int_frac_ex2(x_rounded: Float32, frac_ex2: Float32, *, loc=None, ip=None) -> Float32: - return cutlass.Float32( - llvm.inline_asm( - T.f32(), - [ - Float32(x_rounded).ir_value(loc=loc, ip=ip), - Float32(frac_ex2).ir_value(loc=loc, ip=ip), - ], - "{\n\t" - ".reg .s32 x_rounded_i, frac_ex_i, x_rounded_e, out_i;\n\t" - "mov.b32 x_rounded_i, $1;\n\t" - "mov.b32 frac_ex_i, $2;\n\t" - "shl.b32 x_rounded_e, x_rounded_i, 23;\n\t" - # add.u32 generates IMAD instruction and add.s32 generates LEA instruction - # IMAD uses the FMA pipeline and LEA uses the ALU pipeline, afaik - "add.s32 out_i, x_rounded_e, frac_ex_i;\n\t" - "mov.b32 $0, out_i;\n\t" - "}\n", - "=f,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - -@dsl_user_op -def ex2_emulation(x: Float32, *, loc=None, ip=None) -> Float32: - # We assume x <= 127.0 - poly_ex2_deg3 = ( - 1.0, - 0.695146143436431884765625, - 0.227564394474029541015625, - 0.077119089663028717041015625, - ) - fp32_round_int = float(2**23 + 2**22) - x_clamped = cute.arch.fmax(x, -127.0) - # We want to round down here, so that the fractional part is in [0, 1) - x_rounded = add_round_down(x_clamped, fp32_round_int, loc=loc, ip=ip) - # The integer floor of x is now in the last 8 bits of x_rounded - # We assume the next 2 ops round to nearest even. The rounding mode is important. - x_rounded_back = x_rounded - fp32_round_int - x_frac = x_clamped - x_rounded_back - x_frac_ex2 = evaluate_polynomial(x_frac, poly_ex2_deg3, loc=loc, ip=ip) - return combine_int_frac_ex2(x_rounded, x_frac_ex2, loc=loc, ip=ip) - - -# TODO: check that the ex2_emulation_2 produces the same SASS as the ptx version -@dsl_user_op -def ex2_emulation_2(x: Float32, y: Float32, *, loc=None, ip=None) -> Tuple[Float32, Float32]: - # We assume x <= 127.0 and y <= 127.0 - poly_ex2_deg3 = ( - 1.0, - 0.695146143436431884765625, - 0.227564394474029541015625, - 0.077119089663028717041015625, - ) - fp32_round_int = float(2**23 + 2**22) - xy_clamped = (cute.arch.fmax(x, -127.0), cute.arch.fmax(y, -127.0)) - # We want to round down here, so that the fractional part is in [0, 1) - xy_rounded = cute.arch.add_packed_f32x2(xy_clamped, (fp32_round_int, fp32_round_int), rnd="rm") - # The integer floor of x & y are now in the last 8 bits of xy_rounded - # We want the next 2 ops to round to nearest even. The rounding mode is important. - xy_rounded_back = sub_packed_f32x2(xy_rounded, (fp32_round_int, fp32_round_int)) - xy_frac = sub_packed_f32x2(xy_clamped, xy_rounded_back) - xy_frac_ex2 = evaluate_polynomial_2(*xy_frac, poly_ex2_deg3, loc=loc, ip=ip) - x_out = combine_int_frac_ex2(xy_rounded[0], xy_frac_ex2[0], loc=loc, ip=ip) - y_out = combine_int_frac_ex2(xy_rounded[1], xy_frac_ex2[1], loc=loc, ip=ip) - return x_out, y_out - - -@dsl_user_op -def e2e_asm2(x: Float32, y: Float32, *, loc=None, ip=None) -> Tuple[Float32, Float32]: - out_f32x2 = llvm.inline_asm( - llvm.StructType.get_literal([T.f32(), T.f32()]), - [Float32(x).ir_value(loc=loc, ip=ip), Float32(y, loc=loc, ip=ip).ir_value()], - "{\n\t" - ".reg .f32 f1, f2, f3, f4, f5, f6, f7;\n\t" - ".reg .b64 l1, l2, l3, l4, l5, l6, l7, l8, l9, l10;\n\t" - ".reg .s32 r1, r2, r3, r4, r5, r6, r7, r8;\n\t" - "max.ftz.f32 f1, $2, 0fC2FE0000;\n\t" - "max.ftz.f32 f2, $3, 0fC2FE0000;\n\t" - "mov.b64 l1, {f1, f2};\n\t" - "mov.f32 f3, 0f4B400000;\n\t" - "mov.b64 l2, {f3, f3};\n\t" - "add.rm.ftz.f32x2 l7, l1, l2;\n\t" - "sub.rn.ftz.f32x2 l8, l7, l2;\n\t" - "sub.rn.ftz.f32x2 l9, l1, l8;\n\t" - "mov.f32 f7, 0f3D9DF09D;\n\t" - "mov.b64 l6, {f7, f7};\n\t" - "mov.f32 f6, 0f3E6906A4;\n\t" - "mov.b64 l5, {f6, f6};\n\t" - "mov.f32 f5, 0f3F31F519;\n\t" - "mov.b64 l4, {f5, f5};\n\t" - "mov.f32 f4, 0f3F800000;\n\t" - "mov.b64 l3, {f4, f4};\n\t" - "fma.rn.ftz.f32x2 l10, l9, l6, l5;\n\t" - "fma.rn.ftz.f32x2 l10, l10, l9, l4;\n\t" - "fma.rn.ftz.f32x2 l10, l10, l9, l3;\n\t" - "mov.b64 {r1, r2}, l7;\n\t" - "mov.b64 {r3, r4}, l10;\n\t" - "shl.b32 r5, r1, 23;\n\t" - "add.s32 r7, r5, r3;\n\t" - "shl.b32 r6, r2, 23;\n\t" - "add.s32 r8, r6, r4;\n\t" - "mov.b32 $0, r7;\n\t" - "mov.b32 $1, r8;\n\t" - "}\n", - "=r,=r,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - out0 = Float32(llvm.extractvalue(T.f32(), out_f32x2, [0], loc=loc, ip=ip)) - out1 = Float32(llvm.extractvalue(T.f32(), out_f32x2, [1], loc=loc, ip=ip)) - return out0, out1 - - -@dsl_user_op -def domain_offset_aligned( - coord: cute.Coord, tensor: cute.Tensor, *, loc=None, ip=None -) -> cute.Tensor: - assert isinstance(tensor.iterator, cute.Pointer) - # We assume that applying the offset does not change the pointer alignment - new_ptr = cute.make_ptr( - tensor.element_type, - elem_pointer(tensor, coord).toint(), - tensor.memspace, - assumed_align=tensor.iterator.alignment, - ) - return cute.make_tensor(new_ptr, tensor.layout) - - -@dsl_user_op -def domain_offset_i64(coord: cute.Coord, tensor: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: - flat_coord_i64 = tuple(cutlass.Int64(c) for c in cute.flatten(coord)) - flat_stride = cute.flatten_to_tuple(tensor.stride) - assert len(flat_coord_i64) == len(flat_stride), ( - "Coordinate and stride must have the same length" - ) - offset = sum(c * s for c, s in zip(flat_coord_i64, flat_stride)) - assert isinstance(tensor.iterator, cute.Pointer) - # HACK: we assume that applying the offset does not change the pointer alignment - new_ptr = cute.make_ptr( - tensor.element_type, - tensor.iterator.toint() + offset * tensor.element_type.width // 8, - tensor.memspace, - assumed_align=tensor.iterator.max_alignment, - ) - return cute.make_tensor(new_ptr, tensor.layout) - - -@dsl_user_op -def coord_offset_i64( - tensor: cute.Tensor, idx: cute.typing.Int, dim: int, *, loc=None, ip=None -) -> cute.Tensor: - offset = cutlass.Int64(idx) * cute.size(tensor.stride[dim]) - assert isinstance(tensor.iterator, cute.Pointer) - # HACK: we assume that applying the offset does not change the pointer alignment - new_ptr = cute.make_ptr( - tensor.element_type, - tensor.iterator.toint() + offset * tensor.element_type.width // 8, - tensor.memspace, - assumed_align=tensor.iterator.max_alignment, - ) - new_layout = cute.slice_( - tensor.layout, (*[None] * dim, 0, *[None] * (cute.rank(tensor) - dim - 1)) - ) - return cute.make_tensor(new_ptr, new_layout) - - -@cute.jit -def scalar_to_ssa(a: cute.Numeric, dtype) -> cute.TensorSSA: - """Convert a scalar to a cute TensorSSA of shape (1,) and given dtype""" - vec = cute.make_fragment(1, dtype) - vec[0] = a - return vec.load() - - -def ssa_to_scalar(val): - """Could inline but nice for reflecting the above api""" - return val[0] From 1a5292091da192aa0be97018b0768cb20db8287a Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Wed, 27 May 2026 17:32:03 -0700 Subject: [PATCH 066/308] [TRTLLM-12648][test] add disagg cancellation stress-test harness skeleton (#14375) Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- .../defs/stress_test/disagg_cancel/README.md | 126 ++++ .../stress_test/disagg_cancel/__init__.py | 22 + .../disagg_cancel/configs/README.md | 62 ++ .../configs/marathon_cpp_v1_deepseek.yaml | 129 ++++ .../configs/marathon_python_v2_qwen.yaml | 129 ++++ .../defs/stress_test/disagg_cancel/harness.py | 705 ++++++++++++++++++ .../test_disagg_cancel_stress.py | 115 +++ .../disagg_cancel/test_log_scanner.py | 522 +++++++++++++ 8 files changed, 1810 insertions(+) create mode 100644 tests/integration/defs/stress_test/disagg_cancel/README.md create mode 100644 tests/integration/defs/stress_test/disagg_cancel/__init__.py create mode 100644 tests/integration/defs/stress_test/disagg_cancel/configs/README.md create mode 100644 tests/integration/defs/stress_test/disagg_cancel/configs/marathon_cpp_v1_deepseek.yaml create mode 100644 tests/integration/defs/stress_test/disagg_cancel/configs/marathon_python_v2_qwen.yaml create mode 100644 tests/integration/defs/stress_test/disagg_cancel/harness.py create mode 100644 tests/integration/defs/stress_test/disagg_cancel/test_disagg_cancel_stress.py create mode 100644 tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py diff --git a/tests/integration/defs/stress_test/disagg_cancel/README.md b/tests/integration/defs/stress_test/disagg_cancel/README.md new file mode 100644 index 000000000000..8251840a2d61 --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/README.md @@ -0,0 +1,126 @@ +# Disaggregated Cancellation Stress-Test Suite + +Marathon-style stress tests that gate regressions of the bug class +fixed by [PR #13713](https://github.com/NVIDIA/TensorRT-LLM/pull/13713) +(cleanup / lifetime / quiescence invariants in the disagg KV +transceiver under heavy mid-flight cancellation). + +| | | +|---|---| +| **Tracked by** | [TRTLLM-12648](https://jirasw.nvidia.com/browse/TRTLLM-12648), [TRTLLM-12721](https://jirasw.nvidia.com/browse/TRTLLM-12721) | +| **Bug it gates** | NVBug 6104831 (disaggregated permanent wedge) | +| **Fix it gates** | [PR #13713](https://github.com/NVIDIA/TensorRT-LLM/pull/13713) | + +## Status + +This is the **skeleton** stage. The harness class structure is in +place; thread bodies are intentionally stubs that exit immediately. +The pytest test exercises the lifecycle (`setup → start → +wait_until_done → stop`) only. + +Thread bodies are implemented incrementally in subsequent commits, +in roughly this order (read-only first, side-effecting later): + +1. `log_scanner_thread` (read-only — easiest) +2. `metrics_thread` (read-only — almost as easy) +3. `injector_thread` (subprocess control) +4. `canary_thread` (HTTP client + token-equivalence) +5. `load_thread` (wraps existing `run_cancel_stress_test`) + +## File layout + +``` +tests/integration/defs/stress_test/disagg_cancel/ +├── README.md (this file) +├── __init__.py +├── harness.py (DisaggCancellationStressHarness) +├── test_disagg_cancel_stress.py (pytest entry point) +└── configs/ + ├── README.md (YAML schema + how to add a config) + ├── marathon_cpp_v1_deepseek.yaml + └── marathon_python_v2_qwen.yaml (placeholder; not yet parametrized) +``` + +Future additions: +- `tools/generate_canary_references.py` — one-shot reference generator + that records greedy-decode token IDs for the canary prompts. +- `configs/stress_canary_prompts.json` — canary prompts + recorded + reference token IDs (consumed by the canary thread). +- Per-scenario YAMLs covering additional axes: 1P1D, 4P2D, + V1+Python, UCX, block-reuse-off, overlap-off, aggressive-timeout, + multi-node (all Python-only test-side configuration). + +## How to run + +The marathons are **not** registered in pre-merge CI. They are run +nightly / weekly via +`tests/integration/test_lists/qa/llm_function_stress.txt` (wiring +lands together with the load-thread implementation). + +### Local smoke (skeleton stage) + +```bash +cd /path/to/TensorRT-LLM +LLM_MODELS_ROOT=/path/to/models \ + pytest -sv tests/integration/defs/stress_test/disagg_cancel/ +``` + +In the skeleton stage this should complete in seconds because all +threads are no-ops; it only verifies the harness lifecycle compiles +and the YAMLs parse. + +### Local marathon (once thread bodies are wired) + +Once thread bodies are implemented, the same command will run the +full 2-hour marathon against the C++ marathon config. To run a shorter smoke +during development, set `duration_min: 10` and trim +`injections:` in the YAML. + +## Pass criteria + +A marathon run is "clean" iff all of the following hold: + +- No hard-zero log patterns (e.g. `Cannot cancel request`, `Broken + promise`, `unquiesced`, double-free / UAF traces) appear in any + worker's stderr. +- All workers (context + generation) are alive at the end of the + marathon. The injector-induced respawns must succeed; subsequent + SIGKILLs would otherwise be observable as never-recovered ranks. +- Final 5/5 canary requests pass within 30 s of marathon end, with + 100 % token-equivalence against the recorded references. +- Canary error rate < 1 % overall and < 10 % within any 1-min + injection window. +- Recovery time < 30 s after each SIGCONT / SIGKILL-respawn event + (measured against the canary stream). +- KV-cache utilization growth ≤ 10 percentage points end-to-end + (leak guard). + +Concrete thresholds for each metric are declared in the marathon +YAML's `pass_criteria:` block. + +## How to debug a failure + +(Stub — the full debug guide lands together with the thread +implementations.) + +For now, when the skeleton test fails: + +1. Confirm the YAML parses: + ```bash + python -c "from harness import StressConfig; StressConfig.from_yaml_path('configs/marathon_cpp_v1_deepseek.yaml')" + ``` +2. Check the `failure_reason` field in `collect_results()` output. +3. Look at the pytest stdout for harness `logger` lines (each thread + logs its identity on entry / exit). + +## Cross-references + +- [PR #13713](https://github.com/NVIDIA/TensorRT-LLM/pull/13713) — + the bug fix this suite gates regressions against. +- [TRTLLM-12648](https://jirasw.nvidia.com/browse/TRTLLM-12648), + [TRTLLM-12721](https://jirasw.nvidia.com/browse/TRTLLM-12721) — + the cancellation / poison hardening initiative this suite is part + of. +- `tests/integration/defs/disaggregated/test_disaggregated.py` — + `run_cancel_stress_test`, `setup_disagg_cluster` (the + building blocks the harness composes). diff --git a/tests/integration/defs/stress_test/disagg_cancel/__init__.py b/tests/integration/defs/stress_test/disagg_cancel/__init__.py new file mode 100644 index 000000000000..dc5e857d9baf --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/__init__.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Disaggregated KV-transfer cancellation stress-test harness. + +Marathon-style stress suite that drives a disaggregated TRT-LLM +cluster under cancellation-heavy load for hours and asserts the +deployment recovers cleanly from transient + terminal worker +failures. See the directory ``README.md`` for the contract and +operational guidance. +""" diff --git a/tests/integration/defs/stress_test/disagg_cancel/configs/README.md b/tests/integration/defs/stress_test/disagg_cancel/configs/README.md new file mode 100644 index 000000000000..a95545b7284a --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/configs/README.md @@ -0,0 +1,62 @@ +# Marathon configs + +Per-marathon YAML files consumed by ``test_disagg_cancel_stress.py``. + +| File | Model | KV cache | Transceiver | +|------|-------|----------|-------------| +| `marathon_cpp_v1_deepseek.yaml` | `DeepSeek-V3-Lite/bf16` (MLA) | V1 | C++ | +| `marathon_python_v2_qwen.yaml` | `Qwen2.5-7B-Instruct` (GQA) | V2 | Python | + +Only configurations listed in `_MARATHON_CONFIGS` in +`test_disagg_cancel_stress.py` are actually parametrized; the other +YAMLs live in this directory as ready-to-wire templates. + +## Schema + +The YAML extends the existing +`tests/integration/defs/disaggregated/test_configs/disagg_config_cancel_stress_test*.yaml` +schema. The top-level `hostname / model / backend / context_servers / +generation_servers` keys pass through to +`tests/integration/defs/disaggregated/disagg_test_utils.py::setup_disagg_cluster` +unchanged. + +The new `stress_config:` top-level block is consumed by +`harness.py::StressConfig`. Field-level documentation lives in +`StressConfig` itself (dataclass field docstrings) and the +example values in `marathon_cpp_v1_deepseek.yaml`. + +## Backend-knob axis: KV-cache manager × transceiver runtime + +Two knobs select which (KV cache manager × transceiver runtime) +backend the marathon exercises: + +- `stress_config.kv_cache_manager: v1 | v2` controls + `kv_cache_config.use_kv_cache_manager_v2: false | true` on each + worker. V1 is the C++ KV cache manager (`KVCacheManager`); V2 is + the newer pure-Python manager (`KVCacheManagerV2`). +- `stress_config.transceiver: cpp | python` controls + `cache_transceiver_config.transceiver_runtime: "CPP" | "PYTHON"` + on each worker. `cpp` selects the C++-backed transceiver + (`BindKvCacheTransceiver`); `python` selects the pure-Python + transceiver (`KvCacheTransceiverV2`). + +The `(v2, cpp)` combination is unsupported (the C++ transceiver +requires the V1 KV cache manager) and rejected by +`StressConfig.validate()`. + +## Adding a new YAML + +Additional marathon scenarios land as new YAMLs here, with **no +Python changes** required beyond extending the parametrize list. To +add a new config: + +1. Copy `marathon_cpp_v1_deepseek.yaml` as a template. +2. Adjust `model`, `kv_cache_manager`, `transceiver`, and any + load-shape knobs (`base_concurrency`, `client_cancel_rate`, + `output_length`, `injections:`, `pass_criteria:`). +3. Add the new filename to `_MARATHON_CONFIGS` in + `test_disagg_cancel_stress.py`. +4. (If applicable) generate canary references for the new model and + add the reference JSON next to this directory. +5. Register the new test ID in + `tests/integration/test_lists/qa/llm_function_stress.txt`. diff --git a/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_cpp_v1_deepseek.yaml b/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_cpp_v1_deepseek.yaml new file mode 100644 index 000000000000..6f8258b1c5aa --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_cpp_v1_deepseek.yaml @@ -0,0 +1,129 @@ +# C++ transceiver marathon: V1 KV cache + C++ transceiver + NIXL, DeepSeek-class MLA. +# +# Exercises the C++-backed disagg path (BindKvCacheTransceiver + V1 +# KVCacheManager + NIXL backend) — the configuration NVBug 6104831 +# was filed against and that PR #13713 stabilizes. Schema +# documentation lives in ../README.md. + +hostname: localhost +model: DeepSeek-V3-Lite/bf16 +backend: pytorch + +context_servers: + num_instances: 3 # 3P3D cluster (3 context + 3 generation workers) + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + disable_overlap_scheduler: false # overlap ON + max_num_tokens: 16384 + max_seq_len: 16384 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true # block reuse ON + enable_partial_reuse: true + free_gpu_memory_fraction: 0.3 + use_kv_cache_manager_v2: false # V1 KV cache (default) + cache_transceiver_config: + backend: NIXL + transceiver_runtime: CPP # C++ transceiver (default; explicit for clarity) + max_tokens_in_buffer: 16384 + kv_transfer_timeout_ms: 60000 # production default + +generation_servers: + num_instances: 3 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + disable_overlap_scheduler: false + max_num_tokens: 16384 + max_seq_len: 16384 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true + enable_partial_reuse: true + free_gpu_memory_fraction: 0.3 + use_kv_cache_manager_v2: false + cache_transceiver_config: + backend: NIXL + transceiver_runtime: CPP + max_tokens_in_buffer: 16384 + kv_transfer_timeout_ms: 60000 + +# ============================================================================ +# Stress harness configuration (consumed by harness.py::StressConfig). +# Schema documented in ../README.md. +# ============================================================================ +stress_config: + duration_min: 120 # 2 h marathon + + # Backend-knob axis selectors. Must match + # context_servers / generation_servers above. + kv_cache_manager: v1 # V1 (C++) KV cache manager + transceiver: cpp # C++-backed transceiver (BindKvCacheTransceiver) + + base_concurrency: 64 + client_cancel_rate: 0.10 + input_length: + distribution: uniform + min_tokens: 4096 + max_tokens: 12288 + output_length: 512 + + bursts: + interval_min: 8 + concurrency: 256 + duration_s: 60 + input_length: + distribution: uniform + min_tokens: 12288 + max_tokens: 16384 + + injections: + - at_min: 15 + type: sigstop + target: gen_worker_random + duration_s: 20 + - at_min: 30 + type: sigstop + target: gen_worker_random + duration_s: 30 + - at_min: 45 + type: sigstop + target: ctx_worker_random + duration_s: 20 + - at_min: 60 + type: sigkill + target: gen_worker_0 + respawn_within_s: 60 + - at_min: 75 + type: sigstop + target: gen_worker_random + duration_s: 20 + - at_min: 90 + type: sigstop + target: ctx_worker_random + duration_s: 30 + - at_min: 105 + type: sigstop + target: gen_worker_random + duration_s: 20 + + canary: + prompts_file: stress_canary_prompts.json + rate_per_min: 5 + greedy_decoding: true + seed: 42 + max_tokens: 128 + check_token_equivalent: true + error_rate_overall_max: 0.01 + error_rate_injection_window_max: 0.10 + recovery_time_max_s: 30 + + log_scan: + hard_zero_patterns: + - "Broken promise" + - "NO RECOVERY" + - "Segfault" + - "SIGSEGV" + - "0xffffffffffffffff" + - "Poisoned .* cache transfer buffer" + + kv_cache_growth_max: 0.10 # final utilization ≤ baseline + 10 percentage points diff --git a/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_python_v2_qwen.yaml b/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_python_v2_qwen.yaml new file mode 100644 index 000000000000..d74108335002 --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/configs/marathon_python_v2_qwen.yaml @@ -0,0 +1,129 @@ +# Python transceiver marathon: V2 KV cache + Python transceiver + NIXL, Qwen-class GQA. +# +# Exercises the pure-Python disagg path (KvCacheTransceiverV2 + +# KVCacheManagerV2 + NIXL backend) on a non-MLA, GQA-style model so +# the marathon coverage is symmetric across the two supported +# (KV-cache manager × transceiver runtime) backends. Schema +# documentation lives in ../README.md. +# +# This YAML is intentionally not parametrized in +# `_MARATHON_CONFIGS` yet; it ships as a ready-to-wire template that +# becomes active once canary references for Qwen2.5-7B-Instruct are +# recorded. + +hostname: localhost +model: Qwen2.5-7B-Instruct +backend: pytorch + +context_servers: + num_instances: 3 # 3P3D cluster (3 context + 3 generation workers) + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + disable_overlap_scheduler: false # overlap ON + max_num_tokens: 16384 + max_seq_len: 16384 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true # block reuse ON + enable_partial_reuse: true + free_gpu_memory_fraction: 0.3 + use_kv_cache_manager_v2: true # V2 KV cache + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON # Python transceiver + max_tokens_in_buffer: 16384 + kv_transfer_timeout_ms: 60000 + +generation_servers: + num_instances: 3 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + disable_overlap_scheduler: false + max_num_tokens: 16384 + max_seq_len: 16384 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true + enable_partial_reuse: true + free_gpu_memory_fraction: 0.3 + use_kv_cache_manager_v2: true + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + max_tokens_in_buffer: 16384 + kv_transfer_timeout_ms: 60000 + +stress_config: + duration_min: 120 + + kv_cache_manager: v2 + transceiver: python + + base_concurrency: 64 + client_cancel_rate: 0.10 + input_length: + distribution: uniform + min_tokens: 4096 + max_tokens: 12288 + output_length: 512 + + bursts: + interval_min: 8 + concurrency: 256 + duration_s: 60 + input_length: + distribution: uniform + min_tokens: 12288 + max_tokens: 16384 + + injections: + - at_min: 15 + type: sigstop + target: gen_worker_random + duration_s: 20 + - at_min: 30 + type: sigstop + target: gen_worker_random + duration_s: 30 + - at_min: 45 + type: sigstop + target: ctx_worker_random + duration_s: 20 + - at_min: 60 + type: sigkill + target: gen_worker_0 + respawn_within_s: 60 + - at_min: 75 + type: sigstop + target: gen_worker_random + duration_s: 20 + - at_min: 90 + type: sigstop + target: ctx_worker_random + duration_s: 30 + - at_min: 105 + type: sigstop + target: gen_worker_random + duration_s: 20 + + canary: + prompts_file: stress_canary_prompts.json + rate_per_min: 5 + greedy_decoding: true + seed: 42 + max_tokens: 128 + check_token_equivalent: true + error_rate_overall_max: 0.01 + error_rate_injection_window_max: 0.10 + recovery_time_max_s: 30 + + log_scan: + hard_zero_patterns: + - "Broken promise" + - "NO RECOVERY" + - "Segfault" + - "SIGSEGV" + - "0xffffffffffffffff" + - "Poisoned .* cache transfer buffer" + + kv_cache_growth_max: 0.10 diff --git a/tests/integration/defs/stress_test/disagg_cancel/harness.py b/tests/integration/defs/stress_test/disagg_cancel/harness.py new file mode 100644 index 000000000000..0b324a993037 --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/harness.py @@ -0,0 +1,705 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Disaggregated cancellation stress-test harness. + +Five-thread architecture that drives a 3P3D disaggregated TRT-LLM +cluster under cancellation-heavy load for hours, with scheduled +failure injection (SIGSTOP / SIGCONT / SIGKILL+respawn), a parallel +canary client for use-after-free detection, log-pattern fail-fast, +and KV-cache utilization scraping for leak detection. + +The class structure and lifecycle (``setup`` -> ``start`` -> +``wait_until_done`` -> ``stop`` -> ``collect_results``) is wired up +here; individual thread bodies are filled in incrementally as the +suite matures. +""" + +from __future__ import annotations + +import logging +import re +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import IO, Any, Callable, Optional + +import yaml + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Config dataclasses +# --------------------------------------------------------------------------- + +# YAML -> Python type coercion for the typed ``stress_config`` fields. +# Keys must match the ``StressConfig`` dataclass field names. Defaults +# live on the dataclass declarations themselves; missing YAML keys +# simply aren't passed to the constructor, so the field defaults +# apply automatically and are not duplicated here. +_STRESS_CONFIG_COERCERS: dict[str, Callable[[Any], Any]] = { + "duration_min": int, + "kv_cache_manager": str, + "transceiver": str, + "base_concurrency": int, + "client_cancel_rate": float, + "output_length": int, +} + + +@dataclass +class StressConfig: + """Parsed contents of a marathon YAML's ``stress_config:`` block. + + The schema is documented in the directory ``README.md``. The + fields here are exposed as a flat dataclass so the harness can + pass them around without re-parsing. + """ + + duration_min: int = 120 + kv_cache_manager: str = "v1" # v1 | v2 (v2 + CPP is invalid) + transceiver: str = "cpp" # cpp | python + base_concurrency: int = 64 + client_cancel_rate: float = 0.10 + output_length: int = 512 + # Full YAML subtree, for fields the harness consumes lazily + # (bursts, injections, canary, log_scan, kv_cache_growth_max). + raw: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_yaml_path(cls, path: Path) -> "StressConfig": + """Parse and validate a marathon YAML in one call. + + Callers that construct ``StressConfig`` directly via + ``__init__`` (no current consumer, but the API is left open) + remain responsible for invoking ``validate()`` themselves. + + Args: + path: Path to a marathon YAML containing a top-level + ``stress_config:`` block. See ``configs/README.md`` + for the schema. + + Returns: + A validated ``StressConfig`` whose typed fields are + coerced from the YAML and whose ``raw`` attribute + carries the full ``stress_config:`` subtree. + + Raises: + ValueError: If the YAML is missing the ``stress_config:`` + block, or if ``validate()`` rejects the resulting + backend-knob combination. + yaml.YAMLError: If the file is not valid YAML. + OSError: If ``path`` cannot be opened. + """ + with path.open("r", encoding="utf-8") as f: + doc = yaml.safe_load(f) + if "stress_config" not in doc: + raise ValueError( + f"YAML at {path} is missing the 'stress_config:' top-level block; " + "see configs/README.md for the schema." + ) + sc = doc["stress_config"] + kwargs: dict[str, Any] = {"raw": sc} + for name, coerce in _STRESS_CONFIG_COERCERS.items(): + if name in sc: + kwargs[name] = coerce(sc[name]) + cfg = cls(**kwargs) + cfg.validate() + return cfg + + def validate(self) -> None: + """Reject backend-knob combinations that are not supported. + + Raises: + ValueError: If ``kv_cache_manager`` is not ``"v1"`` or + ``"v2"``, if ``transceiver`` is not ``"cpp"`` or + ``"python"``, or if the pair ``(v2, cpp)`` is + supplied (the C++ transceiver only supports the V1 + KV cache manager). + """ + if self.kv_cache_manager == "v2" and self.transceiver == "cpp": + # The C++ transceiver (BindKvCacheTransceiver) only supports + # the V1 KV cache manager. V2 must be paired with the Python + # transceiver (KvCacheTransceiverV2). + raise ValueError( + "(kv_cache_manager=v2, transceiver=cpp) is an unsupported " + "combination; pair V2 with the Python transceiver." + ) + if self.kv_cache_manager not in ("v1", "v2"): + raise ValueError( + f"kv_cache_manager must be 'v1' or 'v2', got {self.kv_cache_manager!r}" + ) + if self.transceiver not in ("cpp", "python"): + raise ValueError(f"transceiver must be 'cpp' or 'python', got {self.transceiver!r}") + + +@dataclass +class WorkerLaunchSpec: + """Shadow-tracked launch context for a single worker. + + Recorded at cluster-setup time so the injector thread can relaunch + a SIGKILLed worker via ``_run_worker(*spec)`` without extending + ``ProcessWrapper`` in shared disagg test infrastructure. + """ + + role: str # "ctx" or "gen" + index: int # 0..(num_instances-1), e.g. gen_worker_0 + model_name: str + worker_config: dict[str, Any] + work_dir: str + port: int # the originally-allocated port; respawn may end up on a different one + device: str # CUDA_VISIBLE_DEVICES string, e.g. "0" or "0,1" + env: dict[str, str] + # Worker stdout/stderr log path (from ``ProcessWrapper.log_path``). + # ``None`` when launched with ``save_log=False`` (output inherits + # pytest stdout); log_scanner skips ``None`` paths with a warning. + log_path: Optional[str] = None + + +# --------------------------------------------------------------------------- +# Log-scanner helpers +# --------------------------------------------------------------------------- + + +@dataclass +class _LogSource: + r"""One tailed log file plus the per-file state the scanner needs. + + The scanner runs in a poll loop (single thread, single file + descriptor per source) rather than a per-source thread, because + file tailing is bursty and the number of sources is small (3 ctx + + 3 gen + 1 disagg server = 7 fds in the common 3P3D shape). + + Why ``_carry`` exists: ``trtllm-serve``'s C++ stack is + block-buffered when stdout is a file, so reads are chunked rather + than line-bounded. A naive ``for line in fh:`` loop would either + block waiting for newlines (defeating the whole point of polling) + or silently swallow the partial trailing line. We read all + available bytes per poll, prepend the previous tail-byte carry, + split on ``\n``, and treat the final element as either a complete + line (if the chunk ended on a newline) or carry for the next + poll. + """ + + spec: WorkerLaunchSpec + path: Path + _fh: Optional[IO[str]] = None + _carry: str = "" + + def poll( + self, + patterns: list[tuple[str, re.Pattern[str]]], + mark_failed: Callable[[str], None], + ) -> bool: + """Read new content and scan; report any pattern hit via ``mark_failed``. + + Maintains a tail-byte carry (``_carry``) across calls so + block-buffered C++ writes (chunked rather than line-bounded) + reconstruct into whole lines before being matched against + patterns. + + Args: + patterns: Compiled hard-zero patterns as + ``(source_str, regex)`` pairs. ``source_str`` is the + original YAML string and is embedded in the + ``mark_failed`` reason for debugging. + mark_failed: Callback invoked on the first pattern hit. + Treated as one-shot per ``poll()`` call (the first + match returns immediately without scanning the + remainder of the chunk). + + Returns: + True if a pattern matched and ``mark_failed`` was + called; False if no new content was available, no + pattern matched, or the file did not yet exist (handled + silently so the scanner can be started before every + worker has flushed its first bytes). + """ + if self._fh is None: + if not self.path.exists(): + return False + self._fh = self.path.open("r", encoding="utf-8", errors="replace") + + chunk = self._fh.read() + if not chunk: + return False + + buf = self._carry + chunk + lines = buf.split("\n") + # If buf ended on '\n', lines[-1] == "" (a complete line + # terminator). Either way, lines[-1] is the partial-or-empty + # tail carried into the next poll. + self._carry = lines[-1] + + for line in lines[:-1]: + for pat_str, pat in patterns: + if pat.search(line): + role_idx = f"{self.spec.role}_{self.spec.index}" + mark_failed( + f"hard-zero log pattern hit in {role_idx} " + f"({self.path.name}): {pat_str!r} matched {line.strip()!r}" + ) + return True + return False + + def close(self) -> None: + """Close the tailed file handle, if any. + + Idempotent. ``OSError`` from the underlying close is logged + at DEBUG and swallowed — the scanner is exiting and a + failed close is not worth tripping fail-fast. + """ + if self._fh is not None: + try: + self._fh.close() + except OSError: + logger.debug("[log_scanner] closing %s raised; ignoring", self.path) + self._fh = None + + +def _compile_patterns(raw_patterns: list[Any]) -> list[tuple[str, re.Pattern[str]]]: + """Compile the YAML's ``log_scan.hard_zero_patterns`` list to regex objects. + + Patterns that fail to compile are skipped with an ERROR log so + config typos surface during the scanner's startup banner rather + than as silent misses at marathon hour 1.5. + + Args: + raw_patterns: Pattern entries as parsed from YAML; non-string + entries are skipped with an ERROR log. + + Returns: + List of ``(source_str, compiled_regex)`` pairs. + ``source_str`` is the original YAML string, retained so + failure reasons can name which pattern matched. + """ + compiled: list[tuple[str, re.Pattern[str]]] = [] + for entry in raw_patterns: + if not isinstance(entry, str): + logger.error( + "[log_scanner] hard_zero_patterns entry %r is not a string; skipping", + entry, + ) + continue + try: + compiled.append((entry, re.compile(entry))) + except re.error as exc: + logger.error( + "[log_scanner] failed to compile hard_zero pattern %r: %s; skipping", + entry, + exc, + ) + return compiled + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +class DisaggCancellationStressHarness: + """Marathon harness coordinating cluster + load + canary + injector + log scan + metrics. + + Lifecycle: ``__init__`` → ``setup`` → ``start`` → ``wait_until_done`` + → ``stop`` → ``collect_results``. ``stop`` is idempotent; the + pytest fixture should call it in ``finally`` to guarantee teardown. + + Threads are all daemon=True so they don't block process exit on + catastrophic failure. Coordination via two ``threading.Event``s: + + - ``stop_event``: set by ``stop()`` (clean shutdown). + - ``failed_event``: set by any thread on fail-fast condition (hard-zero + log pattern, worker death). All other threads observe it and + wind down promptly. + + Thread-based composition (rather than asyncio) keeps the + subprocess-control injector, the file-tailing log scanner, and + the HTTP/Prometheus metrics scraper failure-isolated and debugged + independently. The load and canary threads each run their own + asyncio event loops internally for HTTP I/O. + """ + + def __init__( + self, + yaml_path: Path, + *, + log_scanner_poll_interval_s: float = 0.5, + ) -> None: + """Construct a marathon harness. + + Args: + yaml_path: Path to a marathon YAML; loaded and validated + eagerly via ``StressConfig.from_yaml_path``. + log_scanner_poll_interval_s: Poll cadence (seconds) for + the log-scanner thread. Default 0.5 s is reactive + enough for human-scale debugging without becoming a + measurable load source on its own; tests pass a + smaller value (e.g. 0.02 s) to keep real-clock + latency bounded. + + Raises: + ValueError: If the YAML is malformed or its + ``stress_config:`` block is missing / rejects + validation. + """ + self.yaml_path: Path = yaml_path + self.config: StressConfig = StressConfig.from_yaml_path(yaml_path) + + # Coordination primitives. + self.stop_event: threading.Event = threading.Event() + self.failed_event: threading.Event = threading.Event() + self._failure_reason: Optional[str] = None + self._failure_lock = threading.Lock() + + # Per-thread tunables. The default cadence is reactive enough + # for human-scale debugging (~0.5 s lag from log line to + # fail-fast) without becoming a measurable load source on its + # own; tests pass a smaller value via the constructor to keep + # real-clock latency bounded. + self._log_scanner_poll_interval_s: float = log_scanner_poll_interval_s + + # Cluster + worker tracking (populated by setup()). + self._cluster: Any = None # tuple returned by setup_disagg_cluster + self._worker_specs: list[WorkerLaunchSpec] = [] + + # Thread handles (populated by start()). + self._load_thread: Optional[threading.Thread] = None + self._canary_thread: Optional[threading.Thread] = None + self._injector_thread: Optional[threading.Thread] = None + self._log_scanner_thread: Optional[threading.Thread] = None + self._metrics_thread: Optional[threading.Thread] = None + + # Result collection. Populated by thread bodies as they run. + self._canary_records: list[dict[str, Any]] = [] + self._kv_utilization_samples: list[dict[str, Any]] = [] + self._injection_events: list[dict[str, Any]] = [] + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def setup(self) -> None: + """Launch the disagg cluster from the YAML and record launch specs. + + Stub: real implementation delegates to ``setup_disagg_cluster`` + in ``tests/integration/defs/disaggregated/test_disaggregated.py`` + and shadow-tracks per-worker ``WorkerLaunchSpec`` so the + injector thread can later relaunch a SIGKILLed worker without + modifying shared infrastructure. + """ + logger.info("[harness] setup() — stub: cluster not actually launched") + + def start(self) -> None: + """Spawn the five worker threads. Returns immediately. + + Stub stage: each thread body is a no-op that returns + immediately. The load-thread stub signals ``stop_event`` on + exit so the lifecycle smoke ``start() -> wait_until_done() -> + stop()`` completes cleanly without waiting out the + ``wait_until_done`` timeout. + """ + logger.info("[harness] start() — spawning 5 stub threads") + self._load_thread = threading.Thread( + target=self._load_thread_body, name="stress-load", daemon=True + ) + self._canary_thread = threading.Thread( + target=self._canary_thread_body, name="stress-canary", daemon=True + ) + self._injector_thread = threading.Thread( + target=self._injector_thread_body, name="stress-injector", daemon=True + ) + self._log_scanner_thread = threading.Thread( + target=self._log_scanner_thread_body, name="stress-log-scanner", daemon=True + ) + self._metrics_thread = threading.Thread( + target=self._metrics_thread_body, name="stress-metrics", daemon=True + ) + for t in self._all_threads(): + t.start() + + def wait_until_done(self, timeout_s: Optional[float] = None) -> bool: + """Block until ``stop_event`` or ``failed_event`` is set, or timeout. + + Stub: in this skeleton, ``start()``'s no-op threads exit + immediately and set ``stop_event`` themselves, so this returns + ``True`` almost instantly. + + Deadline is a ``time.monotonic()`` reading computed once and + checked on each 0.5 s poll wake-up (worst-case lateness ~0.5 + s, well below the marathon's ``timeout_s`` scale). The earlier + ``threading.Timer`` approach leaked a non-daemon timer thread + for the residual window on early-stop. + + Args: + timeout_s: Optional ceiling on the wait, in seconds. + ``None`` (the default) waits indefinitely until one + of the events fires. + + Returns: + True if stopped cleanly (``stop_event``); False if any + thread tripped fail-fast (``failed_event``) or the + timeout expired without either event being set. + """ + deadline_at = None if timeout_s is None else (time.monotonic() + timeout_s) + while not self.stop_event.is_set() and not self.failed_event.is_set(): + if deadline_at is not None and time.monotonic() >= deadline_at: + logger.warning("[harness] wait_until_done timed out after %ss", timeout_s) + return False + # Sleep with a small poll cadence so stop()/fail-fast wake us promptly. + self.stop_event.wait(timeout=0.5) + return not self.failed_event.is_set() + + def stop(self) -> None: + """Signal threads to wind down, join them, then tear down the cluster. + + Idempotent. Safe to call from pytest's ``finally``. + + Threads that fail to join within the join timeout are logged + as a warning but do not trip fail-fast on their own: a + stale-thread leak during teardown is a hygiene issue, not a + verdict on whether the marathon itself passed its pass + criteria (failure_reason is reserved for actual harness + observations like hard-zero log patterns, worker death, or + respawn-timeout). The cluster is torn down regardless so its + GPUs/ports do not leak alongside the thread. + """ + if not self.stop_event.is_set(): + logger.info("[harness] stop() — setting stop_event") + self.stop_event.set() + for t in self._all_threads(): + if t is not None and t.is_alive(): + t.join(timeout=10.0) + if t.is_alive(): + logger.warning("[harness] thread %s did not join within 10s", t.name) + self._teardown_cluster() + + # ------------------------------------------------------------------ + # Fail-fast support + # ------------------------------------------------------------------ + + def mark_failed(self, reason: str) -> None: + """Trip the fail-fast event with a structured reason. + + First-reason-wins: subsequent calls (e.g. a second worker's + log scan firing right after the first) are silently dropped + so ``failure_reason`` reflects the original root cause rather + than a later cascade. + + Called by ``log_scanner_thread`` on hard-zero pattern hit, by + ``injector_thread`` on respawn timeout, by ``wait_until_done`` + on worker death detection, etc. + + Args: + reason: Human-readable failure description. Logged at + ERROR and exposed via the ``failure_reason`` + property. + """ + with self._failure_lock: + if self._failure_reason is None: + self._failure_reason = reason + logger.error("[harness] FAIL-FAST tripped: %s", reason) + self.failed_event.set() + + @property + def failure_reason(self) -> Optional[str]: + """Reason of the first ``mark_failed`` call, or ``None`` if not tripped. + + Read under the failure lock so it stays consistent with + ``failed_event``: if ``failed_event.is_set()`` is True, this + is guaranteed to return a non-None string. + """ + with self._failure_lock: + return self._failure_reason + + # ------------------------------------------------------------------ + # Result collection (called by pytest after stop()) + # ------------------------------------------------------------------ + + def collect_results(self) -> dict[str, Any]: + """Aggregate per-thread observations into a final pass/fail dict. + + Stub: returns empty buckets in the skeleton. The full + implementation computes canary error rates, recovery times, + and KV-cache growth from ``self._canary_records``, etc. + + Returns: + A dict with four keys (the list values are copies, safe + for the caller to mutate without affecting the harness): + + - ``canary_records``: per-canary request outcomes. + - ``kv_utilization_samples``: timestamped KV-cache + utilization scrapes from the metrics thread. + - ``injection_events``: SIGSTOP / SIGCONT / SIGKILL + events fired by the injector thread. + - ``failure_reason``: ``None`` if the marathon completed + cleanly, otherwise the first reason passed to + ``mark_failed``. + """ + return { + "canary_records": list(self._canary_records), + "kv_utilization_samples": list(self._kv_utilization_samples), + "injection_events": list(self._injection_events), + "failure_reason": self.failure_reason, + } + + # ------------------------------------------------------------------ + # Thread bodies (stubs — implemented incrementally) + # ------------------------------------------------------------------ + + def _load_thread_body(self) -> None: + """Wrap ``run_cancel_stress_test`` in a duration-bounded loop. + + Stub: no-op that immediately signals end-of-marathon via + ``stop_event``. The real implementation loops until either + ``duration_min`` elapses or ``stop_event`` is set, calling + ``run_cancel_stress_test`` repeatedly; at end-of-marathon it + sets ``stop_event`` so the other four threads wind down. + Setting ``stop_event`` here in the stub preserves that + downstream contract and lets ``wait_until_done`` return + cleanly from the lifecycle smoke. + """ + logger.debug("[load_thread] stub — exiting and signalling stop_event") + self.stop_event.set() + + def _canary_thread_body(self) -> None: + """Send greedy-decode canaries, check token-equivalence. + + Stub: no-op. Real implementation loads + ``stress_canary_prompts.json``, sends 5 reqs/min, asserts + token IDs match the recorded reference. + """ + logger.debug("[canary_thread] stub — exiting immediately") + + def _injector_thread_body(self) -> None: + """Fire SIGSTOP / SIGCONT / SIGKILL+respawn on the configured schedule. + + Stub: no-op. Real implementation reads ``injections:`` schedule + from config and uses ``os.kill`` + ``_run_worker`` to act on + the shadow-tracked ``self._worker_specs``. + """ + logger.debug("[injector_thread] stub — exiting immediately") + + def _log_scanner_thread_body(self) -> None: + """Tail all worker logs; fail-fast on any hard-zero pattern hit. + + Reads ``hard_zero_patterns`` from ``stress_config.log_scan``, + compiles each as a regex, then poll-tails every worker's + ``log_path`` at ``_log_scanner_poll_interval_s``. On the first + match in any worker, ``mark_failed`` is called (idempotent — + first-reason-wins in the existing skeleton plumbing) and the + thread continues polling until ``stop_event`` / + ``failed_event``: the first failure is enough for fail-fast, + but tailing past it costs us nothing and keeps the file + handles drained until teardown. + + Workers without a ``log_path`` (``save_log=False`` at launch) + are skipped with a warning. The scanner is intentionally + permissive about not-yet-created files: ``_LogSource.poll`` + retries lazily on each cycle, so the thread can be started + before every worker has flushed its first bytes. + """ + raw_log_scan = self.config.raw.get("log_scan") or {} + patterns = _compile_patterns(raw_log_scan.get("hard_zero_patterns") or []) + if not patterns: + logger.warning( + "[log_scanner] no usable hard_zero_patterns; exiting immediately. " + "Check stress_config.log_scan.hard_zero_patterns in the YAML." + ) + return + + sources: list[_LogSource] = [] + for spec in self._worker_specs: + if spec.log_path is None: + logger.warning( + "[log_scanner] worker %s_%d has no log_path; skipping " + "(launch with save_log=True to capture worker output)", + spec.role, + spec.index, + ) + continue + sources.append(_LogSource(spec=spec, path=Path(spec.log_path))) + + if not sources: + logger.warning( + "[log_scanner] no log sources to tail; exiting immediately. " + "The marathon will run without log-pattern fail-fast coverage." + ) + return + + logger.info( + "[log_scanner] tailing %d worker log(s) against %d hard_zero pattern(s)", + len(sources), + len(patterns), + ) + + try: + while not self.stop_event.is_set() and not self.failed_event.is_set(): + for source in sources: + if self.stop_event.is_set() or self.failed_event.is_set(): + break + source.poll(patterns, self.mark_failed) + self.stop_event.wait(timeout=self._log_scanner_poll_interval_s) + finally: + for source in sources: + source.close() + logger.debug("[log_scanner] exiting; closed %d source(s)", len(sources)) + + def _metrics_thread_body(self) -> None: + """Scrape ``/prometheus/metrics`` for KV-cache utilization at ~30 s cadence. + + Stub: no-op. Real implementation parses + ``trtllm_kv_cache_utilization`` and appends timestamped + samples to ``self._kv_utilization_samples``. + """ + logger.debug("[metrics_thread] stub — exiting immediately") + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _all_threads(self) -> list[threading.Thread]: + """Return the subset of thread handles that have been instantiated. + + Used by ``start()`` to launch and by ``stop()`` to join. + Filters out ``None`` so callers can iterate without guarding + (e.g. when a thread handle was never populated because + ``start()`` failed partway through). + + Returns: + Thread handles in deterministic order (load, canary, + injector, log_scanner, metrics), skipping any not yet + instantiated. + """ + return [ + t + for t in ( + self._load_thread, + self._canary_thread, + self._injector_thread, + self._log_scanner_thread, + self._metrics_thread, + ) + if t is not None + ] + + def _teardown_cluster(self) -> None: + """Best-effort cluster shutdown via ``terminate()``. + + Stub: no-op since ``setup()`` doesn't actually launch yet. + """ + if self._cluster is None: + return + logger.info("[harness] _teardown_cluster — stub") diff --git a/tests/integration/defs/stress_test/disagg_cancel/test_disagg_cancel_stress.py b/tests/integration/defs/stress_test/disagg_cancel/test_disagg_cancel_stress.py new file mode 100644 index 000000000000..7065e0a53ab8 --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/test_disagg_cancel_stress.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Marathon-style cancellation stress tests for disagg KV transfer. + +Parametrized pytest entry point for the cancellation stress suite; +the actual work happens in ``harness.py``. Marathon YAMLs live in +``configs/`` next to this file, one per backend-knob combination +(KV cache manager x transceiver runtime). + +Additional marathon configurations land here as the suite expands; +each one is added to ``_MARATHON_CONFIGS`` below. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from .harness import DisaggCancellationStressHarness, StressConfig + +_CONFIG_DIR = Path(__file__).parent / "configs" + +# Marathon configurations exercised by the parametrized test below. +# Add a new entry here to wire an additional YAML into the suite. +_MARATHON_CONFIGS: list[str] = [ + "marathon_cpp_v1_deepseek.yaml", +] + + +def test_all_marathon_yamls_parse_and_validate() -> None: + """Every ``marathon_*.yaml`` in ``configs/`` must parse and validate. + + Catches YAML-syntax and schema-validity regressions in marathon + configs that are not yet parametrized into ``_MARATHON_CONFIGS`` + (e.g. configs whose canary references are still being recorded). + Without it, broken-but-not-yet-wired YAMLs sit undetected until + the day they are parametrized, which can be PRs away. + """ + marathon_yamls = sorted(_CONFIG_DIR.glob("marathon_*.yaml")) + assert marathon_yamls, f"no marathon_*.yaml configs found under {_CONFIG_DIR}" + for path in marathon_yamls: + StressConfig.from_yaml_path(path) + + +@pytest.mark.parametrize("config_filename", _MARATHON_CONFIGS) +def test_disagg_cancellation_marathon(config_filename: str) -> None: + """Drive a long-running disagg cancellation marathon and assert pass criteria. + + Current scope: only what the already-implemented thread bodies + can contribute. The marathon entry point exists; the marathon + *content* lands incrementally as each thread body is wired up: + + - lifecycle plumbing (setup -> start -> wait -> stop -> + collect_results, fail-fast event propagation, dict-shape + contract). + - log-pattern fail-fast — a hard-zero pattern in any worker log + trips ``failure_reason`` via the log_scanner thread + (component-level coverage in ``test_log_scanner.py``). + + Marathon pass criteria not yet enforced here (will land alongside + their owning thread bodies in follow-up changes): canary error + rate, recovery time after each injection, KV-cache utilization + growth bound, injection-schedule completeness, sustained load + throughput. Until those land, this test passes trivially after + the lifecycle smoke completes; the value at this stage is that + the entry point and result-dict contract are pinned down so the + follow-up commits can extend in place rather than restructure. + """ + config_path = _CONFIG_DIR / config_filename + assert config_path.exists(), ( + f"Marathon config not found: {config_path}. " + "Did you regenerate the symlinks or rename the YAML?" + ) + + harness = DisaggCancellationStressHarness(config_path) + try: + harness.setup() + harness.start() + # Skeleton stage: stub threads exit immediately; the + # load-thread stub signals ``stop_event`` on exit so this + # returns cleanly (True) almost instantly. Once the + # duration-bounded load thread is wired up, the timeout + # becomes ``stress_config.duration_min`` plus a safety + # margin, and ``clean`` reports whether the marathon ran to + # completion without tripping fail-fast. + clean = harness.wait_until_done(timeout_s=10.0) + assert clean is True, ( + f"wait_until_done did not return cleanly; failure_reason={harness.failure_reason!r}" + ) + finally: + harness.stop() + + results = harness.collect_results() + # Skeleton stage: no real pass criteria yet. Just confirm the + # collector returns the expected shape so future commits can + # extend in place. + assert "canary_records" in results + assert "kv_utilization_samples" in results + assert "injection_events" in results + assert results["failure_reason"] is None, ( + f"Harness tripped fail-fast in skeleton run: {results['failure_reason']!r}" + ) diff --git a/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py b/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py new file mode 100644 index 000000000000..e874377f2fed --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py @@ -0,0 +1,522 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ``DisaggCancellationStressHarness._log_scanner_thread_body``. + +The log-scanner thread is the easiest component to test in isolation +because it has no dependency on a running disagg cluster: feed it a +list of ``WorkerLaunchSpec`` whose ``log_path`` points at a temp file +the test owns, write log lines into that file, and assert on +``harness.failure_reason``. + +Buffering caveat the production scanner has to handle (chunked C++ +``stdout`` writes from ``trtllm-serve``) is exercised by the +"partial / multi-line / no-trailing-newline" cases below. +""" + +from __future__ import annotations + +import re +import textwrap +import threading +import time +from pathlib import Path +from typing import Callable + +import pytest + +from .harness import DisaggCancellationStressHarness, WorkerLaunchSpec, _LogSource + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +_DUMMY_YAML = textwrap.dedent( + """\ + hostname: localhost + model: dummy + backend: pytorch + context_servers: {} + generation_servers: {} + stress_config: + duration_min: 1 + kv_cache_manager: v1 + transceiver: cpp + log_scan: + hard_zero_patterns: + - "Broken promise" + - "Segfault" + - "Poisoned .* cache transfer buffer" + """ +) + + +def _make_spec(role: str, index: int, log_path: Path | None) -> WorkerLaunchSpec: + """Tiny ``WorkerLaunchSpec`` factory for unit tests. + + Only ``role`` / ``index`` / ``log_path`` are scanner-relevant — + the other fields are placeholders that would normally be set by + ``setup_disagg_cluster``. + + Args: + role: Worker role tag, ``"ctx"`` or ``"gen"``. Surfaces in + failure reasons via ``_LogSource``. + index: Per-role index (``ctx_0``, ``gen_0``, ...). Surfaces + in failure reasons. + log_path: Path the scanner should tail, or ``None`` to + simulate ``save_log=False`` workers. + + Returns: + A ``WorkerLaunchSpec`` with placeholder values for the + scanner-irrelevant fields. + """ + return WorkerLaunchSpec( + role=role, + index=index, + model_name="dummy", + worker_config={}, + work_dir="/tmp", + port=18000 + index, + device="0", + env={}, + log_path=str(log_path) if log_path is not None else None, + ) + + +@pytest.fixture +def harness_with_two_workers(tmp_path: Path): + """Harness wired to two temp-file log sources (ctx_0 + gen_0). + + Args: + tmp_path: pytest-provided per-test temp dir. + + Returns: + A 3-tuple ``(harness, ctx_log, gen_log)`` where the + ``*_log`` entries are ``Path`` objects the test can write + to. The harness is constructed with a 20 ms scanner cadence + so test wall-clock times stay bounded. + """ + yaml_path = tmp_path / "stress.yaml" + yaml_path.write_text(_DUMMY_YAML) + + # Tighten the scanner cadence so the tests finish quickly. The + # default 0.5 s makes tests slow; 20 ms keeps real-clock latency + # bounded. + h = DisaggCancellationStressHarness(yaml_path, log_scanner_poll_interval_s=0.02) + + ctx_log = tmp_path / "worker_ctx_18000.log" + gen_log = tmp_path / "worker_gen_18001.log" + ctx_log.touch() + gen_log.touch() + + h._worker_specs = [ + _make_spec("ctx", 0, ctx_log), + _make_spec("gen", 0, gen_log), + ] + return h, ctx_log, gen_log + + +def _run_scanner_until_failure_or_timeout( + h: DisaggCancellationStressHarness, + timeout_s: float = 2.0, + settle_s: float = 0.05, +) -> bool: + """Spawn the log-scanner thread; wait for ``failed_event`` or timeout. + + Args: + h: Configured harness with ``_worker_specs`` already + populated by the calling test. + timeout_s: Maximum wall-clock seconds to wait for the + scanner to trip ``failed_event``. Defaults to 2.0 s. + settle_s: Pause after fail-fast fires (before + ``stop_event.set()``) to let the scanner clean up file + handles. Functionally optional; cosmetic for cleaner + debug logs. + + Returns: + True if fail-fast tripped within the timeout; False if the + scanner exited cleanly without firing (e.g. no log sources, + no patterns, or no matching content within the window). + """ + t = threading.Thread(target=h._log_scanner_thread_body, name="test-log-scanner", daemon=True) + t.start() + fired = h.failed_event.wait(timeout=timeout_s) + # Give the thread a moment to clean up file handles before we + # signal stop_event (this keeps the test's debug logs more + # readable; functionally optional). + time.sleep(settle_s) + h.stop_event.set() + t.join(timeout=2.0) + assert not t.is_alive(), "log_scanner thread did not exit after stop_event" + return fired + + +# --------------------------------------------------------------------------- +# Tests: pattern matching +# --------------------------------------------------------------------------- + + +def test_hard_zero_pattern_in_ctx_log_trips_fail_fast(harness_with_two_workers): + h, ctx_log, _ = harness_with_two_workers + ctx_log.write_text("startup ok\nBroken promise: future destroyed\ntail\n") + + fired = _run_scanner_until_failure_or_timeout(h) + + assert fired is True + assert h.failure_reason is not None + assert "Broken promise" in h.failure_reason + assert "ctx_0" in h.failure_reason + assert "worker_ctx_18000.log" in h.failure_reason + + +def test_hard_zero_pattern_in_gen_log_trips_fail_fast(harness_with_two_workers): + h, _, gen_log = harness_with_two_workers + gen_log.write_text("warmup\nSegfault at 0xdeadbeef\n") + + fired = _run_scanner_until_failure_or_timeout(h) + + assert fired is True + assert h.failure_reason is not None + assert "Segfault" in h.failure_reason + assert "gen_0" in h.failure_reason + + +def test_regex_metacharacters_match_correctly(harness_with_two_workers): + """Pattern 'Poisoned .* cache transfer buffer' must match across a wildcard.""" + h, _, gen_log = harness_with_two_workers + gen_log.write_text("Poisoned sender cache transfer buffer for request 42\n") + + fired = _run_scanner_until_failure_or_timeout(h) + + assert fired is True + assert h.failure_reason is not None + assert "Poisoned" in h.failure_reason + + +def test_benign_lines_do_not_trip_fail_fast(harness_with_two_workers): + h, ctx_log, gen_log = harness_with_two_workers + ctx_log.write_text( + "INFO: trtllm-serve listening on 18000\nINFO: cache transceiver initialised\nINFO: ready\n" + ) + gen_log.write_text("INFO: warmup complete\nINFO: serving requests\n") + + # No pattern hit ever, so we wait the full window then assert + # nothing fired. + fired = _run_scanner_until_failure_or_timeout(h, timeout_s=0.5) + + assert fired is False + assert h.failure_reason is None + + +def test_first_match_wins_when_both_workers_hit(harness_with_two_workers): + h, ctx_log, gen_log = harness_with_two_workers + ctx_log.write_text("Broken promise: A\n") + gen_log.write_text("Segfault at 0xbeef\n") + + fired = _run_scanner_until_failure_or_timeout(h) + + assert fired is True + reason_before = h.failure_reason + assert reason_before is not None + # mark_failed is documented as first-reason-wins. Another call + # should not overwrite the original reason. This is a guard + # against future refactors weakening that contract. + h.mark_failed("manual override that must not stick") + assert h.failure_reason == reason_before + + +# --------------------------------------------------------------------------- +# Tests: I/O quirks (no trailing newline, chunked writes) +# --------------------------------------------------------------------------- + + +def test_partial_line_without_trailing_newline_is_carried(harness_with_two_workers): + r"""A log write that doesn't end on '\n' must not lose the line. + + Simulates the C++ block-buffered-stdout pattern: two writes + where the first write ends mid-line and the second completes it. + The complete reconstructed line carries the hard-zero pattern. + """ + h, ctx_log, _ = harness_with_two_workers + # First write: partial line, no newline. + ctx_log.write_text("Broken promi") + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + time.sleep(0.1) # let the scanner poll once and stash the carry + + assert not h.failed_event.is_set(), "scanner fired before the partial line was completed" + + # Second write: complete the line. + with ctx_log.open("a", encoding="utf-8") as f: + f.write("se: future destroyed\n") + + fired = h.failed_event.wait(timeout=2.0) + h.stop_event.set() + t.join(timeout=2.0) + + assert fired is True + assert h.failure_reason is not None + assert "Broken promise" in h.failure_reason + + +def test_chunked_multiline_write_finds_pattern_on_any_line(harness_with_two_workers): + """A single fwrite of many lines must scan all of them.""" + h, _, gen_log = harness_with_two_workers + lines = ["INFO: line 1\n", "INFO: line 2\n", "Segfault at 0x1\n", "INFO: line 4\n"] + gen_log.write_text("".join(lines)) + + fired = _run_scanner_until_failure_or_timeout(h) + + assert fired is True + assert "Segfault" in (h.failure_reason or "") + + +# --------------------------------------------------------------------------- +# Tests: lifecycle / cancellation +# --------------------------------------------------------------------------- + + +def test_stop_event_exits_cleanly(harness_with_two_workers): + h, ctx_log, gen_log = harness_with_two_workers + ctx_log.write_text("INFO: nothing interesting\n") + gen_log.write_text("INFO: also nothing\n") + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + time.sleep(0.1) # let the scanner reach its poll loop + h.stop_event.set() + t.join(timeout=2.0) + + assert not t.is_alive(), "log_scanner did not exit after stop_event" + assert h.failure_reason is None + + +def test_failed_event_set_externally_exits_thread(harness_with_two_workers): + """Externally-set ``failed_event`` must make the scanner wind down. + + Models the case where some other thread (canary / injector) trips + fail-fast before the log scanner gets a chance to fire on its + own. The scanner must observe the event and exit rather than + continuing to poll. + """ + h, _, _ = harness_with_two_workers + h.mark_failed("set externally by some other thread") + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + t.join(timeout=2.0) + + assert not t.is_alive(), "log_scanner did not observe failed_event set at start" + + +def test_log_file_created_after_scanner_starts(harness_with_two_workers): + """A log file created after scanner startup must be picked up lazily. + + Models the race where the scanner spawns before the worker has + flushed its first bytes (so the file doesn't exist yet at poll + #0). The scanner must skip the missing source silently and retry + on each cycle. + """ + h, ctx_log, gen_log = harness_with_two_workers + # Remove the touch()ed files so they don't exist when the scanner + # spawns. ``_LogSource.poll`` should silently skip them. + ctx_log.unlink() + gen_log.unlink() + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + time.sleep(0.1) # scanner is polling; both files absent + + assert not h.failed_event.is_set() + + # Now create gen_log with a hit; scanner should pick it up. + gen_log.write_text("Segfault at 0x0\n") + + fired = h.failed_event.wait(timeout=2.0) + h.stop_event.set() + t.join(timeout=2.0) + + assert fired is True + assert "Segfault" in (h.failure_reason or "") + + +# --------------------------------------------------------------------------- +# Tests: empty / misconfigured input +# --------------------------------------------------------------------------- + + +def test_no_worker_specs_exits_immediately(tmp_path: Path): + yaml_path = tmp_path / "stress.yaml" + yaml_path.write_text(_DUMMY_YAML) + h = DisaggCancellationStressHarness(yaml_path) + # No specs at all. + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + t.join(timeout=2.0) + + assert not t.is_alive(), "scanner did not exit with no log sources" + assert h.failure_reason is None + + +def test_specs_with_none_log_path_skip_gracefully(tmp_path: Path): + yaml_path = tmp_path / "stress.yaml" + yaml_path.write_text(_DUMMY_YAML) + h = DisaggCancellationStressHarness(yaml_path) + h._worker_specs = [ + _make_spec("ctx", 0, None), + _make_spec("gen", 0, None), + ] + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + t.join(timeout=2.0) + + assert not t.is_alive() + assert h.failure_reason is None + + +def test_no_hard_zero_patterns_exits_immediately(tmp_path: Path): + yaml_path = tmp_path / "stress.yaml" + yaml_path.write_text( + textwrap.dedent( + """\ + hostname: localhost + model: dummy + backend: pytorch + context_servers: {} + generation_servers: {} + stress_config: + duration_min: 1 + kv_cache_manager: v1 + transceiver: cpp + log_scan: + hard_zero_patterns: [] + """ + ) + ) + h = DisaggCancellationStressHarness(yaml_path) + ctx_log = tmp_path / "worker_ctx_18000.log" + ctx_log.write_text("Broken promise: A\n") # would match in normal config + h._worker_specs = [_make_spec("ctx", 0, ctx_log)] + + t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) + t.start() + t.join(timeout=2.0) + + assert not t.is_alive() + # No patterns -> scanner can't fire regardless of what's in the log. + assert h.failure_reason is None + + +def test_invalid_regex_is_skipped_with_warning(tmp_path: Path, caplog): + """A malformed regex must not crash the scanner. + + Bad entries in ``hard_zero_patterns`` should be reported as ERROR + and skipped; the scanner continues with whatever other patterns + compiled successfully. + """ + yaml_path = tmp_path / "stress.yaml" + yaml_path.write_text( + textwrap.dedent( + """\ + hostname: localhost + model: dummy + backend: pytorch + context_servers: {} + generation_servers: {} + stress_config: + duration_min: 1 + kv_cache_manager: v1 + transceiver: cpp + log_scan: + hard_zero_patterns: + - "(unclosed" + - "Broken promise" + """ + ) + ) + h = DisaggCancellationStressHarness(yaml_path, log_scanner_poll_interval_s=0.02) + ctx_log = tmp_path / "worker_ctx_18000.log" + ctx_log.write_text("Broken promise: A\n") + h._worker_specs = [_make_spec("ctx", 0, ctx_log)] + + with caplog.at_level("ERROR"): + fired = _run_scanner_until_failure_or_timeout(h) + + # The bad pattern was rejected but the good one still tripped. + assert fired is True + assert "Broken promise" in (h.failure_reason or "") + assert any("(unclosed" in record.getMessage() for record in caplog.records), ( + "expected an ERROR log naming the malformed regex" + ) + + +# --------------------------------------------------------------------------- +# Tests: _LogSource unit behaviour (without the harness wrapper) +# --------------------------------------------------------------------------- + + +def _consume_marks(seen: list[str]) -> Callable[[str], None]: + """Build a ``mark_failed``-shaped callback that appends to a test list. + + Used by the standalone ``_LogSource`` tests (below the harness + tests) to introspect what the source would have reported, + without needing a full harness instance. + + Args: + seen: Test-owned list that each invocation appends its + ``reason`` argument to. + + Returns: + A single-arg callable matching the + ``Callable[[str], None]`` shape that ``_LogSource.poll`` + expects for its ``mark_failed`` parameter. + """ + + def _mark(reason: str) -> None: + seen.append(reason) + + return _mark + + +def test_log_source_poll_returns_false_when_file_absent(tmp_path: Path): + spec = _make_spec("ctx", 0, tmp_path / "missing.log") + src = _LogSource(spec=spec, path=Path(spec.log_path)) # type: ignore[arg-type] + seen: list[str] = [] + assert src.poll([("X", re.compile("X"))], _consume_marks(seen)) is False + assert seen == [] + + +def test_log_source_poll_returns_false_on_empty_read(tmp_path: Path): + log = tmp_path / "empty.log" + log.touch() + spec = _make_spec("gen", 0, log) + src = _LogSource(spec=spec, path=log) + seen: list[str] = [] + assert src.poll([("X", re.compile("X"))], _consume_marks(seen)) is False + assert seen == [] + src.close() + + +def test_log_source_close_is_idempotent(tmp_path: Path): + log = tmp_path / "x.log" + log.write_text("hello\n") + spec = _make_spec("ctx", 0, log) + src = _LogSource(spec=spec, path=log) + src.poll([("X", re.compile("X"))], lambda r: None) + src.close() + src.close() # second close must not raise From db347a948470581f3bc34364a0834aba0c2b276d Mon Sep 17 00:00:00 2001 From: Jianbo Hu Date: Thu, 28 May 2026 09:34:04 +0800 Subject: [PATCH 067/308] [None][feat] support non-divisible EP in MoE alltoall and slurm benchmark (#13888) Signed-off-by: JacobHu-NV <266902545+JacobHu-NV@users.noreply.github.com> Signed-off-by: JacobHu-NV --- .../moeAlltoAllKernels.cu | 60 ++++++-- cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp | 4 +- .../disaggregated/slurm/benchmark/README.md | 11 ++ .../disaggregated/slurm/benchmark/config.yaml | 7 + .../slurm/benchmark/disaggr_torch.slurm | 8 + .../slurm/benchmark/run_benchmark.sh | 3 +- .../slurm/benchmark/start_worker.sh | 24 ++- .../disaggregated/slurm/benchmark/submit.py | 144 ++++++++++++++---- .../communication/communication_factory.py | 17 +++ .../modules/fused_moe/configurable_moe.py | 7 + .../modules/fused_moe/fused_moe_cute_dsl.py | 4 + .../modules/fused_moe/fused_moe_vanilla.py | 10 ++ .../_torch/modules/fused_moe/interface.py | 106 ++++++++++--- .../_torch/modules/moe/test_moe_comm.py | 122 ++++++++++++--- 14 files changed, 435 insertions(+), 92 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu index 7b8c1ecd1723..91cb5725fede 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu @@ -170,19 +170,44 @@ using tensorrt_llm::common::launchWithPdlWhenEnabled; // Helper Functions for Expert-to-Rank Mapping // ============================================================================ -__device__ int compute_target_rank_id(int expert_id, int num_experts_per_rank) +// Compute which rank owns a given expert using contiguous ceil/floor partitioning. +// Supports non-divisible distribution when num_experts % ep_size != 0: +// base = num_experts / ep_size +// remainder = num_experts % ep_size +// - Ranks [0, remainder) each own (base + 1) experts. +// - Ranks [remainder, ep_size) each own base experts. +// +// Example A (uniform): 32 experts, 4 ranks -> base=8, remainder=0 +// - Rank 0: experts 0-7 +// - Rank 1: experts 8-15 +// - Rank 2: experts 16-23 +// - Rank 3: experts 24-31 +// +// Example B (non-divisible): 384 experts, 5 ranks -> base=76, remainder=4 +// - Rank 0: experts 0-76 (77 experts) +// - Rank 1: experts 77-153 (77 experts) +// - Rank 2: experts 154-230 (77 experts) +// - Rank 3: experts 231-307 (77 experts) +// - Rank 4: experts 308-383 (76 experts) +// +// `base` and `remainder` are precomputed by the caller once outside the per-token TOP_K loop +// so the hot path performs at most one integer divide. +__device__ __forceinline__ int compute_target_rank_id(int expert_id, int base, int remainder) { - // Compute which rank owns a given expert using contiguous partitioning - // Experts are divided evenly across EP ranks: - // - Rank 0 gets experts [0, num_experts_per_rank) - // - Rank 1 gets experts [num_experts_per_rank, 2*num_experts_per_rank) - // - etc. - // Example: 32 experts, 4 ranks -> 8 experts per rank - // - Rank 0: experts 0-7 - // - Rank 1: experts 8-15 - // - Rank 2: experts 16-23 - // - Rank 3: experts 24-31 - return expert_id / num_experts_per_rank; + // Fast path for the uniform (num_experts % ep_size == 0) case: identical to the + // pre-ceil/floor implementation, so existing divisible deployments incur no overhead. + if (remainder == 0) + { + return expert_id / base; + } + int const split = remainder * (base + 1); // boundary expert id + if (expert_id < split) + { + // Falls inside the (base + 1)-sized prefix block. + return expert_id / (base + 1); + } + // Falls inside the base-sized suffix block. + return remainder + (expert_id - split) / base; } // ============================================================================ @@ -392,15 +417,20 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ int* smem_topk_send_indices = smem + TOP_K; uint64_t already_copied = 0; - int num_experts_per_rank = num_experts / ep_size; + // Precompute the ceil/floor partition parameters once per thread, outside the + // per-token TOP_K loop. The fast path (remainder == 0) then collapses to a single + // integer divide per call, matching the pre-PR uniform-partition cost exactly. + int const ep_base = num_experts / ep_size; + int const ep_remainder = num_experts - ep_base * ep_size; // == num_experts % ep_size #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) cudaGridDependencySynchronize(); #endif for (int k = 0; k < TOP_K; k++) { int expert_id = token_selected_experts[local_token_idx * TOP_K + k]; - // Use contiguous partitioning to determine target rank - int target_rank = compute_target_rank_id(expert_id, num_experts_per_rank); + // Use contiguous ceil/floor partitioning to determine target rank. + // Supports the non-divisible case where num_experts % ep_size != 0. + int target_rank = compute_target_rank_id(expert_id, ep_base, ep_remainder); if (already_copied & (1ULL << target_rank)) { diff --git a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp index eb74c27ffc89..7a767976dabd 100644 --- a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp @@ -208,7 +208,9 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( TORCH_CHECK(!inputPayloads.empty(), "inputPayloads must not be empty"); TORCH_CHECK(inputPayloads.size() <= kMaxPayloads, "Too many input payloads"); TORCH_CHECK(numExperts >= epSize, "numExperts must be greater than or equal to epSize"); - TORCH_CHECK(numExperts % epSize == 0, "numExperts must be divisible by epSize for contiguous partitioning"); + // numExperts does not need to be divisible by epSize: the kernel performs + // ceil/floor contiguous partitioning so ranks [0, numExperts % epSize) + // own (numExperts / epSize + 1) experts and the rest own (numExperts / epSize). bool enableEplb = eplbLocalStats.has_value(); int64_t eplbStatsNumExperts = 0; if (enableEplb) diff --git a/examples/disaggregated/slurm/benchmark/README.md b/examples/disaggregated/slurm/benchmark/README.md index 5feb896aee23..d8593c460479 100644 --- a/examples/disaggregated/slurm/benchmark/README.md +++ b/examples/disaggregated/slurm/benchmark/README.md @@ -54,8 +54,19 @@ hardware: gpus_per_node: 4 # GPUs per node in your cluster num_ctx_servers: 1 # Number of context processing servers num_gen_servers: 1 # Number of generation servers + # compact_packing: false # Opt-in; see note below ``` +By default each worker owns whole nodes (round-robin), so cross-worker +traffic never crosses a node boundary. Set `compact_packing: true` to pack +all workers into `ceil(total_gpus / gpus_per_node)` nodes, allowing two +workers to share a physical node when their GPU counts don't align with +node boundaries (e.g. two TP=6 ctx workers fit in 3 four-GPU nodes via +4+2 / 2+4). This is **only recommended on full-mesh NVLink fabrics like +GB200/GB300 NVL72**, where the cross-worker NVLink traffic on the shared +node is free; on PCIe or partitioned-NVLink hosts the shared node will +become a bottleneck and you should leave it off. + ### 4. Environment Configuration ```yaml environment: diff --git a/examples/disaggregated/slurm/benchmark/config.yaml b/examples/disaggregated/slurm/benchmark/config.yaml index 92a275c40d1f..a47def615e71 100644 --- a/examples/disaggregated/slurm/benchmark/config.yaml +++ b/examples/disaggregated/slurm/benchmark/config.yaml @@ -26,6 +26,13 @@ hardware: gpus_per_node: 4 # Modify this with your hardware configuration num_ctx_servers: 1 # Number of context servers num_gen_servers: 1 # Number of generation servers + # compact_packing: false # Opt-in: pack workers into the minimum number + # of nodes and let two workers share a physical node when their GPU counts + # straddle a node boundary (e.g. two TP=6 ctx workers in 3 four-GPU nodes + # via 4+2 / 2+4). Only recommended on full-mesh NVLink fabrics like + # GB200/GB300 NVL72 where cross-worker NVLink traffic on the shared node + # is free; on PCIe / partitioned-NVLink hosts the shared node will + # bottleneck and you should leave this off. # Environment Configuration environment: diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm index 9e0d66144478..275016c255dd 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm @@ -154,6 +154,14 @@ client_cmds_base_file=${full_logdir}/client_cmds_base.sh client_cmds_file=${full_logdir}/client_cmds.sh replace_placeholder "${client_cmds_base_file}" "${all_nodes_str}" "${client_cmds_file}" +# Per-worker hostfile / gpu_map files for srun --distribution=arbitrary. +# submit.py emits *_base.txt with ; rewrite them here. +for base_file in "${full_logdir}"/hostfile_*_base.txt "${full_logdir}"/gpu_map_*_base.txt; do + [ -f "${base_file}" ] || continue + runtime_file="${base_file%_base.txt}.txt" + replace_placeholder "${base_file}" "${all_nodes_str}" "${runtime_file}" +done + # start the servers (skip ctx workers if TRTLLM_DISAGG_BENCHMARK_GEN_ONLY is set). echo "Starting worker commands from ${start_server_cmds_file}..." cat ${start_server_cmds_file} | while read cmd; do diff --git a/examples/disaggregated/slurm/benchmark/run_benchmark.sh b/examples/disaggregated/slurm/benchmark/run_benchmark.sh index 69d6e7f84f80..8fa89815704c 100644 --- a/examples/disaggregated/slurm/benchmark/run_benchmark.sh +++ b/examples/disaggregated/slurm/benchmark/run_benchmark.sh @@ -127,7 +127,8 @@ if [ "${ucx_warmup_requests}" -gt 0 ]; then --host ${hostname} \ --port ${port} \ --ignore-eos \ - --non-streaming + --non-streaming \ + --trust-remote-code echo "UCX warmup done" fi diff --git a/examples/disaggregated/slurm/benchmark/start_worker.sh b/examples/disaggregated/slurm/benchmark/start_worker.sh index c2da6ae5cdb8..78bd5f4c5edc 100644 --- a/examples/disaggregated/slurm/benchmark/start_worker.sh +++ b/examples/disaggregated/slurm/benchmark/start_worker.sh @@ -11,11 +11,25 @@ numa_bind=${5} log_dir=${6} enable_nsys=${7} config_file=${8} -cuda_devices=${9} - -# Set CUDA_VISIBLE_DEVICES from script argument (srun --export cannot -# reliably pass comma-separated values inside shared containers). -export CUDA_VISIBLE_DEVICES=${cuda_devices} +# CUDA_VISIBLE_DEVICES selection: +# - Default packing (no gpu_map file): each node is dedicated to one +# worker, so SLURM_LOCALID maps directly to the physical GPU id. +# - Compact packing (gpu_map file emitted by submit.py): two workers may +# share a node and would both see LOCALID=0, so look up the per-worker +# gpu_map " " by SLURM_PROCID. srun +# --distribution=arbitrary assigns PROCID in hostfile order, so it +# indexes directly into the map. +gpu_map_file="${log_dir}/gpu_map_${role}_${instance_id}.txt" +if [ -f "${gpu_map_file}" ]; then + gpu_id=$(awk -v p="${SLURM_PROCID}" '$1==p {print $3; exit}' "${gpu_map_file}") + if [ -z "${gpu_id}" ]; then + echo "ERROR: no GPU mapping for SLURM_PROCID=${SLURM_PROCID} in ${gpu_map_file}" >&2 + exit 1 + fi + export CUDA_VISIBLE_DEVICES=${gpu_id} +else + export CUDA_VISIBLE_DEVICES=${SLURM_LOCALID} +fi # Clear UCX_TLS for specific clusters unset UCX_TLS diff --git a/examples/disaggregated/slurm/benchmark/submit.py b/examples/disaggregated/slurm/benchmark/submit.py index 2e2ed61e8f65..75c21ffe79cd 100644 --- a/examples/disaggregated/slurm/benchmark/submit.py +++ b/examples/disaggregated/slurm/benchmark/submit.py @@ -50,8 +50,17 @@ def save_worker_config(worker_config, output_path): def calculate_nodes(world_size, num_servers, gpus_per_node): - """Calculate required nodes based on world size and server count.""" - return math.ceil(world_size * num_servers / gpus_per_node) + """Required node count under per-worker whole-node ownership. + + Mirrors assign_server_round_robin: each worker consumes + ceil(world_size / gpus_per_node) whole nodes, with no node sharing + across workers. Pooling all GPUs and rounding once under-counts + nodes whenever world_size is not a multiple of gpus_per_node + (e.g. world_size=1, num_servers=2, gpus_per_node=4: pooled gives 1, + but the allocator needs 2). compact_packing computes nodes + separately because it does share nodes across workers. + """ + return num_servers * math.ceil(world_size / gpus_per_node) def allocate_gpus( @@ -62,28 +71,53 @@ def allocate_gpus( gen_world_size: int, ctx_world_size: int, base_port: int = 8000, + compact_packing: bool = False, ) -> List[Dict[str, Any]]: allocations = {} hostnames = [f"" for i in range(total_nodes)] global_gpu_cursor = 0 - def get_gpu_location(gpus_per_node: int): - node_id = global_gpu_cursor // gpus_per_node - local_gpu_id = global_gpu_cursor % gpus_per_node - return node_id, local_gpu_id - - def assign_server(server_allocation: Dict[str, Any], world_size: int, - gpus_per_node: int): + def assign_server_compact(server_allocation: Dict[str, Any], + world_size: int, gpus_per_node: int): + # Compact packing: advance cursor by one GPU per task. Workers may + # share a physical node when their GPU counts don't align with node + # boundaries (e.g. two TP=6 ctx workers fit in 3 four-GPU nodes via + # 4+2 / 2+4). Each task's (host, local_gpu_id) is recorded in + # insertion order, so the per-worker hostfile/gpu_map drives srun + # --distribution=arbitrary and start_worker.sh's CUDA_VISIBLE_DEVICES. nonlocal global_gpu_cursor for _ in range(world_size): - node_id, gpu_id = get_gpu_location(gpus_per_node) - hostname = hostnames[node_id] + host_idx = global_gpu_cursor // gpus_per_node + gpu_idx = global_gpu_cursor % gpus_per_node + hostname = hostnames[host_idx] if hostname not in server_allocation["nodes"]: server_allocation["nodes"][hostname] = [] - server_allocation["nodes"][hostname].append(gpu_id) + server_allocation["nodes"][hostname].append(gpu_idx) global_gpu_cursor += 1 + def assign_server_round_robin(server_allocation: Dict[str, Any], + world_size: int, gpus_per_node: int): + # Default packing: each worker owns ceil(world_size/gpus_per_node) + # whole nodes, round-robin across nodes so each node gets + # ceil(world_size/num_nodes) GPUs. Matches SLURM block distribution, + # so SLURM_LOCALID directly maps to the physical GPU id in + # start_worker.sh. + nonlocal global_gpu_cursor + num_nodes_for_server = math.ceil(world_size / gpus_per_node) + start_node = global_gpu_cursor // gpus_per_node + for task_idx in range(world_size): + node_within = task_idx % num_nodes_for_server + gpu_within = task_idx // num_nodes_for_server + hostname = hostnames[start_node + node_within] + if hostname not in server_allocation["nodes"]: + server_allocation["nodes"][hostname] = [] + server_allocation["nodes"][hostname].append(gpu_within) + global_gpu_cursor += num_nodes_for_server * gpus_per_node + + assign_server = (assign_server_compact + if compact_packing else assign_server_round_robin) + port = base_port def assign_servers( @@ -410,19 +444,21 @@ def submit_job(config, log_dir, dry_run): ctx_num = hw_config['num_ctx_servers'] gen_num = hw_config['num_gen_servers'] gpus_per_node = hw_config['gpus_per_node'] + # Only recommended on full-mesh NVLink fabrics like GB200/GB300 NVL72 + # where two workers sharing a physical node pay no extra cost. On + # PCIe or partitioned-NVLink hosts the shared node becomes a bottleneck. + compact_packing = bool(hw_config.get('compact_packing', False)) # Calculate nodes based on world sizes ctx_tp_size = worker_config['ctx'].get('tensor_parallel_size', 1) ctx_cp_size = worker_config['ctx'].get('context_parallel_size', 1) ctx_pp_size = worker_config['ctx'].get('pipeline_parallel_size', 1) ctx_world_size = ctx_tp_size * ctx_cp_size * ctx_pp_size - ctx_nodes = calculate_nodes(ctx_world_size, ctx_num, gpus_per_node) gen_tp_size = worker_config['gen'].get('tensor_parallel_size', 1) gen_cp_size = worker_config['gen'].get('context_parallel_size', 1) gen_pp_size = worker_config['gen'].get('pipeline_parallel_size', 1) gen_world_size = gen_tp_size * gen_cp_size * gen_pp_size - gen_nodes = calculate_nodes(gen_world_size, gen_num, gpus_per_node) ctx_dp_size = ctx_tp_size if worker_config['ctx'].get( 'enable_attention_dp', False) else 1 @@ -431,7 +467,17 @@ def submit_job(config, log_dir, dry_run): ucx_warmup_requests = 2 * ctx_num * ctx_dp_size * gen_num * gen_dp_size \ if benchmark_config['mode'] == "e2e" else 0 - total_nodes = ctx_nodes + gen_nodes + if compact_packing: + # Compact packing: pack all workers into the minimum number of nodes, + # letting workers share a node when their GPU counts straddle a node + # boundary. Matches the per-GPU cursor in allocate_gpus. + total_gpus = ctx_world_size * ctx_num + gen_world_size * gen_num + total_nodes = math.ceil(total_gpus / gpus_per_node) + else: + # Default: each worker owns whole nodes, no node sharing. + ctx_nodes = calculate_nodes(ctx_world_size, ctx_num, gpus_per_node) + gen_nodes = calculate_nodes(gen_world_size, gen_num, gpus_per_node) + total_nodes = ctx_nodes + gen_nodes total_tasks = total_nodes * gpus_per_node # Generate log directory path based on configuration @@ -453,8 +499,14 @@ def submit_job(config, log_dir, dry_run): mtp_size = worker_config['gen'].get('speculative_config', {}).get('max_draft_len') + # Get gen moe_expert_parallel_size; only tag when it differs from TP + gen_ep_size = worker_config['gen'].get('moe_expert_parallel_size', + gen_tp_size) + ep_tag = f"_ep{gen_ep_size}" if gen_ep_size != gen_tp_size else "" + # Create base log directory path - if 'log_dir' in env_config and env_config['log_dir']: + # CLI --log-dir takes precedence; fall back to yaml log_dir only when not set + if log_dir is None and 'log_dir' in env_config and env_config['log_dir']: log_dir = env_config['log_dir'] if log_dir is None: log_base = os.path.join(script_dir, "logs") @@ -466,9 +518,9 @@ def submit_job(config, log_dir, dry_run): # Determine directory suffix based on attention_dp if gen_enable_attention_dp: - dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_dep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_dep{gen_tp_size}{ep_tag}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" else: - dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_tep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_tep{gen_tp_size}{ep_tag}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" # Create full log directory path log_dir = os.path.join(log_base, dir_suffix) @@ -507,6 +559,7 @@ def submit_job(config, log_dir, dry_run): num_ctx_servers=ctx_num, gen_world_size=gen_world_size, ctx_world_size=ctx_world_size, + compact_packing=compact_packing, ) with open(os.path.join(log_dir, "allocations.json"), "w") as f: json.dump(allocations, f, indent=2) @@ -547,9 +600,35 @@ def submit_job(config, log_dir, dry_run): for server_id in allocations[server_type].keys(): allocation = allocations[server_type][server_id] - gpu_ids = list(allocation["nodes"].values())[0] - cuda_devices = ','.join(map(str, gpu_ids)) + if compact_packing: + # Emit per-worker hostfile (one host per task in rank order) + # and gpu_map (rank -> local gpu id). Nodes carry placeholder + # names at this point; disaggr_torch.slurm rewrites them at + # runtime. SLURM_HOSTFILE + --distribution=arbitrary lets us + # express non-uniform layouts (e.g. 4+2 / 2+4) so two workers + # can share one physical node. + hostfile_base = os.path.join( + log_dir, f"hostfile_{server_type}_{server_id}_base.txt") + gpu_map_base = os.path.join( + log_dir, f"gpu_map_{server_type}_{server_id}_base.txt") + hostfile_runtime = os.path.join( + log_dir, f"hostfile_{server_type}_{server_id}.txt") + with open(hostfile_base, 'w') as hf, \ + open(gpu_map_base, 'w') as gm: + rank = 0 + for host, gpus in allocation["nodes"].items(): + for gpu in gpus: + hf.write(f"{host}\n") + gm.write(f"{rank} {host} {gpu}\n") + rank += 1 + else: + # Default packing: each node is dedicated to one worker, so + # SLURM_LOCALID directly maps to the physical GPU id in + # start_worker.sh (no hostfile/gpu_map needed). + node_list = list(allocation["nodes"].keys()) + num_nodes = len(node_list) + worker_env = build_worker_environment( worker_config=worker_config, env_config=env_config, @@ -561,12 +640,22 @@ def submit_job(config, log_dir, dry_run): ) export_str = format_export_string(worker_env) - cmd = [ - "srun -l", - f"--nodelist {','.join(allocation['nodes'].keys())}", - f"-N {len(allocation['nodes'])}", - f"--ntasks {server_cfg['world_size']}", - f"--ntasks-per-node {gpus_per_node}", + if compact_packing: + srun_prefix = [ + f"SLURM_HOSTFILE={hostfile_runtime}", + "srun -l", + f"--ntasks {server_cfg['world_size']}", + "--distribution=arbitrary", + ] + else: + srun_prefix = [ + "srun -l", + f"--nodelist {','.join(node_list)}", + f"-N {num_nodes}", + f"--ntasks {server_cfg['world_size']}", + ] + + cmd = srun_prefix + [ f"--export=\"{export_str}\"", f"--container-image {env_config['container_image']}", f"--container-name {container_name}", @@ -581,7 +670,6 @@ def submit_job(config, log_dir, dry_run): log_dir, str(profiling_config['nsys_on']).lower(), server_cfg['config_path'], - cuda_devices, f"&> {log_dir}/3_output_{server_type}_{server_id}.log &", ] start_server_cmds.append(" ".join(cmd)) @@ -631,7 +719,7 @@ def submit_job(config, log_dir, dry_run): client_slurm_prefix = [ f"srun -l --container-name={container_name}", f"--container-mounts={container_mount_str}", - f"--mpi=pmix --overlap -N 1 -n 1", + "--no-container-mount-home --mpi=pmix --overlap -N 1 -n 1", ] # Append benchmark commands if benchmark_config.get('enable_benchmark', True): diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py index e57ae6896425..a4ec2ceefe44 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py @@ -149,6 +149,14 @@ def create_strategy( except Exception as e: logger.info(f"NVLinkOneSided not available: {e}") + # Non-divisible EP: NVLinkTwoSided and DeepEP require num_experts % ep_size == 0. + if num_experts % mapping.moe_ep_size != 0: + logger.info( + f"Non-divisible EP (num_experts={num_experts}, ep_size={mapping.moe_ep_size}): " + "falling back to AllGatherReduceScatter" + ) + return AllGatherReduceScatter(mapping) + try: if use_flashinfer: strategy = NVLinkTwoSidedFlashinfer( @@ -246,6 +254,15 @@ def _create_forced_method( method = method.upper() + # Whitelist check: non-divisible EP only supports NVLinkOneSided and AllGather. + _NONDIVISIBLE_EP_ALLOWED = {"NVLINK_ONE_SIDED", "ALLGATHER"} + if num_experts % mapping.moe_ep_size != 0 and method not in _NONDIVISIBLE_EP_ALLOWED: + raise ValueError( + f"Communication method '{method}' requires num_experts % ep_size == 0, " + f"but got num_experts={num_experts}, ep_size={mapping.moe_ep_size}. " + f"Allowed methods for non-divisible EP: {sorted(_NONDIVISIBLE_EP_ALLOWED)}" + ) + # Create strategy - will raise RuntimeError if platform not supported if method in ["NVLINK_TWO_SIDED"]: if use_flashinfer: diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index 690894af84a6..52c1a62c1249 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -67,6 +67,13 @@ class ConfigurableMoE(MoE): + # ConfigurableMoE is a thin wrapper that dispatches to a concrete backend + # (CuteDslFusedMoE / CutlassFusedMoE / ...). Allow the wrapper itself to + # pass the non-divisible-EP gate so the inner backend's own gate is the + # authoritative check -- if the chosen inner backend doesn't opt in, its + # ``MoE.__init__`` will still raise. + _supports_non_divisible_ep: bool = True + """ Configurable MoE layer using composition pattern with automatic configuration diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py index 793ad5afee77..9f033d5997b0 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py @@ -336,6 +336,10 @@ def runner_tactic_comb_checker( class CuteDslFusedMoE(CutlassFusedMoE): + # CuteDSL dispatch/combine path exercises the ceil/floor partition + # (NVLinkOneSided alltoall with kernel-level remainder handling), so this + # backend is the only opt-in for non-divisible EP today. + _supports_non_divisible_ep: bool = True """CuteDSL flow of fused mixture of experts (MoE) Layer. Args: diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_vanilla.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_vanilla.py index c0b85e299be6..db0afbac1b3f 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_vanilla.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_vanilla.py @@ -92,6 +92,16 @@ def __init__( self.intermediate_size_per_partition = intermediate_size // self.tp_size + # VanillaMoE uses uniform expert partitioning; non-divisible EP would require + # ceil/floor distribution (see MoE._compute_ep_partition in interface.py). + # Reject explicitly rather than silently producing wrong local-expert ranges. + if num_experts % self.ep_size != 0: + raise ValueError( + f"VanillaMoE does not support non-divisible EP: " + f"num_experts ({num_experts}) must be divisible by ep_size ({self.ep_size}). " + f"Use CutlassFusedMoE / TRTLLMGenFusedMoE with NVLINK_ONE_SIDED comm for non-divisible EP." + ) + self.expert_size_per_partition = num_experts // self.ep_size self.expert_start = self.ep_rank * self.expert_size_per_partition self.expert_end = min( diff --git a/tensorrt_llm/_torch/modules/fused_moe/interface.py b/tensorrt_llm/_torch/modules/fused_moe/interface.py index f5e3701a0b17..8af319ad7a3a 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/interface.py +++ b/tensorrt_llm/_torch/modules/fused_moe/interface.py @@ -56,6 +56,33 @@ def _warn_and_return(reason: str) -> Tuple[bool, Optional[str]]: precompute_common_perfect_router_logits) +def _compute_ep_partition(num_experts: int, ep_size: int, + ep_rank: int) -> tuple: + """Compute per-rank expert count and slot boundaries. + + Uses ceil/floor distribution: ranks 0..remainder-1 hold (base+1) experts + and remaining ranks hold base. Covers all experts even when + num_experts % ep_size != 0. + + Returns: + (expert_size, slot_start, slot_end) + """ + # Reject num_experts < ep_size: would yield zero-expert ranks, which no + # MoE backend / comm strategy supports end-to-end. The downstream alltoall + # op has a backstop check (numExperts >= epSize) but the AllGatherReduceScatter + # fallback path bypasses it, so guard upfront here. + if num_experts < ep_size: + raise ValueError( + f"num_experts ({num_experts}) must be >= ep_size ({ep_size}); " + f"configurations producing ranks with zero local experts are not supported." + ) + base = num_experts // ep_size + remainder = num_experts % ep_size + expert_size = base + (1 if ep_rank < remainder else 0) + slot_start = ep_rank * base + min(ep_rank, remainder) + return expert_size, slot_start, slot_start + expert_size + + class MoEWeightLoadingMode(Enum): # Gate and up projection are not fused VANILLA = 0 @@ -199,6 +226,12 @@ class MoE(nn.Module): # override this to ``MoESchedulerKind.FUSED_COMM``. scheduler_kind: MoESchedulerKind = MoESchedulerKind.EXTERNAL_COMM + # Opt-in flag for non-divisible EP (num_experts % ep_size != 0). False by default + # so backends whose dispatch/combine paths still assume uniform partitioning fail + # fast with a clear error. Backends that fully exercise the ceil/floor partition + # (see ``_compute_ep_partition``) should override this to ``True``. + _supports_non_divisible_ep: bool = False + @classmethod @abstractmethod def can_implement( @@ -298,6 +331,20 @@ def __init__( self.ep_size = model_config.mapping.moe_ep_size self.ep_rank = model_config.mapping.moe_ep_rank + # Non-divisible EP gate. Backends opt in via the class attribute + # ``_supports_non_divisible_ep``. Default is to reject so that any backend + # whose dispatch/combine paths still assume uniform partitioning fails fast + # with a clear error instead of silently using a wrong local-expert range. + if (self.num_experts % self.ep_size != 0 + and not type(self)._supports_non_divisible_ep): + raise ValueError( + f"{type(self).__name__} does not support non-divisible EP: " + f"num_experts ({self.num_experts}) must be divisible by " + f"ep_size ({self.ep_size}). Override " + f"`_supports_non_divisible_ep = True` on the subclass after " + f"verifying the kernel/comm path handles ceil/floor partitioning." + ) + self.moe_backend = model_config.moe_backend self.use_dp = model_config.mapping.enable_attention_dp @@ -332,10 +379,13 @@ def __init__( self.layer_load_balancer = None self.repeat_idx = 0 self.repeat_count = 1 - self.expert_size_per_partition = self.num_experts // self.ep_size + _size, _start, _end = _compute_ep_partition(self.num_experts, + self.ep_size, + self.ep_rank) + self.expert_size_per_partition = _size self.num_slots = self.num_experts - self.slot_start = self.ep_rank * self.expert_size_per_partition - self.slot_end = self.slot_start + self.expert_size_per_partition + self.slot_start = _start + self.slot_end = _end self.initial_local_expert_ids = list( range(self.slot_start, self.slot_end)) self.initial_global_assignments = list(range(self.num_experts)) @@ -425,15 +475,16 @@ def _init_load_balancer( moe_load_balancer_config = model_config.moe_load_balancer # Calculate initial expert assignments - init_expert_size_per_partition = ( - moe_load_balancer_config.num_local_slots - if moe_load_balancer_config else self.num_experts // self.ep_size) - - self.initial_global_assignments = [ - (ep_rank * self.num_experts // self.ep_size + local_slot_id) % - self.num_experts for ep_rank in range(self.ep_size) - for local_slot_id in range(init_expert_size_per_partition) - ] + if moe_load_balancer_config: + init_expert_size_per_partition = moe_load_balancer_config.num_local_slots + self.initial_global_assignments = [ + (ep_rank * self.num_experts // self.ep_size + local_slot_id) % + self.num_experts for ep_rank in range(self.ep_size) + for local_slot_id in range(init_expert_size_per_partition) + ] + else: + # Sequential mapping: expert i → slot i; covers all experts regardless of divisibility + self.initial_global_assignments = list(range(self.num_experts)) # Setup load balancer if available if moe_load_balancer: @@ -478,19 +529,28 @@ def _init_load_balancer( logger.info( f"initial_global_assignments (layer {self.layer_idx}) = {self.initial_global_assignments}" ) + + # Slot boundaries for EPLB (uniform: all ranks hold same num_local_slots) + self.slot_start = self.ep_rank * self.expert_size_per_partition + self.slot_end = self.slot_start + self.expert_size_per_partition + self.initial_local_expert_ids = self.initial_global_assignments[ + self.slot_start:self.slot_end] + assert len( + self.initial_local_expert_ids) == self.expert_size_per_partition else: - # Fallback when no load balancer - assert self.num_experts % self.ep_size == 0 - self.expert_size_per_partition = self.num_experts // self.ep_size + # Fallback: ceil/floor distribution across ranks. + # Ranks 0..remainder-1 each hold (base+1) experts; remaining ranks hold base. + _size, _start, _end = _compute_ep_partition(self.num_experts, + self.ep_size, + self.ep_rank) + self.expert_size_per_partition = _size self.num_slots = self.num_experts - - # Calculate slot boundaries - self.slot_start = self.ep_rank * self.expert_size_per_partition - self.slot_end = self.slot_start + self.expert_size_per_partition - self.initial_local_expert_ids = self.initial_global_assignments[ - self.slot_start:self.slot_end] - assert len( - self.initial_local_expert_ids) == self.expert_size_per_partition + self.slot_start = _start + self.slot_end = _end + self.initial_local_expert_ids = self.initial_global_assignments[ + self.slot_start:self.slot_end] + assert len( + self.initial_local_expert_ids) == self.expert_size_per_partition # Setup AllReduce for dynamic routing if needed if self._using_dynamic_load_balancer(): diff --git a/tests/unittest/_torch/modules/moe/test_moe_comm.py b/tests/unittest/_torch/modules/moe/test_moe_comm.py index bc2b081c101f..c59751f42025 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_comm.py +++ b/tests/unittest/_torch/modules/moe/test_moe_comm.py @@ -246,12 +246,40 @@ def decode_source_info( # ============================================================================ +def _compute_ep_partition(num_experts: int, ep_size: int, ep_rank: int) -> Tuple[int, int, int]: + """Compute per-rank expert count and slot boundaries. + + Mirrors the kernel's compute_target_rank_id ceil/floor distribution: + ranks [0, remainder) hold (base + 1) experts, and the remaining ranks + hold base experts. Covers all experts even when num_experts % ep_size + != 0 (non-divisible EP). + + Returns: + (expert_size, slot_start, slot_end) + """ + base = num_experts // ep_size + remainder = num_experts % ep_size + expert_size = base + (1 if ep_rank < remainder else 0) + slot_start = ep_rank * base + min(ep_rank, remainder) + return expert_size, slot_start, slot_start + expert_size + + +def _expert_id_to_rank(expert_id: int, num_experts: int, ep_size: int) -> int: + """Inverse mapping of _compute_ep_partition. Mirrors kernel logic.""" + base = num_experts // ep_size + remainder = num_experts % ep_size + split = remainder * (base + 1) + if expert_id < split: + return expert_id // (base + 1) + return remainder + (expert_id - split) // base + + def simple_moe( hidden_states: torch.Tensor, token_selected_slots: torch.Tensor, token_final_scales: torch.Tensor, - ep_rank: int, - experts_per_rank: int, + slot_start: int, + slot_end: int, ) -> torch.Tensor: """Trivial MoE: weighted sum of hidden_states for local experts only. @@ -265,8 +293,6 @@ def simple_moe( - DeepEPLL: all-ones (combine applies real scales internally) """ output = torch.zeros_like(hidden_states, dtype=torch.float32) - slot_start = ep_rank * experts_per_rank - slot_end = slot_start + experts_per_rank for i in range(hidden_states.shape[0]): for k in range(token_selected_slots.shape[1]): @@ -421,8 +447,18 @@ def check_platform_support(comm_type: str) -> Optional[str]: def check_feasibility(comm_type: str, config: CommTestConfig) -> Optional[str]: """Return skip reason string if config is infeasible, else None.""" - if config.num_experts % config.ep_size != 0: - return f"num_experts={config.num_experts} not divisible by ep_size={config.ep_size}" + if config.num_experts % config.ep_size != 0 and comm_type not in ( + COMM_NVLINK_ONE_SIDED, + COMM_ALLGATHER_RS, + ): + # NVLinkOneSided supports non-divisible EP natively (ceil/floor + # partitioning); AllGatherReduceScatter is the production fallback + # selected by CommunicationFactory for the same case. Other comm + # types still require num_experts divisible by ep_size. + return ( + f"comm_type={comm_type} requires num_experts divisible by ep_size, " + f"got num_experts={config.num_experts}, ep_size={config.ep_size}" + ) if config.use_low_precision_combine and not _supports_low_precision_combine(config): return ( @@ -777,7 +813,11 @@ def _worker_full_pipeline(config: CommTestConfig) -> dict: config.quant_mode, ) - experts_per_rank = config.num_experts // config.ep_size + # Use ceil/floor partitioning so the slot range is correct for both + # uniform and non-divisible EP (num_experts % ep_size != 0). + _, local_slot_start, local_slot_end = _compute_ep_partition( + config.num_experts, config.ep_size, rank + ) # Run a minimal local-expert compute step before combine(). This keeps # the test aligned with the real MoE pipeline, where combine() consumes # per-rank expert outputs rather than raw dispatch payloads. @@ -785,8 +825,8 @@ def _worker_full_pipeline(config: CommTestConfig) -> dict: recv_hs_bf16, dispatch_outputs.recv_slots, dispatch_outputs.recv_scales, - rank, - experts_per_rank, + local_slot_start, + local_slot_end, ) combined = comm.combine( @@ -868,10 +908,9 @@ def _compute_expected_tokens_per_rank( """Compute the set of (src_rank, token_idx) each rank should receive. A token is expected at dest_rank if at least one of its top_k expert - selections falls in dest_rank's expert range. + selections falls in dest_rank's expert range. Uses the same ceil/floor + partitioning as the kernel so this works for non-divisible EP as well. """ - experts_per_rank = config.num_experts // config.ep_size - expected: Dict[int, Set[Tuple[int, int]]] = {r: set() for r in range(config.ep_size)} for result in all_results: src_rank = result["rank"] @@ -881,7 +920,8 @@ def _compute_expected_tokens_per_rank( for k in range(slots.shape[1]): eid = slots[i, k].item() if 0 <= eid < config.num_experts: - expected[eid // experts_per_rank].add((src_rank, i)) + target_rank = _expert_id_to_rank(eid, config.num_experts, config.ep_size) + expected[target_rank].add((src_rank, i)) return expected @@ -1006,7 +1046,6 @@ def verify_dispatch_alltoall( 4. All tokens that should be routed to this rank are present (completeness) """ num_experts = config.num_experts - experts_per_rank = num_experts // config.ep_size # For fp8/w4afp8, data is transported as fp8 viewed as bf16 (half width). # For nvfp4, data is packed uint8 (half width). @@ -1028,8 +1067,8 @@ def verify_dispatch_alltoall( recv_hs = result["recv_hs"] recv_slots = result["recv_slots"] - slot_start = recv_rank * experts_per_rank - slot_end = slot_start + experts_per_rank + # Per-rank slot range — handles non-divisible EP. + _, slot_start, slot_end = _compute_ep_partition(num_experts, config.ep_size, recv_rank) decoded = decode_source_info(recv_hs, decode_dtype, decode_hidden) actually_received: Set[Tuple[int, int]] = set() @@ -1175,7 +1214,6 @@ def _build_combine_reference( """ num_tokens = config.all_num_tokens[target_rank] hidden_size = config.hidden_size - experts_per_rank = config.num_experts // config.ep_size # Use float32 accumulator for all paths (matches kernel behavior) ref = torch.zeros(num_tokens, hidden_size, dtype=torch.float32) @@ -1222,8 +1260,9 @@ def _build_combine_reference( recv_hs_bf16 = proc_result["recv_hs_bf16"] recv_slots = proc_result["recv_slots"] source_info = proc_result["recv_source_info"] - slot_start = proc_rank * experts_per_rank - slot_end = slot_start + experts_per_rank + _, slot_start, slot_end = _compute_ep_partition( + config.num_experts, config.ep_size, proc_rank + ) for i, (src_rank, token_idx) in enumerate(source_info): if src_rank == target_rank and token_idx < num_tokens: @@ -1474,6 +1513,41 @@ def _make_boundary_test_params(): return params +def _make_non_divisible_ep_test_params(): + """Generate non-divisible EP test parameters for NVLinkOneSided. + + Exercises ceil/floor expert partitioning where num_experts % ep_size != 0: + base = num_experts // ep_size + remainder = num_experts % ep_size + Ranks [0, remainder) own (base + 1) experts each + Ranks [remainder, ep_size) own base experts each + + Limited to NVLinkOneSided because other comm backends still require + num_experts divisible by ep_size (enforced by check_feasibility). + """ + params = [] + # (ep_size, num_experts, top_k, all_num_tokens) — picked to cover both + # uneven ratios (5 / 2 = 2 + 1) and a larger remainder (17 / 4). + cases = [ + (2, 5, 2, [16, 16]), # base=2, remainder=1: rank0=3, rank1=2 + (4, 17, 2, [16, 16, 16, 16]), # base=4, remainder=1: rank0=5, others=4 + (4, 22, 4, [8, 8, 8, 8]), # base=5, remainder=2: rank0..1=6, rank2..3=5 + ] + for ep_size, num_experts, top_k, all_num_tokens in cases: + for use_low_precision_combine in [False, True]: + config = CommTestConfig( + comm_type=COMM_NVLINK_ONE_SIDED, + ep_size=ep_size, + num_experts=num_experts, + top_k=top_k, + hidden_size=DEFAULT_HIDDEN_SIZE, + all_num_tokens=all_num_tokens, + use_low_precision_combine=use_low_precision_combine, + ) + params.append(pytest.param(ep_size, config, id=str(config))) + return params + + def _make_postquant_test_params(): """Generate post-quant test parameters using POSTQUANT_COMM_MAP. @@ -1598,3 +1672,13 @@ def test_moe_comm_boundary(self, mpi_pool_executor, config: CommTestConfig): def test_moe_comm_postquant(self, mpi_pool_executor, config: CommTestConfig): """Verify post-quant dispatch -> dequant -> MoE -> combine pipeline.""" _run_full_test(mpi_pool_executor, config) + + @pytest.mark.threadleak(enabled=False) + @pytest.mark.parametrize( + "mpi_pool_executor,config", + _make_non_divisible_ep_test_params(), + indirect=["mpi_pool_executor"], + ) + def test_moe_comm_non_divisible_ep(self, mpi_pool_executor, config: CommTestConfig): + """Verify NVLinkOneSided with non-divisible EP (num_experts % ep_size != 0).""" + _run_full_test(mpi_pool_executor, config) From c90fe19ebce6038848695a037b69abb4c8740b28 Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Thu, 28 May 2026 09:35:32 +0800 Subject: [PATCH 068/308] [https://nvbugs/6094100][fix] add ucx tls env in disagg related tests (#14626) Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- .../disaggregated/test_disaggregated_etcd.py | 20 ++++++++++++ .../test_disaggregated_single_gpu.py | 31 ++++++++++++++----- .../defs/disaggregated/test_workers.py | 21 +++++++++++-- tests/integration/test_lists/waives.txt | 11 ------- 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/tests/integration/defs/disaggregated/test_disaggregated_etcd.py b/tests/integration/defs/disaggregated/test_disaggregated_etcd.py index aa3a2930115f..a3a047a93652 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated_etcd.py +++ b/tests/integration/defs/disaggregated/test_disaggregated_etcd.py @@ -14,15 +14,33 @@ # limitations under the License. import os +import platform import signal import subprocess import time import pytest import requests +from defs.conftest import get_sm_version from tensorrt_llm.logger import logger + +def get_ucx_tls(): + """Get UCX_TLS value based on GPU architecture. + + Pre-Hopper GPUs need cuda_ipc excluded from UCX transports. + On some gb300 cluster, we need to set `cuda_copy,cuda_ipc,sm,self,tcp` + for UCX_TLS. + """ + sm = get_sm_version() + if sm == 103 and "aarch" in platform.machine().lower(): + return "cuda_copy,cuda_ipc,sm,self,tcp" + if sm < 90: + return "^cuda_ipc,ib,gdr_copy" + return "^ib,gdr_copy" + + # Configuration file paths EXAMPLES_DIR = "examples/disaggregated" CLIENTS_DIR = f"{EXAMPLES_DIR}/clients" @@ -69,6 +87,7 @@ def start_context_server(config, server_env = env.copy() if env else os.environ.copy() server_env["CUDA_VISIBLE_DEVICES"] = str(gpu_id) server_env["TRTLLM_USE_UCX_KVCACHE"] = "1" + server_env["UCX_TLS"] = get_ucx_tls() logger.info(f"Starting CONTEXT server on GPU {gpu_id} (port {port})...") process = subprocess.Popen(cmd, @@ -95,6 +114,7 @@ def start_generation_server(config, server_env = env.copy() if env else os.environ.copy() server_env["CUDA_VISIBLE_DEVICES"] = str(gpu_id) server_env["TRTLLM_USE_UCX_KVCACHE"] = "1" + server_env["UCX_TLS"] = get_ucx_tls() logger.info(f"Starting GENERATION server on GPU {gpu_id} (port {port})...") process = subprocess.Popen(cmd, diff --git a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py index b9c61f62ed62..da6336af6722 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py +++ b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py @@ -1,11 +1,12 @@ import asyncio import os import pickle +import platform import sys import cloudpickle import pytest -from defs.conftest import skip_no_hopper +from defs.conftest import get_sm_version, skip_no_hopper from mpi4py import MPI from mpi4py.futures import MPIPoolExecutor @@ -15,6 +16,22 @@ KvCacheConfig, MpiCommSession) from tensorrt_llm.llmapi.llm_args import Eagle3DecodingConfig + +def get_ucx_tls(): + """Get UCX_TLS value based on GPU architecture. + + Pre-Hopper GPUs need cuda_ipc excluded from UCX transports. + On some gb300 cluster, we need to set `cuda_copy,cuda_ipc,sm,self,tcp` + for UCX_TLS. + """ + sm = get_sm_version() + if sm == 103 and "aarch" in platform.machine().lower(): + return "cuda_copy,cuda_ipc,sm,self,tcp" + if sm < 90: + return "^cuda_ipc,ib,gdr_copy" + return "^ib,gdr_copy" + + cloudpickle.register_pickle_by_value(sys.modules[__name__]) MPI.pickle.__init__( cloudpickle.dumps, @@ -267,7 +284,7 @@ def verify_disaggregated(model, generation_overlap, enable_cuda_graph, prompt, with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y" }) as executor: futures = [] @@ -416,7 +433,7 @@ def test_disaggregated_llama_context_capacity(model, enable_cuda_graph, with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y" }) as executor: futures = [] @@ -527,7 +544,7 @@ def test_disaggregated_spec_dec_batch_slot_limit(model, spec_dec_model_path, mpi_info.Set("oversubscribe", "true") with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y", "OMPI_MCA_rmaps_base_oversubscribe": "1" }, @@ -618,7 +635,7 @@ def test_disaggregated_logprobs(model, generation_overlap): with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y" }) as executor: futures = [] @@ -727,7 +744,7 @@ def test_disaggregated_cancel_gen_requests(model): with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y", }) as executor: futures = [] @@ -833,7 +850,7 @@ def test_disaggregated_logits(model, generation_overlap): with MPIPoolExecutor(max_workers=2, env={ - "UCX_TLS": "^ib,gdr_copy", + "UCX_TLS": get_ucx_tls(), "UCX_MM_ERROR_HANDLING": "y" }) as executor: futures = [] diff --git a/tests/integration/defs/disaggregated/test_workers.py b/tests/integration/defs/disaggregated/test_workers.py index ad6df3e1eb08..7e6f0ad1bef7 100644 --- a/tests/integration/defs/disaggregated/test_workers.py +++ b/tests/integration/defs/disaggregated/test_workers.py @@ -3,6 +3,7 @@ import copy import json import os +import platform import tempfile from typing import List @@ -10,7 +11,7 @@ import pytest import yaml from defs.common import get_free_port_in_ci as get_free_port -from defs.conftest import skip_no_hopper +from defs.conftest import get_sm_version, skip_no_hopper from disagg_test_utils import (HEARTBEAT_INTERVAL, INACTIVE_TIMEOUT, run_ctx_worker, run_disagg_server, run_gen_worker, terminate, @@ -26,6 +27,21 @@ block_key_hasher) +def get_ucx_tls(): + """Get UCX_TLS value based on GPU architecture. + + Pre-Hopper GPUs need cuda_ipc excluded from UCX transports. + On some gb300 cluster, we need to set `cuda_copy,cuda_ipc,sm,self,tcp` + for UCX_TLS. + """ + sm = get_sm_version() + if sm == 103 and "aarch" in platform.machine().lower(): + return "cuda_copy,cuda_ipc,sm,self,tcp" + if sm < 90: + return "^cuda_ipc,ib,gdr_copy" + return "^ib,gdr_copy" + + def build_worker_config(base_config, server_type_config, disagg_cluster): """Build worker configuration by merging base config with server-type specific config. @@ -525,7 +541,8 @@ def load_default_prompts(disaggregated_example_root: str): def background_workers(llm_venv, config_file: str): cwd = llm_venv.get_working_directory() os.chdir(cwd) - env = llm_venv._new_env + env = llm_venv._new_env.copy() + env["UCX_TLS"] = get_ucx_tls() with open(config_file, 'r') as f: config = yaml.safe_load(f) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f53f0443a92a..600ca036a69a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -158,9 +158,7 @@ cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-mpi_kvcache cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] SKIP (https://nvbugs/6093820) cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] SKIP (https://nvbugs/6093820) cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (https://nvbugs/5838199) -disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) -disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114140) disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_cache_aware_balance[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) @@ -179,19 +177,10 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) -disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) -disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) -disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) -disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6094100) disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) -disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6094100) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6114139) From d636ba02c5f0d23179bafc920016943a52c75c4e Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Thu, 28 May 2026 09:50:07 +0800 Subject: [PATCH 069/308] [None][test] Update stress tests (#14454) Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .coderabbit.yaml | 46 +--- .../defs/disaggregated/disagg_test_utils.py | 16 +- ...fig_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml | 50 ++++ ...sagg_config_ctxtp2_gentp2_gptoss_tllm.yaml | 4 +- ...fig_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml | 57 ++++ ...onfig_ctxtp2_gentp2_gptoss_tllm_eagle.yaml | 67 +++++ ...ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml | 4 +- ...p4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml | 68 +++++ .../defs/disaggregated/test_disaggregated.py | 256 +++++++++++++++--- tests/integration/defs/test_e2e.py | 2 - .../test_lists/qa/llm_function_multinode.txt | 1 - .../test_lists/qa/llm_function_stress.txt | 11 +- tests/integration/test_lists/waives.txt | 8 +- 13 files changed, 486 insertions(+), 104 deletions(-) create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 739ec8dd7348..f8f8946577f4 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -37,51 +37,7 @@ reviews: path_instructions: - path: "tests/**" instructions: | - Act as a QA engineer reviewing test changes for TensorRT-LLM. - - QA test list hygiene (integration / release runs): - - If the change adds or materially alters an integration test under - tests/integration/defs/ (or otherwise affects what QA should run on a - schedule), call out whether an entry is needed under - tests/integration/test_lists/qa/. See tests/integration/test_lists/qa/README.md - for which file to use (e.g. llm_function_core.txt for primary single-node - multi-GPU functional cases; llm_function_multinode.txt for multi-node; - llm_perf_*.yml for perf; llm_triton_integration.txt for Triton). - - If the PR only touches unittest/ or narrow unit scope, say explicitly - whether QA list updates are unnecessary or optional. - - Coverage expectations: - - Assess whether new/changed tests cover happy path, important edge cases, - and failure modes relevant to the feature or fix (skips, guards, env - vars like LLM_MODELS_ROOT, GPU count, backend-specific paths). - - Flag tests that assert overly brittle behavior (e.g. exact token match - across speculative vs non-speculative paths) unless the product contract - requires it. - - Note missing negative tests, missing parametrization where multiple - backends or dtypes apply, or tests that cannot fail when behavior regresses. - - Performance test coverage: - - If the PR touches performance-sensitive paths (attention kernels, MoE - routing/dispatch, KV cache management, scheduler, batching logic, - CUDA graph capture, speculative decoding, or quantization kernels), - check whether a perf test entry is present or updated in: - (a) tests/integration/test_lists/test-db/l0_perf.yml or the - appropriate per-GPU l0_*.yml (primary requirement — this is how - tests enter pre-merge and scheduled CI); and - (b) tests/integration/test_lists/qa/llm_perf_*.yml (single-node) or - tests/integration/test_lists/qa/llm_perf_multinode.txt (multi-node) - for QA scheduled runs. - Flag if (a) is missing even when (b) is present, since test-db is - the authoritative source for CI execution. - - Flag if a test in tests/integration/defs/perf/test_perf_sanity.py is - missing for a new model or serving configuration that warrants a - latency/throughput baseline (examples/benchmark/ is deprecated). - - Note if only functional correctness is tested for a change where a - performance regression would not be caught (e.g. a kernel rewrite with - no perf test, or a new scheduling policy with no throughput assertion). - - Do not require perf tests for doc-only, infra, style, or pure refactor - changes where no runtime behavior changes. - + Act as a QA engineer reviewing test changes and coverage for TensorRT-LLM. Keep feedback actionable: suggest concrete list file names and whether coverage is sufficient, insufficient, or needs follow-up outside the PR. knowledge_base: diff --git a/tests/integration/defs/disaggregated/disagg_test_utils.py b/tests/integration/defs/disaggregated/disagg_test_utils.py index 510035977922..9393d8fa2ef7 100644 --- a/tests/integration/defs/disaggregated/disagg_test_utils.py +++ b/tests/integration/defs/disaggregated/disagg_test_utils.py @@ -145,20 +145,28 @@ def _run_worker( ) -def run_ctx_worker(model_name, ctx_worker_config, work_dir, port=0, device=0, env=None): +def run_ctx_worker( + model_name, ctx_worker_config, work_dir, port=0, device=0, env=None, save_log=False +): """Launch a context worker with service discovery. Use port=0 to let the worker choose a free port. """ - return _run_worker(model_name, ctx_worker_config, "ctx", port, work_dir, device, env=env) + return _run_worker( + model_name, ctx_worker_config, "ctx", port, work_dir, device, save_log=save_log, env=env + ) -def run_gen_worker(model_name, gen_worker_config, work_dir, port=0, device=1, env=None): +def run_gen_worker( + model_name, gen_worker_config, work_dir, port=0, device=1, env=None, save_log=False +): """Launch a generation worker with service discovery. Use port=0 to let the worker choose a free port. """ - return _run_worker(model_name, gen_worker_config, "gen", port, work_dir, device, env=env) + return _run_worker( + model_name, gen_worker_config, "gen", port, work_dir, device, save_log=save_log, env=env + ) def run_disagg_server(disagg_cluster_config, work_dir, port=0, save_log=False, env=None, cwd=None): diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml new file mode 100644 index 000000000000..aaf757903d9e --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml @@ -0,0 +1,50 @@ +model: Qwen3.5-4B-FP8 +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + enable_attention_dp: false + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + enable_chunked_prefill: true + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + enable_attention_dp: false + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml index dc90d3bf6d36..7bfe604313cd 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml @@ -6,7 +6,7 @@ context_servers: tensor_parallel_size: 2 pipeline_parallel_size: 1 moe_expert_parallel_size: 2 - enable_attention_dp: false + enable_attention_dp: true max_num_tokens: 16640 max_seq_len: 8232 max_batch_size: 128 @@ -29,7 +29,7 @@ generation_servers: tensor_parallel_size: 2 pipeline_parallel_size: 1 moe_expert_parallel_size: 2 - enable_attention_dp: false + enable_attention_dp: true max_num_tokens: 10240 max_seq_len: 10240 max_batch_size: 128 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml new file mode 100644 index 000000000000..47b619faf141 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml @@ -0,0 +1,57 @@ +model: gpt_oss/gpt-oss-120b +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 2 + enable_attention_dp: true + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + disable_overlap_scheduler: true + moe_config: + backend: CUTLASS + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 2 + enable_attention_dp: true + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + disable_overlap_scheduler: true + moe_config: + backend: CUTLASS + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml new file mode 100644 index 000000000000..d5fa9d45a931 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml @@ -0,0 +1,67 @@ +model: gpt_oss/gpt-oss-120b +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: &eagle_spec + decoding_type: Eagle + max_draft_len: 3 + eagle3_one_model: true + speculative_model: gpt_oss/gpt-oss-120b-Eagle3 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: CUTLASS + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: *eagle_spec + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: CUTLASS + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml index 58053cc0013c..189c85a55ec0 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml @@ -6,7 +6,7 @@ context_servers: tensor_parallel_size: 4 pipeline_parallel_size: 1 moe_expert_parallel_size: 4 - enable_attention_dp: false + enable_attention_dp: true max_num_tokens: 16640 max_seq_len: 8232 max_batch_size: 128 @@ -28,7 +28,7 @@ generation_servers: tensor_parallel_size: 4 pipeline_parallel_size: 1 moe_expert_parallel_size: 4 - enable_attention_dp: false + enable_attention_dp: true max_num_tokens: 10240 max_seq_len: 10240 max_batch_size: 128 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml new file mode 100644 index 000000000000..82902fa21f6b --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml @@ -0,0 +1,68 @@ +model: DeepSeek-R1/DeepSeek-R1-0528-FP4-v2 +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: false + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: TRTLLM + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: false + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: TRTLLM + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 81b36db52e76..f320690ad787 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -24,7 +24,7 @@ import time from collections import namedtuple from dataclasses import dataclass -from typing import Any +from typing import Any, Optional import aiohttp import numpy as np @@ -52,6 +52,9 @@ class TestConfig: test_desc: str request_count: int accuracy_threshold: float + speculative_model_path: Optional[str] = None + cancellation_rate: Optional[int] = None + cancellation_delay: Optional[float] = None def __str__(self): return self.test_desc @@ -82,6 +85,41 @@ def cleanup_output_files(): pass +# Fatal patterns whose presence in worker/server logs after a stress run +# indicates the cluster did not stay healthy and the test should be failed. +_FATAL_LOG_PATTERNS = ( + "Hang detected on rank", + "RuntimeError: Cluster is not ready", + "Internal server error", +) + + +def scan_logs_for_fatal_errors(processes): + """Scan saved process logs for fatal disagg/worker error patterns. + + Returns a dict mapping log path -> {pattern: count} for any pattern that + appears at least once. Skips processes that did not save a log file. + """ + findings: dict[str, dict[str, int]] = {} + for proc in processes: + log_path = getattr(proc, "log_path", None) + if not log_path or not os.path.exists(log_path): + continue + counts = {pat: 0 for pat in _FATAL_LOG_PATTERNS} + try: + with open(log_path, "r", errors="replace") as f: + for line in f: + for pat in _FATAL_LOG_PATTERNS: + if pat in line: + counts[pat] += 1 + except OSError: + continue + hits = {pat: c for pat, c in counts.items() if c > 0} + if hits: + findings[log_path] = hits + return findings + + def get_default_disagg_cluster_config(): """Get default disaggregated cluster configuration.""" return { @@ -258,8 +296,16 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp2_gentp1cp2_deepseek_v3_lite_bf16_tllm_gen.yaml", "deepseek_r1_v2_fp4_stress": f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml", + "deepseek_r1_v2_fp4_mtp_stress": + f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml", "gpt_oss_120b_stress": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", + "gpt_oss_120b_eagle_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml", + "gpt_oss_120b_cutlass_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml", + "qwen3_5_4b_fp8_stress": + f"{test_configs_root}/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml", "gpt_oss_120b_harmony": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", "cancel_stress_test": @@ -552,6 +598,7 @@ def setup_disagg_cluster( cwd: str | None = None, server_start_timeout: int = 300, schedule_style: str | None = None, + save_log: bool = False, ) -> tuple[dict[str, Any], list[ProcessWrapper], list[ProcessWrapper], ProcessWrapper, int, str]: """Load config, launch workers + disagg server, wait for ready. @@ -618,7 +665,8 @@ def setup_disagg_cluster( work_dir, port=0, device=device_ids, - env=env)) + env=env, + save_log=save_log)) next_device += gpus_per_ctx for i in range(num_gen_instances): @@ -631,7 +679,8 @@ def setup_disagg_cluster( work_dir, port=0, device=device_ids, - env=env)) + env=env, + save_log=save_log)) next_device += gpus_per_gen # Build minimal server config and launch @@ -658,6 +707,7 @@ def setup_disagg_cluster( disagg_server = run_disagg_server(server_config, work_dir, server_port, + save_log=save_log, env=env, cwd=cwd) @@ -1825,6 +1875,8 @@ def run_disaggregated_aiperf(config_file, server_start_timeout=1200, input_tokens=128, output_tokens=100, + input_tokens_stddev=0, + output_tokens_stddev=0, concurrency=1, endpoint_type='chat', request_count=None, @@ -1833,6 +1885,8 @@ def run_disaggregated_aiperf(config_file, random_seed=100, accuracy_test=False, threshold=0.8, + cancellation_rate=None, + cancellation_delay=None, env=None, cwd=None): """Run disaggregated test with genai-perf for performance/stress testing. @@ -1861,7 +1915,8 @@ def run_disaggregated_aiperf(config_file, config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ setup_disagg_cluster(config_file, model_name=model_path, env=run_env, cwd=cwd, - server_start_timeout=server_start_timeout) + server_start_timeout=server_start_timeout, + save_log=True) server_host = config.get("hostname", "localhost") artifact_dir = os.path.join(cwd or ".", "benchmark-results") @@ -1890,16 +1945,34 @@ def run_disaggregated_aiperf(config_file, # Add common parameters aiperf_cmd.extend([ - '--url', f'{server_host}:{server_port}', + '--url', + f'{server_host}:{server_port}', '--synthetic-input-tokens-mean', - str(input_tokens), '--synthetic-input-tokens-stddev', '0', + str(input_tokens), + '--synthetic-input-tokens-stddev', + str(input_tokens_stddev), '--output-tokens-mean', - str(output_tokens), '--output-tokens-stddev', '0', '--extra-inputs', - f'max_tokens:{output_tokens}', '--extra-inputs', - f'min_tokens:{output_tokens}', '--extra-inputs', 'ignore_eos:true', + str(output_tokens), + '--output-tokens-stddev', + str(output_tokens_stddev), + '--extra-inputs', + 'ignore_eos:true', + ]) + # When output length is fixed (stddev == 0) pin max/min tokens so the + # server returns exactly output_tokens. When non-zero, let aiperf + # sample the per-request max_tokens from the mean/stddev distribution. + if output_tokens_stddev == 0: + aiperf_cmd.extend([ + '--extra-inputs', + f'max_tokens:{output_tokens}', + '--extra-inputs', + f'min_tokens:{output_tokens}', + ]) + aiperf_cmd.extend([ '--concurrency', - str(concurrency), '--warmup-request-count', - str(warmup_request_count) + str(concurrency), + '--warmup-request-count', + str(warmup_request_count), ]) # Use request-count or num-dataset-entries @@ -1909,6 +1982,15 @@ def run_disaggregated_aiperf(config_file, # Default: use num-dataset-entries for compatibility aiperf_cmd.extend(['--num-dataset-entries', '64']) + if cancellation_rate is not None: + aiperf_cmd.extend( + ['--request-cancellation-rate', + str(cancellation_rate)]) + if cancellation_delay is not None: + aiperf_cmd.extend( + ['--request-cancellation-delay', + str(cancellation_delay)]) + aiperf_cmd.extend( ['--random-seed', str(random_seed), '--artifact-dir', artifact_dir]) @@ -1919,6 +2001,20 @@ def run_disaggregated_aiperf(config_file, env=env, poll_procs=all_worker_procs + [disagg_server.process]) + # Catch cases where aiperf finished but the disagg cluster was unhealthy + # during the run (e.g. context-side hangs, KV transfer timeouts) which + # would otherwise be swallowed because aiperf records 500s as completed. + fatal_findings = scan_logs_for_fatal_errors( + [*ctx_workers, *gen_workers, disagg_server]) + if fatal_findings: + summary = "\n".join(f" {path}: " + + ", ".join(f"{pat}={cnt}" + for pat, cnt in hits.items()) + for path, hits in fatal_findings.items()) + raise AssertionError( + "Fatal error patterns detected in disaggregated worker/server " + f"logs:\n{summary}") + if accuracy_test: accuracy_test_result, accuracy_value = run_accuracy_test( model_path=model_path, @@ -1929,33 +2025,46 @@ def run_disaggregated_aiperf(config_file, max_gen_toks=256, max_length=4096) - # only raise error if accuracy test passed and accuracy value is less than threshold - if accuracy_test_result and (accuracy_value < threshold): + if not accuracy_test_result: + raise AssertionError( + "Accuracy test failed to complete (likely worker hang or " + "crash); inspect saved logs under work_dir " + f"({work_dir}): worker_ctx_*.log, worker_gen_*.log, " + "disagg_server.log") + if accuracy_value < threshold: raise AssertionError( f"Accuracy test failed: accuracy value {accuracy_value} is less than test threshold {threshold}" ) - except Exception: - # Print outputs on error - logger.error("-------- Workers output (last 30 lines) --------") - try: - with open('output_workers.log', 'r') as f: - lines = f.read().split('\n') - for line in lines[-30:]: - if line.strip(): - logger.error(line) - except FileNotFoundError: - pass + # Re-scan logs after the accuracy run to catch issues (e.g. + # cluster-not-ready, internal server errors) that only surfaced + # while lm_eval was driving the server. + fatal_findings = scan_logs_for_fatal_errors( + [*ctx_workers, *gen_workers, disagg_server]) + if fatal_findings: + summary = "\n".join(f" {path}: " + + ", ".join(f"{pat}={cnt}" + for pat, cnt in hits.items()) + for path, hits in fatal_findings.items()) + raise AssertionError( + "Fatal error patterns detected in disaggregated " + f"worker/server logs after accuracy run:\n{summary}") - logger.error("-------- Disagg server output (last 30 lines) --------") - try: - with open('output_disagg.log', 'r') as f: - lines = f.read().split('\n') - for line in lines[-30:]: - if line.strip(): - logger.error(line) - except FileNotFoundError: - pass + except Exception: + # Print tail of each captured worker/server log to aid triage. + for proc in [*ctx_workers, *gen_workers, disagg_server]: + log_path = getattr(proc, "log_path", None) + if not log_path or not os.path.exists(log_path): + continue + logger.error(f"-------- {log_path} (last 30 lines) --------") + try: + from collections import deque + with open(log_path, "r", errors="replace") as f: + for line in deque(f, maxlen=30): + if line.strip(): + logger.error(line.rstrip()) + except OSError: + pass raise finally: terminate(*ctx_workers, *gen_workers, disagg_server) @@ -2033,6 +2142,19 @@ def run_accuracy_test(model_path: str, server_url: str, concurrency: int, print_info(f"Accuracy test result is: {result}") + # lm_eval's async request retry path crashes on certain transport-level + # errors with "UnboundLocalError: cannot access local variable 'outputs'". + # When this happens individual requests are silently dropped but the + # process can still exit 0, so the score is computed against a partial + # set. Treat the presence of this trace as an inconclusive run. + combined_output = (result.stdout or "") + (result.stderr or "") + if ("UnboundLocalError" in combined_output + and "outputs" in combined_output): + logger.warning( + "lm_eval reported UnboundLocalError on 'outputs' " + "(request retry crashed); treating accuracy run as failed") + return False, accuracy_value + # Check if process completed successfully if result.returncode == 0: test_end_time = time.time() @@ -2163,13 +2285,50 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, pytest.param(TestConfig(model_path='DeepSeek-R1/DeepSeek-R1-0528-FP4-v2', test_desc='deepseek_r1_v2_fp4_stress', request_count=35000, - accuracy_threshold=0.92), + accuracy_threshold=0.92, + cancellation_rate=10, + cancellation_delay=0.5), + marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell)), + pytest.param(TestConfig(model_path='DeepSeek-R1/DeepSeek-R1-0528-FP4-v2', + test_desc='deepseek_r1_v2_fp4_mtp_stress', + request_count=35000, + accuracy_threshold=0.90, + cancellation_rate=10, + cancellation_delay=0.5), marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell)), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', test_desc='gpt_oss_120b_stress', request_count=60000, - accuracy_threshold=0.42), + accuracy_threshold=0.42, + cancellation_rate=10, + cancellation_delay=0.5), marks=(pytest.mark.skip_less_device(4), skip_pre_blackwell)), + pytest.param( + TestConfig( + model_path='gpt_oss/gpt-oss-120b', + test_desc='gpt_oss_120b_eagle_stress', + request_count=60000, + accuracy_threshold=0.42, + speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', + cancellation_rate=10, + cancellation_delay=0.5, + ), + marks=(pytest.mark.skip_less_device(8), skip_pre_hopper), + ), + pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', + test_desc='gpt_oss_120b_cutlass_stress', + request_count=60000, + accuracy_threshold=0.42, + cancellation_rate=10, + cancellation_delay=0.5), + marks=(pytest.mark.skip_less_device(4), skip_pre_hopper)), + pytest.param(TestConfig(model_path='Qwen3.5-4B-FP8', + test_desc='qwen3_5_4b_fp8_stress', + request_count=3000, + accuracy_threshold=0.72, + cancellation_rate=10, + cancellation_delay=0.5), + marks=(pytest.mark.skip_less_device(2), skip_no_hopper)), ], ids=lambda x: x.test_desc) @pytest.mark.parametrize("concurrency", [512], ids=lambda x: f"conc{x}") @@ -2190,11 +2349,36 @@ def test_disaggregated_stress_test(disaggregated_test_root, config_file = get_test_config(test_desc, disaggregated_example_root, os.path.dirname(__file__)) + # Resolve speculative_model to an absolute path for worker processes. + if test_config.speculative_model_path is not None: + spec_model_dir = f"{llm_models_root()}/{test_config.speculative_model_path}" + setup_model_symlink(llm_venv, spec_model_dir, + test_config.speculative_model_path) + with open(config_file, 'r') as f: + patched_config = yaml.safe_load(f) + patched_sections = [] + for section in ('context_servers', 'generation_servers'): + spec = patched_config.get(section, {}).get('speculative_config') + if spec is not None and 'speculative_model' in spec: + spec['speculative_model'] = spec_model_dir + patched_sections.append(section) + if not patched_sections: + raise AssertionError( + f"{test_desc} sets speculative_model_path, but no " + "speculative_config.speculative_model field was patched") + patched_path = os.path.join(llm_venv.get_working_directory(), + f"{test_desc}_patched.yaml") + with open(patched_path, 'w') as f: + yaml.safe_dump(patched_config, f) + config_file = patched_path + run_disaggregated_aiperf(config_file=config_file, model_path=model_dir, server_start_timeout=7200, input_tokens=input_tokens, output_tokens=output_tokens, + input_tokens_stddev=0, + output_tokens_stddev=output_tokens // 10, concurrency=concurrency, endpoint_type='completions', request_count=test_config.request_count, @@ -2202,6 +2386,8 @@ def test_disaggregated_stress_test(disaggregated_test_root, streaming=False, accuracy_test=True, threshold=test_config.accuracy_threshold, + cancellation_rate=test_config.cancellation_rate, + cancellation_delay=test_config.cancellation_delay, env=llm_venv._new_env, cwd=llm_venv.get_working_directory()) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index bb3d0ab4bdb2..9c1f5b681926 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -2356,8 +2356,6 @@ def test_ptp_scaffolding(llm_root, llm_venv, model_name, model_path): marks=skip_pre_blackwell), pytest.param('DeepSeek-R1/DeepSeek-R1-0528-FP4', marks=skip_pre_blackwell), pytest.param('Kimi-K2-Thinking-NVFP4', marks=skip_pre_blackwell), - pytest.param('nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1', - marks=skip_pre_hopper), ]) def test_multi_nodes_eval(model_path, tp_size, pp_size, ep_size, eval_task, mmlu_dataset_root): diff --git a/tests/integration/test_lists/qa/llm_function_multinode.txt b/tests/integration/test_lists/qa/llm_function_multinode.txt index f166b10bf38e..898a65e59b89 100644 --- a/tests/integration/test_lists/qa/llm_function_multinode.txt +++ b/tests/integration/test_lists/qa/llm_function_multinode.txt @@ -3,7 +3,6 @@ test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] -test_e2e.py::test_multi_nodes_eval[nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-tp16-mmlu] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp2pp1-gen_tp2pp1] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] diff --git a/tests/integration/test_lists/qa/llm_function_stress.txt b/tests/integration/test_lists/qa/llm_function_stress.txt index baee4172c80d..fa8a710359f8 100644 --- a/tests/integration/test_lists/qa/llm_function_stress.txt +++ b/tests/integration/test_lists/qa/llm_function_stress.txt @@ -1,12 +1,9 @@ -stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp4-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp4-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_mtp_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_5_4b_fp8_stress] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_nvfp4_4gpus accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 600ca036a69a..46312e23c7ca 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -238,6 +238,8 @@ full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-p full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbugs/4979955) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] SKIP (https://nvbugs/6222480) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] SKIP (https://nvbugs/6185173) full:H20/accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True] SKIP (https://nvbugs/6185173) @@ -313,11 +315,6 @@ perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwel perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] SKIP (https://nvbugs/6162857) -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6207678) -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1-0528-FP4_tp4-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6207678) -stress_test/stress_test.py::test_run_stress_test[DeepSeek-R1_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) -stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-GUARANTEED_NO_EVICT-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) -stress_test/stress_test.py::test_run_stress_test[DeepSeek-V3_tp8-stress_time_3600s_timeout_10800s-MAX_UTILIZATION-pytorch-stress-test-with-accuracy] SKIP (https://nvbugs/6143599) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) @@ -327,7 +324,6 @@ test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] S test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] SKIP (https://nvbugs/6114608) test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https://nvbugs/6115560) test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6114608) -test_e2e.py::test_multi_nodes_eval[nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-tp16-mmlu] SKIP (https://nvbugs/6114608) test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] SKIP (https://nvbugs/6190759) From 757f1e73207f485670b69e9843d1a8fa7d44a652 Mon Sep 17 00:00:00 2001 From: xiweny <13230610+VALLIS-NERIA@users.noreply.github.com> Date: Thu, 28 May 2026 10:15:17 +0800 Subject: [PATCH 070/308] [None][chore] Bump version to 1.3.0rc17 (#14657) Signed-off-by: Xiwen Yu <13230610+VALLIS-NERIA@users.noreply.github.com> --- README.md | 2 +- examples/constraints.txt | 2 +- tensorrt_llm/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4c31af50ad47..a2e2f3a606d7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ TensorRT LLM [![python](https://img.shields.io/badge/python-3.10-green)](https://www.python.org/downloads/release/python-31012/) [![cuda](https://img.shields.io/badge/cuda-13.1.1-green)](https://developer.nvidia.com/cuda-downloads) [![torch](https://img.shields.io/badge/torch-2.10.0-green)](https://pytorch.org) -[![version](https://img.shields.io/badge/release-1.3.0rc16-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) +[![version](https://img.shields.io/badge/release-1.3.0rc17-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) [![license](https://img.shields.io/badge/license-Apache%202-blue)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) [Architecture](https://nvidia.github.io/TensorRT-LLM/developer-guide/overview.html)   |   [Performance](https://nvidia.github.io/TensorRT-LLM/developer-guide/perf-overview.html)   |   [Examples](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html)   |   [Documentation](https://nvidia.github.io/TensorRT-LLM/)   |   [Roadmap](https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap) diff --git a/examples/constraints.txt b/examples/constraints.txt index 200ce0891f01..269cf018f98e 100644 --- a/examples/constraints.txt +++ b/examples/constraints.txt @@ -1,3 +1,3 @@ -tensorrt_llm==1.3.0rc16 +tensorrt_llm==1.3.0rc17 evaluate~=0.4.1 rouge_score~=0.1.2 diff --git a/tensorrt_llm/version.py b/tensorrt_llm/version.py index 0ba6bc0e47de..5597d2d3cd86 100644 --- a/tensorrt_llm/version.py +++ b/tensorrt_llm/version.py @@ -12,4 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.3.0rc16" +__version__ = "1.3.0rc17" From fd5fa6169d5e9313d9cc281038d25de769037462 Mon Sep 17 00:00:00 2001 From: Emma Qiao Date: Thu, 28 May 2026 10:44:40 +0800 Subject: [PATCH 071/308] [None][infra] Fix hang when generating report (#14625) Signed-off-by: EmmaQiaoCh --- jenkins/L0_Test.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ecc967d59aca..43ef9179a80a 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3282,7 +3282,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Generate comprehensive rerun report if any reruns occurred stage ("Generate Report") { - generateRerunReport(stageName, llmSrc) + timeout(time: 15, unit: 'MINUTES'){ + generateRerunReport(stageName, llmSrc) + } } if (rerunFailed) { From c0a64cf466e6b2a03f401b9157f16fa0c03965c1 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Thu, 28 May 2026 03:14:24 +0000 Subject: [PATCH 072/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/auto_deploy/poetry.lock | 42 +++--- .../llm-eval/lm-eval-harness/poetry.lock | 42 +++--- .../models/contrib/hyperclovax/poetry.lock | 42 +++--- .../examples/models/contrib/stdit/poetry.lock | 42 +++--- .../examples/models/core/mixtral/poetry.lock | 42 +++--- .../examples/models/core/mllama/poetry.lock | 42 +++--- .../examples/models/core/qwenvl/poetry.lock | 42 +++--- .../examples/models/core/whisper/poetry.lock | 42 +++--- .../examples/ray_orchestrator/poetry.lock | 14 +- .../examples/trtllm-eval/poetry.lock | 42 +++--- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 130 ++++++++++++------ security_scanning/pyproject.toml | 3 +- security_scanning/triton_backend/poetry.lock | 101 ++++++++++---- 14 files changed, 364 insertions(+), 266 deletions(-) diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 779e49242e2f..2f60d0996032 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -544,38 +544,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index e81117d89a49..f09c1a263393 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -465,38 +465,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index ab0009f30043..393281306e37 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -108,38 +108,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/contrib/stdit/poetry.lock b/security_scanning/examples/models/contrib/stdit/poetry.lock index 74c5acb363b1..de50ba70e20e 100644 --- a/security_scanning/examples/models/contrib/stdit/poetry.lock +++ b/security_scanning/examples/models/contrib/stdit/poetry.lock @@ -581,38 +581,38 @@ ssh = ["bcrypt (>=3.1.5)"] [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock index d4dbf489f2be..dbe8a56f13ba 100644 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ b/security_scanning/examples/models/core/mixtral/poetry.lock @@ -197,38 +197,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/core/mllama/poetry.lock b/security_scanning/examples/models/core/mllama/poetry.lock index 61e2b681c6b9..791e5ef97b6e 100644 --- a/security_scanning/examples/models/core/mllama/poetry.lock +++ b/security_scanning/examples/models/core/mllama/poetry.lock @@ -190,38 +190,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 58282ffa4bf1..9fb5b37651e4 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -592,38 +592,38 @@ test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist" [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/models/core/whisper/poetry.lock b/security_scanning/examples/models/core/whisper/poetry.lock index 60c2996cbdd4..5ffb3a92555d 100644 --- a/security_scanning/examples/models/core/whisper/poetry.lock +++ b/security_scanning/examples/models/core/whisper/poetry.lock @@ -523,38 +523,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 2fba6d7b18ff..0bb993d8e616 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -1714,14 +1714,14 @@ typing-extensions = ">=4.14.1" [[package]] name = "python-discovery" -version = "1.3.1" +version = "1.4.0" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "python_discovery-1.3.1-py3-none-any.whl", hash = "sha256:ed188687ebb3b82c01a17cd5ac62fc94d9f6487a7f1a0f9dfe89753fec91039c"}, - {file = "python_discovery-1.3.1.tar.gz", hash = "sha256:62f6db28064c9613e7ca76cb3f00c38c839a07c31c00dfe7ed0986493d2150a6"}, + {file = "python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da"}, + {file = "python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3"}, ] [package.dependencies] @@ -2137,21 +2137,21 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.3.3" +version = "21.4.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "virtualenv-21.3.3-py3-none-any.whl", hash = "sha256:7d5987d8369e098e41406efb780a3d4ca79280097293899e351a6407ee153ab3"}, - {file = "virtualenv-21.3.3.tar.gz", hash = "sha256:f5bda277e553b1c2b3c1a8debfc30496e1288cc93ce6b7b71b3280047e317328"}, + {file = "virtualenv-21.4.0-py3-none-any.whl", hash = "sha256:334fb2e774003e53a889c4808c5392114020813faf8a6c291d0d97dd3cb26b7a"}, + {file = "virtualenv-21.4.0.tar.gz", hash = "sha256:8d401ab6ee040a7c679f2d3947faa9ed07edf284c0a4e7bda9edd2f34b1fe500"}, ] [package.dependencies] distlib = ">=0.3.7,<1" filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} platformdirs = ">=3.9.1,<5" -python-discovery = ">=1.3.1" +python-discovery = ">=1.4" typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} [[package]] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index 2be449206b0b..ecf22f55a4b4 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -465,38 +465,38 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-pathfinder" diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index a818f2514e2c..8b7acebaa41e 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "5dd96d6c5f0030b09edecfc6f2f60b6243d1d224", - "timestamp": "2026-05-27T02:46:59Z" + "commit_hash": "fd5fa6169d5e9313d9cc281038d25de769037462", + "timestamp": "2026-05-28T02:47:45Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index ede1188680e5..e8dc15033ea2 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -568,14 +568,14 @@ serving = ["fastapi (>=0.104.0)", "peft", "pydantic (>=2.0.0)", "uvicorn (>=0.24 [[package]] name = "cache-dit" -version = "1.3.8" +version = "1.3.9" description = "Cache-DiT: A PyTorch-native Inference Engine with Cache, Parallelism and Quantization for Diffusion Transformers." optional = false python-versions = ">=3.12" groups = ["main"] markers = "python_version == \"3.12\"" files = [ - {file = "cache_dit-1.3.8-py3-none-any.whl", hash = "sha256:7c7b2804929c770bbcd4c12ca25697066a51b1b72aec841351c084109cb9fa6a"}, + {file = "cache_dit-1.3.9-py3-none-any.whl", hash = "sha256:fed1fd41efa629465b441cdc86f64b57cd0be03b80e69e4221529e450d3c6b0c"}, ] [package.dependencies] @@ -1142,37 +1142,37 @@ ssh = ["bcrypt (>=3.1.5)"] [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] [[package]] name = "cuda-core" @@ -1224,21 +1224,22 @@ files = [ [[package]] name = "cuda-python" -version = "13.2.0" +version = "13.3.0" description = "CUDA Python: Performance meets Productivity" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_python-13.2.0-py3-none-any.whl", hash = "sha256:2f092b0ec13a860115fa595411889ee939ad203450ea4f91e9461b174ea7b084"}, + {file = "cuda_python-13.3.0-py3-none-any.whl", hash = "sha256:d21872920204eda83344f8e55644a57823c57c5553d23f9e334add7b8aac0cd0"}, ] [package.dependencies] -cuda-bindings = ">=13.2.0,<13.3.0" +cuda-bindings = ">=13.3.0,<13.4.0" +cuda-core = ">=1.0.0,<1.1.0" cuda-pathfinder = ">=1.1,<2.0" [package.extras] -all = ["cuda-bindings[all] (>=13.2.0,<13.3.0)"] +all = ["cuda-bindings[all] (>=13.3.0,<13.4.0)"] [[package]] name = "cuda-tile" @@ -1563,15 +1564,42 @@ files = [ {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, ] +[[package]] +name = "flash-attn-4" +version = "4.0.0b11" +description = "Flash Attention CUTE (CUDA Template Engine) implementation" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "flash_attn_4-4.0.0b11-py3-none-any.whl", hash = "sha256:f028871f46a63d466d05762876506a12957147374b17f66a52376beed4238dc5"}, + {file = "flash_attn_4-4.0.0b11.tar.gz", hash = "sha256:b2ceede17eea2dfe9c62e8ddef454566702b119421bd098f61faa9aab4cd5885"}, +] + +[package.dependencies] +apache-tvm-ffi = ">=0.1.5,<0.2" +einops = "*" +nvidia-cutlass-dsl = ">=4.4.2" +quack-kernels = ">=0.4.0" +torch = "*" +torch-c-dlpack-ext = "*" +typing_extensions = "*" + +[package.extras] +cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.4.2)"] +dev = ["pytest", "pytest-xdist", "ruff"] + [[package]] name = "flashinfer-python" -version = "0.6.12rc1" +version = "0.6.12rc2" description = "FlashInfer: Kernel Library for LLM Serving" optional = false python-versions = "<4.0,>=3.10" groups = ["main"] -files = [] -develop = false +files = [ + {file = "flashinfer_python-0.6.12rc2-py3-none-any.whl", hash = "sha256:90ae4794c90df6a5850671e5e1ed2a7a28c8a37b7d0b92643fe4dc946994eae5"}, + {file = "flashinfer_python-0.6.12rc2.tar.gz", hash = "sha256:f7abd452d85dfe1e50d4dfb4c1455abd042b83fc64b58849a99bf88d41705cb9"}, +] [package.dependencies] apache-tvm-ffi = ">=0.1.6,<0.1.8 || >0.1.8,<0.1.8.post0 || >0.1.8.post0,<0.2" @@ -1594,12 +1622,6 @@ cu12 = ["nvidia-cutlass-dsl (>=4.5.0)"] cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.5.0)"] nvep = ["cuda-python (>=13.0)"] -[package.source] -type = "git" -url = "https://github.com/flashinfer-ai/flashinfer.git" -reference = "v0.6.12rc1" -resolved_reference = "dd5d33bc54df939c59d6d9d963a61dd0de6c47c1" - [[package]] name = "fonttools" version = "4.63.0" @@ -4564,13 +4586,13 @@ kaleido = ["kaleido (>=1.1.0)"] [[package]] name = "polygraphy" -version = "0.49.26" +version = "0.50.3" description = "Polygraphy: A Deep Learning Inference Prototyping and Debugging Toolkit" optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "polygraphy-0.49.26-py2.py3-none-any.whl", hash = "sha256:5787d218a133163b42c92800134afaba6b266127646efb77416a9530137d1a45"}, + {file = "polygraphy-0.50.3-py3-none-any.whl", hash = "sha256:0df1a2336db7e6e95b5e54094de307699593f092597ad9472d7ceceff4a8a46e"}, ] [[package]] @@ -5492,6 +5514,30 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "quack-kernels" +version = "0.4.1" +description = "" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "quack_kernels-0.4.1-py3-none-any.whl", hash = "sha256:c1c8df2935bf5156ec47d2c5384ac08b411fd0ee702d80ae916dbf6d6f5ae813"}, + {file = "quack_kernels-0.4.1.tar.gz", hash = "sha256:9d7d6ba412bc0c8a9b1331c52a73db76280adb9dc2f2750df4851ddabef1466b"}, +] + +[package.dependencies] +apache-tvm-ffi = ">=0.1.6,<0.2" +einops = "*" +nvidia-cutlass-dsl = ">=4.4.2" +torch = "*" +torch-c-dlpack-ext = "*" + +[package.extras] +cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.4.2)"] +dev = ["pre-commit", "pytest", "pytest-xdist", "ruff"] +heuristics = ["nvidia-matmul-heuristics"] + [[package]] name = "referencing" version = "0.37.0" @@ -6112,14 +6158,14 @@ files = [ [[package]] name = "smg-grpc-proto" -version = "0.4.7" +version = "0.4.8" description = "SMG gRPC proto definitions for SGLang, vLLM, TRT-LLM, and MLX" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "smg_grpc_proto-0.4.7-py3-none-any.whl", hash = "sha256:74cf326a58f1c9166fbcccc09580b875e654b47e8b0169903d2982a1213a664e"}, - {file = "smg_grpc_proto-0.4.7.tar.gz", hash = "sha256:5a7754f532ccea434c21a5730f91e5a9b3e0af1e9eb26191eec0a6d25dc351d8"}, + {file = "smg_grpc_proto-0.4.8-py3-none-any.whl", hash = "sha256:c353d827fe868ece60745ded5a142b83dd7140183c08317732c860f593b45d82"}, + {file = "smg_grpc_proto-0.4.8.tar.gz", hash = "sha256:04adac11b847b1ae7ffb27a3f52079034e921c1fbda02bae326c0434724fc876"}, ] [package.dependencies] @@ -7312,4 +7358,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "d869c67d8f42a41bbe0f7dc231030cbef327c25feb579b859866bf864492a3ae" +content-hash = "0fa80701f5dcc452033115bc5e53f4752151e971aeaee32a33900a5fae28b9ac" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 97fe085fd20d..50f0f045c028 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "onnx-graphsurgeon (>=0.5.2)", "graphviz (>=0.21,<0.22)", "openai (>=2.38.0,<3.0.0)", - "polygraphy (>=0.49.26,<0.50.0)", + "polygraphy (>=0.50.3,<0.51.0)", "psutil (>=7.2.2,<8.0.0)", "nvidia-ml-py (>=13)", "pulp (>=3.3.2,<4.0.0)", @@ -76,6 +76,7 @@ dependencies = [ "partial-json-parser (>=0.2.1.1.post7,<0.3.0.0)", "mcp (>=1.27.1,<2.0.0)", "torch-c-dlpack-ext (==0.1.3)", + "flash-attn-4 (==4.0.0b11)", "mistral-common (>=1.10.0)", "torchao (>=0.14.1,<0.16.0)", "cuda-core (>=1.0.1,<2.0.0)", diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 691b697532f7..32fdc737e81b 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -228,6 +228,19 @@ files = [ {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, ] +[[package]] +name = "backports-strenum" +version = "1.3.1" +description = "Base class for creating enumerated constants that are also subclasses of str" +optional = false +python-versions = ">=3.8.6,<3.11" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "backports_strenum-1.3.1-py3-none-any.whl", hash = "sha256:cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83"}, + {file = "backports_strenum-1.3.1.tar.gz", hash = "sha256:77c52407342898497714f0596e86188bb7084f89063226f4ba66863482f42414"}, +] + [[package]] name = "brotli" version = "1.2.0" @@ -478,37 +491,74 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.2.0" +version = "13.3.0" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, - {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, - {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, - {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, - {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, - {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, - {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, + {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, + {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, + {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, + {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, + {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, + {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, ] [package.dependencies] -cuda-pathfinder = ">=1.1,<2.0" +cuda-pathfinder = ">=1.4.2" + +[package.extras] +all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] + +[[package]] +name = "cuda-core" +version = "1.0.1" +description = "cuda.core: pythonic CUDA module" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "cuda_core-1.0.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9632db74eceb1cd72a7c95b61a5e4cfb9cc2291de0503e170334d936cab3316"}, + {file = "cuda_core-1.0.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46690b0864417a5f2f9a7d10408e2570cbacae195c890a41286701eefb01ba79"}, + {file = "cuda_core-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1693fae113604cf9c114bfe3da15a15981f9d44ae78ecceb0b23e24c628ad19b"}, + {file = "cuda_core-1.0.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3effd11283bc46fd06348c2fd18a0941ba7718a6f447343858c944c1a93a6dab"}, + {file = "cuda_core-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1934517ff8a9dcd21b3f4a28e15e12643164b7d3ec187a4ee7560e22fd2dfc17"}, + {file = "cuda_core-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:95c91d434a9baca066646cefa577227385104670a02fbe8e3defaadda84becf5"}, + {file = "cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1"}, + {file = "cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2"}, + {file = "cuda_core-1.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a5c1aa3b738a7599ea289498d038fe625d259fd7ab795394541eee58a8e29bc"}, + {file = "cuda_core-1.0.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c427e5025096d96fcd5092fdc85d5d5e4ac3dea007914e90472ed52f27220446"}, + {file = "cuda_core-1.0.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b392178202c652368883dbe3773cee14f3e1ed6b8bf45d1a1bcdd37c73604e06"}, + {file = "cuda_core-1.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:b99e3ca9bf3bd2c7d3028e5dc541b00e432e21373816889cdf2722b675bd9be8"}, + {file = "cuda_core-1.0.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9b0f115a68f2f84c0f6d9b7e863a29517f6dfe5f7b7d07d1d9da8904754e9a2"}, + {file = "cuda_core-1.0.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af2db9e50e81d73e4f0b72ad279d0a9c789372393938fe75c17236b9ed974d7d"}, + {file = "cuda_core-1.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:29783ba03d36960b6612b15ff97123e88cfadfb7b1884f3581839f6bfba4b29f"}, + {file = "cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4410bf1ef15c2ec23dccc302da76893c9354b530dee422e3277b116231bf5fe1"}, + {file = "cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0f1324486bb90be6bde28bd0afbaf78a38827948d37f93f90318a01da8a3f8c"}, + {file = "cuda_core-1.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:180bc808166483b0d6658a7c0d3f1312083b220f301de49476c1bc459cc46cf8"}, +] + +[package.dependencies] +"backports.strenum" = {version = "*", markers = "python_version < \"3.11\""} +cuda-pathfinder = ">=1.4.2" +numpy = "*" [package.extras] -all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] +cu12 = ["cuda-bindings[all] (==12.*)"] +cu13 = ["cuda-bindings[all] (==13.*)"] [[package]] name = "cuda-pathfinder" @@ -523,21 +573,22 @@ files = [ [[package]] name = "cuda-python" -version = "13.2.0" +version = "13.3.0" description = "CUDA Python: Performance meets Productivity" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_python-13.2.0-py3-none-any.whl", hash = "sha256:2f092b0ec13a860115fa595411889ee939ad203450ea4f91e9461b174ea7b084"}, + {file = "cuda_python-13.3.0-py3-none-any.whl", hash = "sha256:d21872920204eda83344f8e55644a57823c57c5553d23f9e334add7b8aac0cd0"}, ] [package.dependencies] -cuda-bindings = ">=13.2.0,<13.3.0" +cuda-bindings = ">=13.3.0,<13.4.0" +cuda-core = ">=1.0.0,<1.1.0" cuda-pathfinder = ">=1.1,<2.0" [package.extras] -all = ["cuda-bindings[all] (>=13.2.0,<13.3.0)"] +all = ["cuda-bindings[all] (>=13.3.0,<13.4.0)"] [[package]] name = "exceptiongroup" From 1b8c739efcc1813efb79623bcb80caa3fdce17df Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Thu, 28 May 2026 11:21:46 +0800 Subject: [PATCH 073/308] [None][infra] Update blossom-ci allowlist: add nv-anants, guqiqi, jonghyunchoe, belgarten-nv (#14662) Signed-off-by: yiqingy Co-authored-by: yiqingy --- .github/workflows/blossom-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 31d8a5b2a1d1..2f2b79f4ad45 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -65,6 +65,7 @@ jobs: "barry-delaney", "bashimao", "BatshevaBlack", + "belgarten-nv", "benzh-2025", "BestJuly", "binghanc", @@ -130,6 +131,7 @@ jobs: "govind-ramnarayan", "greg-kwasniewski1", "guangyunh-nv", + "guqiqi", "h-guo18", "HandongLi-01", "hchings", @@ -141,7 +143,6 @@ jobs: "Hudayday", "HuiGao-NV", "hvagadia", - "jdebache", "hyukn", "indrajit96", "inocsin", @@ -153,6 +154,7 @@ jobs: "JadoTu", "jaedeok-nvidia", "janbernloehr", + "jdebache", "jdemouth-nvidia", "JennyLiu-nv", "jershi425", @@ -168,6 +170,7 @@ jobs: "jinzh-nvidia", "jmydurant", "johncalesp", + "jonghyunchoe", "joyang-nv", "jthomson04", "juney-nvidia", @@ -228,6 +231,7 @@ jobs: "niukuo", "Njuapp", "nv-ananjappa", + "nv-anants", "nv-guomingz", "nv-lschneider", "nv-yilinf", From 59d43693697a8f57ed9266b1d7f90b13e7439bb9 Mon Sep 17 00:00:00 2001 From: Saravana Periyasamy Date: Wed, 27 May 2026 23:26:41 -0500 Subject: [PATCH 074/308] [https://nvbugs/6115560][fix] catch OSError in config_file_lock for NFS compatibility (#11960) Signed-off-by: Chang Liu <9713593+chang-l@users.noreply.github.com> Co-authored-by: Chang Liu <9713593+chang-l@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index f2a15151d781..fa448d8876c2 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -1,4 +1,5 @@ import contextlib +import errno import json import os import tempfile @@ -86,8 +87,14 @@ def config_file_lock(timeout: int = 10): try: with lock: yield - except (PermissionError, filelock.Timeout): - # Fallback to tempdir + except (PermissionError, OSError, filelock.Timeout) as e: + # Fallback to tempdir when primary lock path is unusable (e.g., + # NFS locking failures like ENOLCK/ESTALE, permission issues, + # or lock acquisition timeouts) + if isinstance(e, + OSError) and e.errno not in (errno.EACCES, errno.EPERM, + errno.ENOLCK, errno.ESTALE): + raise tmp_dir = Path(tempfile.gettempdir()) tmp_dir.mkdir(parents=True, exist_ok=True) tmp_lock_path = tmp_dir / "_remote_code.lock" @@ -101,7 +108,11 @@ def config_file_lock(timeout: int = 10): ) # proceed without lock yield - except (PermissionError) as e: + except (PermissionError, OSError) as e: + if isinstance( + e, OSError) and e.errno not in (errno.EACCES, errno.EPERM, + errno.ENOLCK, errno.ESTALE): + raise logger.warning( f"tempdir config lock unavailable due to OS/permission issue: {e}, proceeding without lock" ) From 89c90eb6038c1222a1515923c34dcd7453979af6 Mon Sep 17 00:00:00 2001 From: Bala Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> Date: Wed, 27 May 2026 22:32:54 -0700 Subject: [PATCH 075/308] [None][fix] Fix OSRB source header and provenance issues in AutoDeploy modeling code (#14670) Signed-off-by: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> --- .../_torch/auto_deploy/models/custom/modeling_gpt_oss.py | 3 +-- .../_torch/auto_deploy/models/custom/modeling_minimax_m2.py | 3 +-- .../auto_deploy/models/custom/modeling_nemotron_flash.py | 2 +- .../auto_deploy/singlegpu/models/test_minimax_m2_modeling.py | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gpt_oss.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gpt_oss.py index 5d57369399f8..1c98b9378cec 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gpt_oss.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gpt_oss.py @@ -2,8 +2,7 @@ # Licensed under the Apache License, Version 2.0. # Original source: https://github.com/huggingface/transformers # -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Slimmed-down PyTorch GPT-OSS model for AutoDeploy export (prefill only). diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py index 18e509ebd91d..c14c1a8fedaf 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py @@ -2,8 +2,7 @@ # Licensed under the Apache License, Version 2.0. # Original source: https://github.com/huggingface/transformers # -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Slimmed-down PyTorch MiniMax-M2 model for AutoDeploy export (prefill only). diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_flash.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_flash.py index 68c10400266a..52524940ec20 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_flash.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_nemotron_flash.py @@ -16,7 +16,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Adapted from NVIDIA's Nemotron-Flash-3B-Instruct model (Apache-2.0, NVIDIA CORPORATION): +# Adapted from NVIDIA's Nemotron-Flash-3B-Instruct model (Apache-2.0, NVIDIA CORPORATION & AFFILIATES.): # https://huggingface.co/nvidia/Nemotron-Flash-3B-Instruct/tree/main import copy import math diff --git a/tests/unittest/auto_deploy/singlegpu/models/test_minimax_m2_modeling.py b/tests/unittest/auto_deploy/singlegpu/models/test_minimax_m2_modeling.py index 67059133cfa7..891c07cb5edd 100644 --- a/tests/unittest/auto_deploy/singlegpu/models/test_minimax_m2_modeling.py +++ b/tests/unittest/auto_deploy/singlegpu/models/test_minimax_m2_modeling.py @@ -1,5 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Hierarchical equivalence tests for MiniMax-M2 AutoDeploy custom model. From fc3b69e52408689453b7722aed7213a439608f44 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Thu, 28 May 2026 14:00:40 +0800 Subject: [PATCH 076/308] [https://nvbugs/6185173][fix] Set mamba ssm cache to fp32 for NemotronV2 (#14448) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Co-authored-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- .../model_registry/configs/nemotron-nano-9b-v2.yaml | 3 +++ tests/integration/test_lists/waives.txt | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/auto_deploy/model_registry/configs/nemotron-nano-9b-v2.yaml b/examples/auto_deploy/model_registry/configs/nemotron-nano-9b-v2.yaml index c9066452d78a..ac93d4a3bc37 100644 --- a/examples/auto_deploy/model_registry/configs/nemotron-nano-9b-v2.yaml +++ b/examples/auto_deploy/model_registry/configs/nemotron-nano-9b-v2.yaml @@ -7,6 +7,9 @@ trust_remote_code: true kv_cache_config: enable_block_reuse: false free_gpu_memory_fraction: 0.7 + # H20 shows an MMLU accuracy drop with the default BF16 recurrent SSM cache. + # Keep SSM state in FP32 for stable Triton SSM generation. + mamba_ssm_cache_dtype: float32 # Keep max_batch_size as in the PyTorch test to avoid OOM max_batch_size: 128 diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 46312e23c7ca..a3e97c56cc2f 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -241,9 +241,6 @@ full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbug full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] SKIP (https://nvbugs/6222480) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) -full:H20/accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] SKIP (https://nvbugs/6185173) -full:H20/accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True] SKIP (https://nvbugs/6185173) -full:H20/accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True] SKIP (https://nvbugs/6185173) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5948435) From 83ec591f085083572f4e6409be8a460ff8f2f236 Mon Sep 17 00:00:00 2001 From: bhsueh_NV <11360707+byshiue@users.noreply.github.com> Date: Thu, 28 May 2026 14:05:44 +0800 Subject: [PATCH 077/308] [https://nvbugs/5800725][fix] Restore Mistral Large 3 text-only processor (#14248) Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../checkpoints/mistral/config_loader.py | 49 ++++++++++++- .../_torch/models/modeling_mistral.py | 11 +-- tests/integration/test_lists/waives.txt | 2 +- .../checkpoints/mistral/test_config_loader.py | 69 +++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 tests/unittest/_torch/models/checkpoints/mistral/test_config_loader.py diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py index 1b566a812ae0..cd73f1b8ead0 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json from pathlib import Path from typing import Any @@ -18,6 +33,34 @@ ################### +class _MistralPretrainedConfig(PretrainedConfig): + def __init__( + self, + max_position_embeddings: int = 128_000, + rope_scaling: dict[str, Any] | None = None, + rope_parameters: dict[str, Any] | None = None, + **kwargs, + ): + original_rope_scaling = rope_scaling + self.max_position_embeddings = max_position_embeddings + self.rope_scaling = rope_scaling + self.rope_parameters = rope_parameters or rope_scaling + super().__init__( + max_position_embeddings=max_position_embeddings, + rope_scaling=rope_scaling, + rope_parameters=rope_parameters, + **kwargs, + ) + if self.rope_scaling is None and original_rope_scaling is not None: + self.rope_scaling = original_rope_scaling + if self.rope_parameters is None and self.rope_scaling is not None: + self.rope_parameters = self.rope_scaling + + +def _mistral_pretrained_config_from_dict(config_dict: dict[str, Any]) -> PretrainedConfig: + return _MistralPretrainedConfig.from_dict(config_dict) + + def adapt_config_dict( config_dict: dict[str, Any], defaults: dict[str, Any] = {}, @@ -72,7 +115,7 @@ def adapt_config_dict( for k, v in defaults.items(): config_dict.setdefault(k, v) - config = PretrainedConfig.from_dict(config_dict) + config = _mistral_pretrained_config_from_dict(config_dict) return config @@ -87,7 +130,7 @@ def _remap_mistral_vision_args(config: dict) -> dict: config = { "model_type": "pixtral", "architectures": ["PixtralForConditionalGeneration"], - "text_config": PretrainedConfig.from_dict(config), + "text_config": _mistral_pretrained_config_from_dict(config), "vision_config": PretrainedConfig.from_dict(vision_config), } if quant_config: @@ -175,7 +218,7 @@ def _remap_mistral_audio_args(config: dict) -> dict: config = { "model_type": "whixtral", "architectures": ["VoxtralForConditionalGeneration"], - "text_config": PretrainedConfig.from_dict(config), + "text_config": _mistral_pretrained_config_from_dict(config), "audio_config": WhisperConfig( num_mel_bins=encoder_args["audio_encoding_args"]["num_mel_bins"], window_size=encoder_args["audio_encoding_args"]["window_size"], diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 77dbcf1d0f31..590d0a860f9a 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -377,19 +377,20 @@ def __init__( use_fast=self.use_fast, trust_remote_code=trust_remote_code) self._model_path = model_path + auto_processor = AutoProcessor.from_pretrained( + model_path, + use_fast=self.use_fast, + trust_remote_code=trust_remote_code) if model_type == "mistral_large_3": # For mistral large 3, we add chat template in the model forward, and the # MistralCommonImageProcessor is used to process the input when both text and images are provided. # When the input only contains text, we use the text processor to process the input. self._processor = MistralCommonImageProcessor( tokenizer=self._tokenizer, dtype=self.dtype) - self.text_processor = self._processor + self.text_processor = auto_processor else: # For other mistral models, we use the AutoProcessor to process the input. - self._processor = AutoProcessor.from_pretrained( - model_path, - use_fast=self.use_fast, - trust_remote_code=trust_remote_code) + self._processor = auto_processor self.text_processor = self._processor @property diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index a3e97c56cc2f..d6add4cf315f 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -127,7 +127,7 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6162121) +accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6163033) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype SKIP (https://nvbugs/6076767) diff --git a/tests/unittest/_torch/models/checkpoints/mistral/test_config_loader.py b/tests/unittest/_torch/models/checkpoints/mistral/test_config_loader.py new file mode 100644 index 000000000000..1334f9ce178f --- /dev/null +++ b/tests/unittest/_torch/models/checkpoints/mistral/test_config_loader.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import adapt_config_dict + + +def test_mistral_large3_vision_config_preserves_max_position_embeddings(): + config = { + "dim": 7168, + "hidden_dim": 16384, + "llama_4_scaling": { + "beta": 0.1, + "original_max_position_embeddings": 8192, + }, + "max_position_embeddings": 294912, + "max_seq_len": 262144, + "moe": { + "expert_hidden_dim": 4096, + "first_k_dense_replace": 3, + "num_expert_groups": 1, + "num_expert_groups_per_tok": 1, + "num_experts": 128, + "num_experts_per_tok": 4, + "num_shared_experts": 1, + "route_every_n": 1, + "routed_scale": 1.0, + }, + "n_heads": 128, + "n_kv_heads": 128, + "n_layers": 61, + "norm_eps": 1e-6, + "rope_theta": 10000.0, + "tied_embeddings": False, + "vision_encoder": { + "hidden_size": 1664, + "image_size": 1540, + "intermediate_size": 8192, + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 48, + "patch_size": 14, + "rope_theta": 10000.0, + }, + "vocab_size": 131072, + "yarn": { + "alpha": 1, + "apply_scale": False, + "beta": 32, + "factor": 36, + "original_max_position_embeddings": 8192, + }, + } + + adapted = adapt_config_dict(config) + + assert adapted.text_config.max_position_embeddings == 294912 + assert adapted.text_config.rope_scaling["rope_type"] == "yarn" From 9feaa76ba043ee10ebf4154aa6de7b07f77a34fd Mon Sep 17 00:00:00 2001 From: sunnyqgg <159101675+sunnyqgg@users.noreply.github.com> Date: Thu, 28 May 2026 15:16:07 +0800 Subject: [PATCH 078/308] [None][test] Unwaive fp8 blockscale baseline mtp1 (#14666) Signed-off-by: qgai --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d6add4cf315f..7be6c4df5a2c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -28,7 +28,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[h accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp1] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline] SKIP (https://nvbugs/6185196) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline_mtp1] SKIP (https://nvbugs/5955792) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/5859886) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline] SKIP (https://nvbugs/6185196) From 82f69a0e947e9479a2783f5fb375b31d668c9deb Mon Sep 17 00:00:00 2001 From: Holegots Date: Thu, 28 May 2026 15:44:02 +0800 Subject: [PATCH 079/308] [None][docs] fix incorrect auto sampler behavior description for beam search (#14487) --- docs/source/features/sampling.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/source/features/sampling.md b/docs/source/features/sampling.md index 13b24f533243..fc3cb3ed9324 100644 --- a/docs/source/features/sampling.md +++ b/docs/source/features/sampling.md @@ -36,10 +36,7 @@ llm = LLM(model='nvidia/Llama-3.1-8B-Instruct-FP8', sampler_type="TRTLLMSampler") ``` -By default, the sampling backend is chosen to be `auto`. This will use: - -* TRTLLM Sampler when using Beam Search. -* Torch Sampler otherwise. +By default, the sampling backend is chosen to be `auto`. This will use Torch Sampler for all requests. Here is an example to run a model with basic usage of sampling parameters. This example prepares two identical prompts which will give different results due to the sampling parameters chosen: From fe079575de5a8c580a8e2cebf1551e07c40d16ed Mon Sep 17 00:00:00 2001 From: "Yueh-Ting (eop) Chen" Date: Thu, 28 May 2026 15:51:36 +0800 Subject: [PATCH 080/308] [None][feat] Expose host/GPU per-iter time and clarify iter labeling in /metrics (#14127) Signed-off-by: Yueh-Ting Chen Signed-off-by: Yueh-Ting Chen --- .../_torch/pyexecutor/adp_iter_stats.py | 22 +++ tensorrt_llm/_torch/pyexecutor/py_executor.py | 177 ++++++++++++++---- tensorrt_llm/executor/base_worker.py | 27 +++ .../executor/test_stats_serializer.py | 21 +++ .../pyexecutor/test_iter_stats_populate.py | 49 +++++ 5 files changed, 262 insertions(+), 34 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py index 2d271de01e76..1f22061f4349 100644 --- a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py @@ -43,6 +43,13 @@ class ADPIterStatsRecord: req_stats: Optional[List[RequestStats]] kv_iter_stats: Optional[Dict[int, object]] attention_dp_rank: int + # Per-loop CPU wall and GPU forward time captured on rank 0 alongside + # the IterationStats at queue time. Currently broadcast unchanged to all + # rank rows during fanout (true per-rank timing would require widening + # the rank-state allgather payload). Under steady-state ADP all ranks + # step in lockstep, so this is a reasonable approximation. + host_step_time_ms: Optional[float] = None + prev_device_step_time_ms: Optional[float] = None class ADPIterStatsBuffer: @@ -66,6 +73,11 @@ def __init__(self) -> None: self._rank0_req_stats: Dict[int, Optional[List[RequestStats]]] = {} # Rank 0 only: KV iteration stats captured with pending IterationStats. self._rank0_kv_iter_stats: Dict[int, Optional[Dict[int, object]]] = {} + # Rank 0 only: per-loop CPU and GPU timings captured with the + # IterationStats. Broadcast to all rank rows at fanout (see + # _make_rank_iter_stats / finalize). + self._rank0_host_step_time_ms: Dict[int, Optional[float]] = {} + self._rank0_prev_device_step_time_ms: Dict[int, Optional[float]] = {} self._oldest_iter: Optional[int] = None @staticmethod @@ -91,6 +103,8 @@ def queue( *, kv_iter_stats: Optional[Dict[int, object]] = None, is_rank0: bool, + host_step_time_ms: Optional[float] = None, + prev_device_step_time_ms: Optional[float] = None, ) -> None: """Queue local stats; rank 0 also keeps objects needed for fanout.""" payload = self.make_payload(stats) @@ -109,6 +123,8 @@ def queue( self._rank0_iter_stats[iter_id] = stats self._rank0_req_stats[iter_id] = req_stats self._rank0_kv_iter_stats[iter_id] = kv_iter_stats + self._rank0_host_step_time_ms[iter_id] = host_step_time_ms + self._rank0_prev_device_step_time_ms[iter_id] = prev_device_step_time_ms def next_payload(self) -> Optional[RankIterStatsPayload]: """Return the oldest pending stats payload to piggyback.""" @@ -140,6 +156,8 @@ def _discard(self, iter_id: int, *, recompute_oldest: bool = True) -> None: self._rank0_iter_stats.pop(iter_id, None) self._rank0_req_stats.pop(iter_id, None) self._rank0_kv_iter_stats.pop(iter_id, None) + self._rank0_host_step_time_ms.pop(iter_id, None) + self._rank0_prev_device_step_time_ms.pop(iter_id, None) if recompute_oldest and iter_id == self._oldest_iter: self._recompute_oldest_iter() @@ -268,6 +286,8 @@ def finalize( req_stats = self._rank0_req_stats.get(iter_stats_iter) kv_iter_stats = self._rank0_kv_iter_stats.get(iter_stats_iter) + host_step_time_ms = self._rank0_host_step_time_ms.get(iter_stats_iter) + prev_device_step_time_ms = self._rank0_prev_device_step_time_ms.get(iter_stats_iter) for rank_state in sorted(matching_states, key=lambda s: s.rank): rank = rank_state.rank @@ -277,6 +297,8 @@ def finalize( req_stats=req_stats if rank == 0 else None, kv_iter_stats=kv_iter_stats if rank == 0 else None, attention_dp_rank=rank, + host_step_time_ms=host_step_time_ms, + prev_device_step_time_ms=prev_device_step_time_ms, ) ) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 91484ee96a20..fedb14e0aa79 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -578,6 +578,15 @@ def on_detected(): getattr(self.llm_args, 'kv_cache_config', None), 'iteration_stats_interval', 1) self._adp_iter_stats = ADPIterStatsBuffer() + # Per-loop CPU wall and GPU forward time captured by the profile_step + # closure (see _profiler). Populated whenever enable_iter_perf_stats or + # print_iter_log is on so the /metrics serializer can read them + # without depending on the log line. host_step_time describes the loop + # body that just finished; prev_device_step_time is the GPU forward + # time read via the ping-pong CUDA event pair (lags host by one loop + # under steady state — see ping-pong comment in _profiler). + self._latest_host_step_time_ms: Optional[float] = None + self._latest_prev_device_step_time_ms: Optional[float] = None self.gather_all_responses = False self.kv_cache_transceiver = kv_cache_transceiver @@ -1081,8 +1090,15 @@ def profile_step(): calibrator.stop() enabled = False - if start_time is not None and self.print_log and ( - log_all_ranks or self.dist.rank in log_ranks): + # Capture per-loop timing whenever stats or the iter log are + # enabled. The reading of the OTHER parity's event pair (the + # ping-pong) is what keeps synchronize() from blocking the GPU + # — the events being read have already passed by the time we + # read them. Stashing on self lets the /metrics serializer pick + # up the values without going through the log line. + should_capture_timing = start_time is not None and ( + self.print_log or self.enable_iter_perf_stats) + if should_capture_timing: end_time = time.time() if it % 2 == 0: end_event_1.record() @@ -1097,30 +1113,35 @@ def profile_step(): prev_device_step_time = start_event_1.elapsed_time( end_event_1) - if prev_device_step_time is None: - prev_device_step_time = "N/A" # Handle first iteration - else: - prev_device_step_time = f"{prev_device_step_time}ms" host_step_time = (end_time - start_time) * 1000 # milliseconds - formatted_timestamp = datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S") - kv_util_str = "N/A" - if self.kv_cache_manager is not None: - kv_stats = self.kv_cache_manager.get_kv_cache_stats() - if kv_stats.max_num_blocks > 0: - kv_util_str = f"{1.0 - kv_stats.free_num_blocks / kv_stats.max_num_blocks:.3f}" - logger.info( - f"iter = {self.iter_counter}, " - f"global_rank = {self.global_rank}, " - f"rank = {self.dist.rank}, " - f"num_scheduled_requests = {self.num_scheduled_requests}, " - f"kv_cache_util = {kv_util_str}, " - f"currank_total_requests = {self.num_fetch_requests_cur_rank}/" - f"{self.num_fetch_requests}, " - f"host_step_time = {host_step_time}ms, " - f"prev_device_step_time = {prev_device_step_time}, " - f"timestamp = {formatted_timestamp}, " - f"states = {self.model_engine.iter_states}") + self._latest_host_step_time_ms = host_step_time + self._latest_prev_device_step_time_ms = prev_device_step_time + + if self.print_log and (log_all_ranks + or self.dist.rank in log_ranks): + if prev_device_step_time is None: + prev_device_step_time_str = "N/A" # Handle first iteration + else: + prev_device_step_time_str = f"{prev_device_step_time}ms" + kv_util_str = "N/A" + if self.kv_cache_manager is not None: + kv_stats = self.kv_cache_manager.get_kv_cache_stats() + if kv_stats.max_num_blocks > 0: + kv_util_str = f"{1.0 - kv_stats.free_num_blocks / kv_stats.max_num_blocks:.3f}" + formatted_timestamp = datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + logger.info( + f"iter = {self.iter_counter}, " + f"global_rank = {self.global_rank}, " + f"rank = {self.dist.rank}, " + f"num_scheduled_requests = {self.num_scheduled_requests}, " + f"kv_cache_util = {kv_util_str}, " + f"currank_total_requests = {self.num_fetch_requests_cur_rank}/" + f"{self.num_fetch_requests}, " + f"host_step_time = {host_step_time}ms, " + f"prev_device_step_time = {prev_device_step_time_str}, " + f"timestamp = {formatted_timestamp}, " + f"states = {self.model_engine.iter_states}") it += 1 @@ -1167,6 +1188,12 @@ def profile_step(): def _get_init_iter_stats(self, num_new_active_requests, new_active_requests_queue_latency_ms): stats = IterationStats() + # Stamp iter at construction time so the JSON `iter` field identifies + # the loop that CONSTRUCTED this batch. Under the overlap and PP + # schedulers _process_iter_stats consumes this record in a later + # loop, so stamping at consumption time (via self.iter_counter) + # would mislabel the record by 1+ iterations. + stats.iter = self.iter_counter stats.timestamp = datetime.datetime.now().strftime( "%m-%d-%Y %H:%M:%S.%f") @@ -1263,7 +1290,11 @@ def _update_iter_stats(self, stats, iter_latency_ms, num_completed_requests, stats.cpu_mem_usage = 0 stats.pinned_mem_usage = 0 - stats.iter = self.iter_counter + # NOTE: stats.iter was stamped at construction time in + # _get_init_iter_stats. Do NOT re-stamp here from self.iter_counter + # — under the overlap and PP schedulers this method runs in a later + # loop than the one that built the batch, so the live counter would + # mislabel the record. kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) @@ -1484,7 +1515,9 @@ def _append_iter_stats(self, stats: IterationStats, req_stats: Optional[List[RequestStats]] = None, kv_iter_stats: Optional[Dict[int, object]] = None, - attention_dp_rank: Optional[int] = None): + attention_dp_rank: Optional[int] = None, + host_step_time_ms: Optional[float] = None, + prev_device_step_time_ms: Optional[float] = None): """Append one iteration's finalized stats to the export buffer. The normal Attention-DP path fans out rank-local rows before calling @@ -1496,12 +1529,42 @@ def _append_iter_stats(self, req_stats: Optional per-request stats. kv_iter_stats: Optional KV iteration stats captured with ``stats``. attention_dp_rank: Optional ADP rank for fanned-out rank-local rows. + host_step_time_ms: Per-loop CPU wall captured by profile_step + for the loop that built this batch. Surfaces as + ``hostStepTimeMS`` in the /metrics JSON. Always a clean + single-loop measurement (matches the log line's + ``host_step_time``). + prev_device_step_time_ms: GPU forward time captured by the + ping-pong CUDA event pair in profile_step. Surfaces as + ``prevDeviceStepTimeMS`` in the /metrics JSON. Note the + value lags by one loop relative to ``host_step_time_ms`` + (its sibling on the same record describes a slightly + older batch); see _profiler ping-pong comment. """ # Non-ADP appends immediately, so the latest KV stats belong to this # IterationStats. ADP appends later and passes the saved iter-matched # KV stats explicitly. if not self.enable_attention_dp: kv_iter_stats = self._latest_kv_iter_stats + # Same reasoning for the per-loop timings: non-ADP captures them + # at append time. ADP passes the values that were saved with the + # IterationStats at queue time. + if host_step_time_ms is None: + host_step_time_ms = self._latest_host_step_time_ms + if prev_device_step_time_ms is None: + prev_device_step_time_ms = self._latest_prev_device_step_time_ms + + # Per-record indicator so consumers can interpret iterLatencyMS + # without inspecting server config. iterLatencyMS spans ~2 loops + # under "overlap" and a clean single loop under "non_overlap"; the + # always-clean alternative is hostStepTimeMS. + # PP (pp_size > 1) is structurally overlap-style regardless of + # disable_overlap_scheduler: _executor_loop_pp queues a BatchStatePP + # in iter N and _process_previous_batch_pp consumes it in a later + # loop, so iterLatencyMS spans ~2 loops there too. + is_overlap_like = (not self.disable_overlap_scheduler + or self.dist.pp_size > 1) + scheduler_mode = "overlap" if is_overlap_like else "non_overlap" tp_size = getattr(self.dist, "tp_size", 1) gather_all_ranks = os.environ.get("TLLM_METRICS_ALL_RANKS", "0") == "1" @@ -1541,6 +1604,11 @@ def _append_iter_stats(self, } for window_size, s in kv_iter_stats.items() } + if host_step_time_ms is not None: + local_dict["hostStepTimeMS"] = host_step_time_ms + if prev_device_step_time_ms is not None: + local_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms + local_dict["schedulerMode"] = scheduler_mode local_dict["rank"] = self.dist.tp_rank gathered = self.dist.tp_allgather(local_dict) @@ -1562,11 +1630,21 @@ def _append_iter_stats(self, return # Legacy path: rank-0-only (single-rank or iter stats disabled). + # Tuple layout (length is sniffed by _stats_serializer with + # len() > N guards so older 4-tuples remain readable): + # [0] stats: IterationStats + # [1] req_stats: Optional[List[RequestStats]] + # [2] kv_iter_stats: Optional[Dict[int, KvCacheIterationStats]] + # [3] attention_dp_rank: Optional[int] + # [4] host_step_time_ms: Optional[float] + # [5] prev_device_step_time_ms: Optional[float] + # [6] scheduler_mode: "overlap" | "non_overlap" with self.stats_lock: if len(self.stats) > self.max_stats_len: self.stats.pop(0) self.stats.append( - (stats, req_stats, kv_iter_stats, attention_dp_rank)) + (stats, req_stats, kv_iter_stats, attention_dp_rank, + host_step_time_ms, prev_device_step_time_ms, scheduler_mode)) def _process_iter_stats( self, @@ -1577,10 +1655,31 @@ def _process_iter_stats( ): """All ranks: build local stats; ADP queues them for later fanout.""" iter_end_time = time.time() + # iterLatencyMS semantics differ by scheduler: + # - Overlap / PP: batch_state.iter_start_time was captured at the top + # of the loop that BUILT this batch (one or more loops ago). The + # end timestamp is taken here, during the consuming loop. So + # iter_latency_ms spans roughly two loops and tracks the batch's + # full lifecycle, not the loop body that produced it. Under steady + # state, sum(iterLatencyMS) ≈ 2 × wall_time. + # - Non-overlap: iter_start_time was captured at the top of THIS + # loop and the end is taken at the bottom of the same loop, so + # iterLatencyMS is the clean per-loop CPU wall. + # The always-clean alternative is host_step_time_ms (set on every + # IterationStats record); the per-record schedulerMode field tells + # consumers which interpretation applies. iter_latency_ms = (iter_end_time - batch_state.iter_start_time) * 1e3 if batch_state.iter_stats is None: return + # Snapshot the per-loop CPU and GPU timings captured by the most + # recent profile_step. These belong to the loop that BUILT this + # batch (host wall) and the loop one earlier than that for the GPU + # forward (per the ping-pong design). Save them now so the values + # ride along with the IterationStats through any ADP fanout delay. + host_step_time_ms = self._latest_host_step_time_ms + prev_device_step_time_ms = self._latest_prev_device_step_time_ms + req_stats = self._populate_req_stats( finished_requests, active_requests, batch_state.scheduled_requests) if ( @@ -1592,12 +1691,19 @@ def _process_iter_stats( batch_state.scheduled_requests, micro_batch_id) if self.enable_attention_dp: - self._adp_iter_stats.queue(stats, - req_stats, - kv_iter_stats=self._latest_kv_iter_stats, - is_rank0=self.dist.rank == 0) + self._adp_iter_stats.queue( + stats, + req_stats, + kv_iter_stats=self._latest_kv_iter_stats, + is_rank0=self.dist.rank == 0, + host_step_time_ms=host_step_time_ms, + prev_device_step_time_ms=prev_device_step_time_ms) else: - self._append_iter_stats(stats, req_stats) + self._append_iter_stats( + stats, + req_stats, + host_step_time_ms=host_step_time_ms, + prev_device_step_time_ms=prev_device_step_time_ms) def _executor_loop_cleanup(self): @@ -3259,7 +3365,10 @@ def _fetch_new_requests( record.stats, record.req_stats, kv_iter_stats=record.kv_iter_stats, - attention_dp_rank=record.attention_dp_rank) + attention_dp_rank=record.attention_dp_rank, + host_step_time_ms=record.host_step_time_ms, + prev_device_step_time_ms=record.prev_device_step_time_ms + ) all_ranks_num_active_requests = [ s.num_active_requests for s in all_rank_states ] diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index c8ed5ec671e7..d8beef865f9b 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -824,6 +824,11 @@ def _stats_serializer(stats) -> str: iteration_stats, req_stats = stats[0], stats[1] kv_iter_stats = stats[2] if len(stats) > 2 else None attention_dp_rank = stats[3] if len(stats) > 3 else None + # Newer slots — guarded with len() checks so historical 4-tuples and + # any external code still appending the legacy shape keep working. + host_step_time_ms = stats[4] if len(stats) > 4 else None + prev_device_step_time_ms = stats[5] if len(stats) > 5 else None + scheduler_mode = stats[6] if len(stats) > 6 else None stats_dict = json.loads(iteration_stats.to_json_str()) # Always tag the row so Dynamo's adapter can read @@ -867,6 +872,28 @@ def _stats_serializer(stats) -> str: for window_size, s in kv_iter_stats.items() } + # Per-loop CPU wall captured by profile_step() — always a clean + # single-loop measurement, matching the log line's `host_step_time`. + # Prefer this over iterLatencyMS when you need absolute per-loop + # CPU cost, especially under the overlap scheduler where + # iterLatencyMS measures the batch's full lifecycle (~2 loops). + if host_step_time_ms is not None: + stats_dict["hostStepTimeMS"] = host_step_time_ms + # GPU forward time read via the ping-pong CUDA event pair in + # profile_step(). Note the "prev" in the name: under steady state + # the value lags its sibling host_step_time on the same record by + # one loop (the event-pair being read corresponds to the loop + # before the one host_step_time describes). See the ping-pong + # comment in PyExecutor._profiler for the design rationale. + if prev_device_step_time_ms is not None: + stats_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms + # Scheduler mode for this record. "overlap" means iterLatencyMS + # spans ~2 loops (use hostStepTimeMS for clean per-loop cost); + # "non_overlap" means iterLatencyMS is itself the clean per-loop + # CPU wall. Set per-record so consumers do not need server config. + if scheduler_mode is not None: + stats_dict["schedulerMode"] = scheduler_mode + # Convert back to JSON string return json.dumps(stats_dict) diff --git a/tests/unittest/executor/test_stats_serializer.py b/tests/unittest/executor/test_stats_serializer.py index 23f071ad12ff..fd2918189eb5 100644 --- a/tests/unittest/executor/test_stats_serializer.py +++ b/tests/unittest/executor/test_stats_serializer.py @@ -212,3 +212,24 @@ def test_serializer_none_attention_dp_rank_defaults_zero(self): result = BaseWorker._stats_serializer((iter_stats, None, None, None)) d = json.loads(result) assert d["attentionDpRank"] == 0 + + def test_serializer_7_tuple_emits_new_timing_and_scheduler_mode(self): + """7-tuple shape: hostStepTimeMS / prevDeviceStepTimeMS / schedulerMode. + + ``PyExecutor._append_iter_stats`` now emits a 7-tuple carrying the + per-loop CPU wall (slot 4), the ping-pong GPU forward time (slot 5), + and the per-record schedulerMode tag (slot 6). The serializer must + surface each under its expected JSON key so /metrics consumers can + interpret iterLatencyMS without reading server config. + + ``prevDeviceStepTimeMS`` is set to None to also guard the first-iter + case where the ping-pong event pair has no prior measurement: the + key must be omitted (not serialized as null or 0.0) so consumers can + distinguish "unavailable" from a real zero. + """ + iter_stats = _make_mock_iteration_stats() + result = BaseWorker._stats_serializer((iter_stats, None, None, None, 12.5, None, "overlap")) + d = json.loads(result) + assert d["hostStepTimeMS"] == 12.5 + assert "prevDeviceStepTimeMS" not in d + assert d["schedulerMode"] == "overlap" diff --git a/tests/unittest/pyexecutor/test_iter_stats_populate.py b/tests/unittest/pyexecutor/test_iter_stats_populate.py index d58e96afaa7b..ce065e592d57 100644 --- a/tests/unittest/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/pyexecutor/test_iter_stats_populate.py @@ -834,3 +834,52 @@ def test_to_json_str_roundtrip_includes_new_inflight_batching_stats_fields(): assert ifb_d["numQueuedGenRequests"] == 6 assert ifb_d["numQueuedGenKvTokens"] == 2345 assert ifb_d["numPausedKvTokens"] == 1500 + + +# --------------------------------------------------------------------------- +# Iter labeling: stats.iter is stamped at batch CONSTRUCTION time in +# _get_init_iter_stats, and _update_iter_stats must NOT overwrite it from the +# live self.iter_counter. Under the overlap and PP schedulers _update_iter_stats +# runs in a later loop than the one that built the batch, so re-stamping from +# the live counter mislabels the record by 1+ iterations. +# --------------------------------------------------------------------------- + + +def test_update_iter_stats_does_not_overwrite_construction_iter(): + """Regression guard: ``_update_iter_stats`` must not re-stamp ``stats.iter``. + + Before the fix this method assigned ``stats.iter = self.iter_counter``, + which under the overlap and PP schedulers reflected the loop CONSUMING + the batch rather than the one that BUILT it — mislabeling the record by + 1+ iterations. Construct an ``IterationStats`` with a known iter, pass it + through ``_update_iter_stats`` with a deliberately different live counter, + and confirm the construction-time stamp survives. + """ + from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor + + fake_self = _build_fake_self(queued_items=[], iter_states={"num_ctx_tokens": 0}) + # Live counter is several loops ahead of when the batch was built; any + # accidental re-stamp would surface here. + fake_self.iter_counter = 999 + + stats = IterationStats() + stats.inflight_batching_stats = InflightBatchingStats() + # Construction-time stamp (what _get_init_iter_stats would have written + # at iter 17, several loops ago under overlap scheduling). + stats.iter = 17 + + with patch( + "tensorrt_llm._torch.pyexecutor.py_executor.torch.cuda.mem_get_info", + return_value=(1 << 30, 1 << 30), + ): + PyExecutor._update_iter_stats( + fake_self, + stats, + iter_latency_ms=10.0, + num_completed_requests=0, + scheduled_batch=_StubScheduledBatch(), + micro_batch_id=0, + ) + + # If someone re-introduces ``stats.iter = self.iter_counter`` this becomes 999. + assert stats.iter == 17 From f391155e5a10b424a68afeca290879eb49d9efe7 Mon Sep 17 00:00:00 2001 From: Pengbo Wang <221450789+pengbowang-nv@users.noreply.github.com> Date: Thu, 28 May 2026 16:02:45 +0800 Subject: [PATCH 081/308] [None][chore] Add test lists with multi-gpu test to CI multi-gpu test trigger files (#14087) Signed-off-by: Pengbo Wang <221450789+pengbowang-nv@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 603e7824625e..e9d45f49daa9 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -948,8 +948,39 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tensorrt_llm/serve/openai_server.py", "tensorrt_llm/serve/router.py", "tests/integration/defs/cpp/test_multi_gpu.py", + "tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml", + "tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_dgx_b200.yml", + "tests/integration/test_lists/test-db/l0_dgx_b300.yml", "tests/integration/test_lists/test-db/l0_dgx_h100.yml", "tests/integration/test_lists/test-db/l0_dgx_h200.yml", + "tests/integration/test_lists/test-db/l0_dgx_h200_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node4_gpu16.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb300.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml", + "tests/integration/test_lists/test-db/l0_verl.yml", "tests/unittest/auto_deploy/multigpu", "tests/unittest/_torch/multi_gpu/", "tests/unittest/_torch/multi_gpu_modeling/", From 09bad734e1972452e33fd62936b0444b361bb7b4 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Thu, 28 May 2026 16:13:36 +0800 Subject: [PATCH 082/308] [None][refactor] Add derived properties for the thop.attention call site (#14279) Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- cpp/kernels/xqa/mla_sm120.cu | 10 +- .../decoderMaskedMultiheadAttentionTemplate.h | 15 +- .../kernels/flashMLA/flash_fwd_mla_kernel.h | 4 +- .../unfusedAttentionKernels_2_template.h | 8 +- cpp/tensorrt_llm/nanobind/thop/bindings.cpp | 15 +- cpp/tensorrt_llm/thop/attentionOp.cpp | 14 +- cpp/tensorrt_llm/thop/attentionOp.h | 2 +- cpp/tensorrt_llm/thop/mlaPreprocessOp.cpp | 46 +- .../_torch/attention_backend/interface.py | 38 +- .../_torch/attention_backend/sparse/dsa.py | 3 +- .../_torch/attention_backend/trtllm.py | 588 ++++++++++-------- .../_torch/attention_backend/trtllm_gen.py | 31 +- .../auto_deploy/custom_ops/mla/trtllm_mla.py | 2 - tensorrt_llm/_torch/modules/attention.py | 26 +- .../test_attention_op_sync.py | 548 ++++++++++++++++ 15 files changed, 1004 insertions(+), 346 deletions(-) create mode 100644 tests/unittest/_torch/attention_backend/test_attention_op_sync.py diff --git a/cpp/kernels/xqa/mla_sm120.cu b/cpp/kernels/xqa/mla_sm120.cu index 84ee4f7ccba4..f77e83732fbc 100644 --- a/cpp/kernels/xqa/mla_sm120.cu +++ b/cpp/kernels/xqa/mla_sm120.cu @@ -536,7 +536,12 @@ struct Producer #endif if (threadIdx.x == 0) { - smem.qkScaleLog2e = args.qScale * args.kvCacheScale[0] * log2e; + // ``kvCacheScale`` is the device-memory fp8 dequant scale. Callers + // may pass nullptr when scales are unset (e.g. unit tests, or + // scale=1.0 paths); default to 1.0 in that case to keep the kernel + // crash-safe. + float const kvScale = (args.kvCacheScale != nullptr) ? args.kvCacheScale[0] : 1.0f; + smem.qkScaleLog2e = args.qScale * kvScale * log2e; } if (threadIdx.x < headGrpSize) @@ -1444,7 +1449,8 @@ __device__ inline void Consumer::compute() smem.mathWarpsBar.arrive(); ThrdRegRowMax const accRowSum = loadShmRowMax(smem.accRowSum[tileIdx.x], tileBase.y, lane); - float const xvScale = computeRowSumFromF8 ? args.kvCacheScale[0] : args.kvCacheScale[0] * xScale; + float const kvScale = (args.kvCacheScale != nullptr) ? args.kvCacheScale[0] : 1.0f; + float const xvScale = computeRowSumFromF8 ? kvScale : kvScale * xScale; WarpOutputTile const output = finalize(acc, accRowSum, xvScale, lane); bool const isMultiBlockMode = (nbSubSeq != 1); diff --git a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderMaskedMultiheadAttentionTemplate.h b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderMaskedMultiheadAttentionTemplate.h index 5bb632465dd9..cde3a765ce82 100644 --- a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderMaskedMultiheadAttentionTemplate.h +++ b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderMaskedMultiheadAttentionTemplate.h @@ -1522,13 +1522,20 @@ __global__ void __launch_bounds__(MAX_THEADS_PER_BLOCK, MIN_BLOCKS_PER_SM) maske bool const load_qkv_quant = params.qkv_scale_quant_orig != nullptr; bool const write_attention_quant = params.attention_out_scale_orig_quant != nullptr; - // Quant/Dequant scales for 8bits kv cache. + // Quant/Dequant scales for 8bits kv cache. The ENABLE_8BITS_* guards are + // compile-time template flags; the runtime pointer can still be nullptr + // when the caller omits scales (e.g. unit tests, non-fp8 paths exercising + // the same template). Default to 1.0 in that case to mirror the FlashMLA + // descale_*_ptr nullptr-safety fix. using T_scale = typename kv_cache_scale_type_t::Type; T_scale kv_scale_orig_quant, k_scale_quant_orig; - float const k_scale_quant_orig_f = (ENABLE_8BITS_K_CACHE ? params.kv_scale_quant_orig[0] : 1.0f); - float const kv_scale_quant_orig_f = (ENABLE_8BITS_KV_CACHE ? params.kv_scale_quant_orig[0] : 1.0f); + float const k_scale_quant_orig_f + = (ENABLE_8BITS_K_CACHE && params.kv_scale_quant_orig != nullptr ? params.kv_scale_quant_orig[0] : 1.0f); + float const kv_scale_quant_orig_f + = (ENABLE_8BITS_KV_CACHE && params.kv_scale_quant_orig != nullptr ? params.kv_scale_quant_orig[0] : 1.0f); convert_from_float(&k_scale_quant_orig, k_scale_quant_orig_f); - convert_from_float(&kv_scale_orig_quant, (ENABLE_8BITS_KV_CACHE ? params.kv_scale_orig_quant[0] : 1.0f)); + convert_from_float(&kv_scale_orig_quant, + (ENABLE_8BITS_KV_CACHE && params.kv_scale_orig_quant != nullptr ? params.kv_scale_orig_quant[0] : 1.0f)); #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) cudaGridDependencySynchronize(); diff --git a/cpp/tensorrt_llm/kernels/flashMLA/flash_fwd_mla_kernel.h b/cpp/tensorrt_llm/kernels/flashMLA/flash_fwd_mla_kernel.h index d34104cb845b..23772e585f94 100644 --- a/cpp/tensorrt_llm/kernels/flashMLA/flash_fwd_mla_kernel.h +++ b/cpp/tensorrt_llm/kernels/flashMLA/flash_fwd_mla_kernel.h @@ -686,8 +686,8 @@ __global__ void __launch_bounds__(Kernel_traits::kNThreads, 1, 1) float scale_softmax_log2 = params.scale_softmax_log2; if constexpr (Kernel_traits::Is_FP8) { - float descale_q = __ldg(params.descale_q_ptr); - descale_k = __ldg(params.descale_k_ptr); + float descale_q = params.descale_q_ptr ? __ldg(params.descale_q_ptr) : 1.f; + descale_k = params.descale_k_ptr ? __ldg(params.descale_k_ptr) : 1.f; scale_softmax = scale_softmax * descale_q * descale_k; scale_softmax_log2 = scale_softmax_log2 * descale_q * descale_k; } diff --git a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h index 78fad6b6a0fd..f8993fed6146 100644 --- a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h +++ b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h @@ -1028,10 +1028,14 @@ __global__ void applyBiasRopeUpdateKVCacheV2(QKVPreprocessingParams::Type; TScale scaleOrigQuant; - mmha::convert_from_float(&scaleOrigQuant, params.qkv_scale_orig_quant[0]); + mmha::convert_from_float(&scaleOrigQuant, + params.qkv_scale_orig_quant != nullptr ? params.qkv_scale_orig_quant[0] : 1.0f); // Store 8bits kv cache. mmha::store_8bits_vec(kDst, k, inBlockIdx, scaleOrigQuant); mmha::store_8bits_vec(vDst, v, inBlockIdx, scaleOrigQuant); diff --git a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp index f19a8799ea8f..76c95037e945 100644 --- a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp @@ -140,13 +140,14 @@ void initBindings(nb::module_& m) nb::arg("kv_scale_quant_orig").none(), nb::arg("out_scale").none(), nb::arg("rotary_inv_freq").none(), nb::arg("rotary_cos_sin").none(), nb::arg("latent_cache").none(), nb::arg("q_pe").none(), nb::arg("block_ids_per_seq").none(), nb::arg("attention_sinks").none(), nb::arg("is_fused_qkv"), - nb::arg("update_kv_cache"), nb::arg("predicted_tokens_per_seq"), nb::arg("layer_idx"), nb::arg("num_heads"), - nb::arg("num_kv_heads"), nb::arg("head_size"), nb::arg("tokens_per_block").none(), nb::arg("max_num_requests"), - nb::arg("max_context_length"), nb::arg("attention_window_size"), nb::arg("beam_width"), nb::arg("mask_type"), - nb::arg("quant_mode"), nb::arg("q_scaling"), nb::arg("position_embedding_type"), - nb::arg("rotary_embedding_dim"), nb::arg("rotary_embedding_base"), nb::arg("rotary_embedding_scale_type"), - nb::arg("rotary_embedding_scales"), nb::arg("rotary_embedding_max_position_info"), - nb::arg("use_paged_context_fmha"), nb::arg("attention_input_type").none(), nb::arg("is_mla_enable"), + nb::arg("update_kv_cache"), nb::arg("predicted_tokens_per_seq"), nb::arg("local_layer_idx"), + nb::arg("num_heads"), nb::arg("num_kv_heads"), nb::arg("head_size"), nb::arg("tokens_per_block").none(), + nb::arg("max_num_requests"), nb::arg("max_context_length"), nb::arg("attention_window_size"), + nb::arg("beam_width"), nb::arg("mask_type"), nb::arg("quant_mode"), nb::arg("q_scaling"), + nb::arg("position_embedding_type"), nb::arg("rotary_embedding_dim"), nb::arg("rotary_embedding_base"), + nb::arg("rotary_embedding_scale_type"), nb::arg("rotary_embedding_scales"), + nb::arg("rotary_embedding_max_position_info"), nb::arg("use_paged_context_fmha"), + nb::arg("attention_input_type").none(), nb::arg("is_mla_enable"), nb::arg("chunked_prefill_buffer_batch_size").none(), nb::arg("q_lora_rank").none(), nb::arg("kv_lora_rank").none(), nb::arg("qk_nope_head_dim").none(), nb::arg("qk_rope_head_dim").none(), nb::arg("v_head_dim").none(), nb::arg("rope_append").none(), nb::arg("mrope_rotary_cos_sin").none(), diff --git a/cpp/tensorrt_llm/thop/attentionOp.cpp b/cpp/tensorrt_llm/thop/attentionOp.cpp index 086e999e8b1d..79447cc93172 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.cpp +++ b/cpp/tensorrt_llm/thop/attentionOp.cpp @@ -927,7 +927,7 @@ void attention(torch::Tensor q, std::optional k, std::optional rotary_cos_sin, std::optional latent_cache, std::optional q_pe, std::optional block_ids_per_seq, std::optional attention_sinks, bool const is_fused_qkv, bool const update_kv_cache, - int64_t const predicted_tokens_per_seq, int64_t const layer_idx, int64_t const num_heads, + int64_t const predicted_tokens_per_seq, int64_t const local_layer_idx, int64_t const num_heads, int64_t const num_kv_heads, int64_t const head_size, std::optional const tokens_per_block, int64_t const max_num_requests, int64_t const max_context_length, int64_t const attention_window_size, int64_t const beam_width, int64_t const mask_type, int64_t const quant_mode, double const q_scaling, @@ -956,7 +956,7 @@ void attention(torch::Tensor q, std::optional k, std::optional compressed_kv_cache_pool_ptr) { - TLLM_LOG_TRACE("Attention op starts at layer %d", layer_idx); + TLLM_LOG_TRACE("Attention op starts at layer %d", local_layer_idx); // Use these tensors to infer if the attention is using KV cache bool const use_kv_cache = kv_cache_block_offsets.has_value() && host_kv_cache_pool_pointers.has_value() && host_kv_cache_pool_mapping.has_value(); @@ -1044,7 +1044,7 @@ void attention(torch::Tensor q, std::optional k, std::optional(); op->mType = dtype; op->mFMHAForceFP32Acc = dtype == nvinfer1::DataType::kBF16; - op->mLayerIdx = layer_idx; + op->mLayerIdx = local_layer_idx; op->mNumHeads = num_heads; op->mNumKVHeads = num_kv_heads; op->mHeadSize = head_size; @@ -1162,13 +1162,13 @@ void attention(torch::Tensor q, std::optional k, std::optional, hash> op_cache; if (auto it = op_cache.find(cache_key); it != op_cache.end()) { - TLLM_LOG_TRACE("Attention op for layer %d is cached", layer_idx); + TLLM_LOG_TRACE("Attention op for layer %d is cached", local_layer_idx); op = it->second; } else { - TLLM_LOG_TRACE( - "Preparing new attention op for layer %d with cache key: %s", layer_idx, to_string(cache_key).c_str()); + TLLM_LOG_TRACE("Preparing new attention op for layer %d with cache key: %s", local_layer_idx, + to_string(cache_key).c_str()); op->initialize(); runner->prepare(*op); op_cache[cache_key] = op; @@ -1257,7 +1257,7 @@ void attention(torch::Tensor q, std::optional k, std::optional k, std::optional rotary_cos_sin, std::optional latent_cache, std::optional q_pe, std::optional block_ids_per_seq, std::optional attention_sinks, bool const is_fused_qkv, bool const update_kv_cache, - int64_t const predicted_tokens_per_seq, int64_t const layer_idx, int64_t const num_heads, + int64_t const predicted_tokens_per_seq, int64_t const local_layer_idx, int64_t const num_heads, int64_t const num_kv_heads, int64_t const head_size, std::optional const tokens_per_block, int64_t const max_num_requests, int64_t const max_context_length, int64_t const attention_window_size, int64_t const beam_width, int64_t const mask_type, int64_t const quant_mode, double const q_scaling, diff --git a/cpp/tensorrt_llm/thop/mlaPreprocessOp.cpp b/cpp/tensorrt_llm/thop/mlaPreprocessOp.cpp index f6c5eba6bc68..a52c86a46917 100644 --- a/cpp/tensorrt_llm/thop/mlaPreprocessOp.cpp +++ b/cpp/tensorrt_llm/thop/mlaPreprocessOp.cpp @@ -107,10 +107,9 @@ void mergeChunkedAttentionForMLAHelper(torch::Tensor& merged_attn, torch::Tensor std::vector loadPagedKVCacheForMLA(torch::ScalarType out_dtype, int64_t const num_contexts, int64_t const num_ctx_cached_tokens, int64_t const max_ctx_cached_kv_len, torch::Tensor& cu_ctx_cached_kv_lens, torch::Tensor const& kv_cache_block_offsets, torch::Tensor const& host_kv_cache_pool_pointers, - torch::Tensor const& host_kv_cache_pool_mapping, torch::optional kv_scale_orig_quant, - torch::optional kv_scale_quant_orig, int64_t const layer_idx, int64_t const lora_size, - int64_t const rope_size, int64_t const tokens_per_block, int64_t const attention_window_size, - int64_t const beam_width, int64_t const quant_mode) + torch::Tensor const& host_kv_cache_pool_mapping, torch::optional kv_scale_quant_orig, + int64_t const layer_idx, int64_t const lora_size, int64_t const rope_size, int64_t const tokens_per_block, + int64_t const attention_window_size, int64_t const beam_width, int64_t const quant_mode) { TORCH_CHECK(out_dtype == torch::kFloat16 || out_dtype == torch::kFloat32 || out_dtype == torch::kBFloat16, "out_dtype only support float16, float32, bfloat16"); @@ -129,17 +128,14 @@ std::vector loadPagedKVCacheForMLA(torch::ScalarType out_dtype, i attention_window_size, beam_width, 0 /*seq_offset*/, true /*is_mla_enable*/, torch::elementSize(out_dtype)) .kvCacheBuffer; - float const* kv_scale_orig_quant_ptr = nullptr; float const* kv_scale_quant_orig_ptr = nullptr; if (kv_cache_quant_mode.hasKvCacheQuant()) { TLLM_CHECK_WITH_INFO(kv_cache_quant_mode.hasFp8KvCache(), "Only FP8 KV cache is supported for now"); - TORCH_CHECK(kv_scale_orig_quant.has_value()); - TORCH_CHECK(kv_scale_quant_orig.has_value()); - kv_scale_orig_quant_ptr = kv_scale_orig_quant.value().data_ptr(); + } + if (kv_scale_quant_orig.has_value()) + { kv_scale_quant_orig_ptr = kv_scale_quant_orig.value().data_ptr(); - TLLM_CHECK(kv_scale_orig_quant_ptr != nullptr); - TLLM_CHECK(kv_scale_quant_orig_ptr != nullptr); } std::vector outputs; @@ -199,9 +195,9 @@ std::vector loadChunkedKVCacheForMLA(torch::ScalarType out_dtype, int64_t const num_ctx_cached_tokens, torch::Tensor const& cu_ctx_chunked_kv_lens, torch::Tensor const& chunked_ld_global_offset, torch::Tensor const& kv_cache_block_offsets, torch::Tensor const& host_kv_cache_pool_pointers, torch::Tensor const& host_kv_cache_pool_mapping, - torch::optional kv_scale_orig_quant, torch::optional kv_scale_quant_orig, - int64_t const layer_idx, int64_t const lora_size, int64_t const rope_size, int64_t const tokens_per_block, - int64_t const max_seq_len, int64_t const attention_window_size, int64_t const beam_width, int64_t const quant_mode) + torch::optional kv_scale_quant_orig, int64_t const layer_idx, int64_t const lora_size, + int64_t const rope_size, int64_t const tokens_per_block, int64_t const max_seq_len, + int64_t const attention_window_size, int64_t const beam_width, int64_t const quant_mode) { TORCH_CHECK(out_dtype == torch::kFloat16 || out_dtype == torch::kFloat32 || out_dtype == torch::kBFloat16, "out_dtype only support float16, float32, bfloat16"); @@ -217,16 +213,10 @@ std::vector loadChunkedKVCacheForMLA(torch::ScalarType out_dtype, attention_window_size, beam_width, 0 /*seq_offset*/, true /*is_mla_enable*/, torch::elementSize(out_dtype)) .kvCacheBuffer; - float const* kv_scale_orig_quant_ptr = nullptr; float const* kv_scale_quant_orig_ptr = nullptr; - if (kv_cache_quant_mode.hasKvCacheQuant()) + if (kv_scale_quant_orig.has_value()) { - TORCH_CHECK(kv_scale_orig_quant.has_value()); - TORCH_CHECK(kv_scale_quant_orig.has_value()); - kv_scale_orig_quant_ptr = kv_scale_orig_quant.value().data_ptr(); kv_scale_quant_orig_ptr = kv_scale_quant_orig.value().data_ptr(); - TLLM_CHECK(kv_scale_orig_quant_ptr != nullptr); - TLLM_CHECK(kv_scale_quant_orig_ptr != nullptr); } std::vector outputs; @@ -293,8 +283,8 @@ void MLARopeAppendPagedKVAssignQ(torch::Tensor& q, torch::Tensor& latent_cache, int64_t const nope_size, int64_t const rope_size, int64_t const lora_size, torch::Tensor const& kv_cache_block_offsets, torch::Tensor const& host_kv_cache_pool_pointers, torch::Tensor const& host_kv_cache_pool_mapping, torch::optional kv_scale_orig_quant, - torch::optional kv_scale_quant_orig, int64_t const layer_idx, int64_t const tokens_per_block, - int64_t const attention_window_size, int64_t const beam_width, int64_t const quant_mode) + int64_t const layer_idx, int64_t const tokens_per_block, int64_t const attention_window_size, + int64_t const beam_width, int64_t const quant_mode) { auto input_dtype = q.scalar_type(); TORCH_CHECK(input_dtype == torch::kFloat16 || input_dtype == torch::kFloat32 || input_dtype == torch::kBFloat16); @@ -321,16 +311,13 @@ void MLARopeAppendPagedKVAssignQ(torch::Tensor& q, torch::Tensor& latent_cache, .kvCacheBuffer; float const* kv_scale_orig_quant_ptr = nullptr; - float const* kv_scale_quant_orig_ptr = nullptr; if (kv_cache_quant_mode.hasKvCacheQuant()) { TLLM_CHECK_WITH_INFO(kv_cache_quant_mode.hasFp8KvCache(), "Only FP8 KV cache is supported for now"); - TORCH_CHECK(kv_scale_orig_quant.has_value()); - TORCH_CHECK(kv_scale_quant_orig.has_value()); + } + if (kv_scale_orig_quant.has_value()) + { kv_scale_orig_quant_ptr = kv_scale_orig_quant.value().data_ptr(); - kv_scale_quant_orig_ptr = kv_scale_quant_orig.value().data_ptr(); - TLLM_CHECK(kv_scale_orig_quant_ptr != nullptr); - TLLM_CHECK(kv_scale_quant_orig_ptr != nullptr); } if (input_dtype == torch::kFloat16) @@ -425,7 +412,6 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) ", Tensor kv_cache_block_offsets" ", Tensor host_kv_cache_pool_pointers" ", Tensor host_kv_cache_pool_mapping" - ", Tensor? kv_scale_orig_quant" ", Tensor? kv_scale_quant_orig" ", int layer_idx" ", int lora_size" @@ -454,7 +440,6 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) ", Tensor kv_cache_block_offsets" ", Tensor host_kv_cache_pool_pointers" ", Tensor host_kv_cache_pool_mapping" - ", Tensor? kv_scale_orig_quant" ", Tensor? kv_scale_quant_orig" ", int layer_idx" ", int lora_size" @@ -491,7 +476,6 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) ", Tensor host_kv_cache_pool_pointers" ", Tensor host_kv_cache_pool_mapping" ", Tensor? kv_scale_orig_quant" - ", Tensor? kv_scale_quant_orig" ", int layer_idx" ", int tokens_per_block" ", int attention_window_size" diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py index bdcdbe7dc6fc..af4fd26ee0ca 100644 --- a/tensorrt_llm/_torch/attention_backend/interface.py +++ b/tensorrt_llm/_torch/attention_backend/interface.py @@ -16,8 +16,8 @@ from ..speculative.spec_tree_manager import SpecTreeManager from tensorrt_llm._utils import get_hf_rope_theta, maybe_pin_memory -from tensorrt_llm.functional import (PositionEmbeddingType, RopeEmbeddingUtils, - RotaryScalingType) +from tensorrt_llm.functional import (AttentionMaskType, PositionEmbeddingType, + RopeEmbeddingUtils, RotaryScalingType) from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -694,6 +694,20 @@ class CustomAttentionMask(str, Enum): AttentionMask = Union[PredefinedAttentionMask, CustomAttentionMask] +@dataclass(kw_only=True, slots=True) +class AttentionSparseArgs: + """Sparse-attention inputs passed to the attention op. + + Backends without sparse attention leave ``AttentionForwardArgs.sparse`` + at its default-constructed value (all-``None`` / ``0`` fields). + """ + sparse_kv_indices: Optional[torch.Tensor] = None + sparse_kv_offsets: Optional[torch.Tensor] = None + sparse_attn_indices: Optional[torch.Tensor] = None + sparse_attn_offsets: Optional[torch.Tensor] = None + sparse_attn_indices_block_size: int = 0 + + @dataclass(kw_only=True, slots=True) class AttentionForwardArgs: """Per-forward optional arguments for attention backends.""" @@ -703,8 +717,8 @@ class AttentionForwardArgs: out_scale: Optional[torch.Tensor] = None out_scale_sf: Optional[torch.Tensor] = None - kv_scales_sf: Optional[torch.Tensor] = None - kv_scales_sf_inv: Optional[torch.Tensor] = None + kv_scale_orig_quant: Optional[torch.Tensor] = None + kv_scale_quant_orig: Optional[torch.Tensor] = None attention_mask: AttentionMask = PredefinedAttentionMask.CAUSAL attention_input_type: AttentionInputType = AttentionInputType.mixed @@ -735,6 +749,22 @@ class AttentionForwardArgs: topk_indices: Optional[torch.Tensor] = None + is_fused_qkv: bool = False + update_kv_cache: bool = True + + sparse: AttentionSparseArgs = field(default_factory=AttentionSparseArgs) + + @property + def mask_type(self) -> int: + """Integer mask type accepted by the C++ attention op + (``causal`` or ``padding``).""" + if self.attention_mask == PredefinedAttentionMask.CAUSAL: + return int(AttentionMaskType.causal) + if self.attention_mask == PredefinedAttentionMask.FULL: + return int(AttentionMaskType.padding) + raise ValueError( + f"Unexpected attention mask type: {self.attention_mask!r}") + _ATTENTION_FORWARD_ARGS_FIELDS = frozenset( AttentionForwardArgs.__dataclass_fields__) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py index 48b2b2b3d8de..204b81d95a58 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py @@ -2532,8 +2532,7 @@ def mla_rope_append_paged_kv_assign_q( block_offsets, metadata.kv_cache_manager.kv_cache_pool_pointers, metadata.kv_cache_manager.kv_cache_pool_mapping, - self.kv_scale_orig_quant, - self.kv_scale_quant_orig, + None, # kv_scale_orig_quant self.get_local_layer_idx(metadata), metadata.kv_cache_manager.tokens_per_block, metadata.kv_cache_manager.max_seq_len, diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index 66899f3d5b26..aedd4c839d03 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -2,7 +2,7 @@ import math import os import weakref -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import TYPE_CHECKING, List, Optional, Tuple import torch @@ -22,14 +22,32 @@ get_model_extra_attrs) from .interface import (AttentionBackend, AttentionForwardArgs, AttentionInputType, AttentionMask, AttentionMetadata, - KVCacheParams, MLAParams, PositionalEmbeddingParams, - PredefinedAttentionMask, RopeParams, - merge_attention_forward_args) + AttentionSparseArgs, KVCacheParams, MLAParams, + PositionalEmbeddingParams, PredefinedAttentionMask, + RopeParams, merge_attention_forward_args) # Enable TRTLLM-Gen attention backend via environment variable (default: on). _TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION = (os.environ.get( "TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION", "0") == "1") +# ``AttentionForwardArgs`` fields that this backend does not consume. +# Sync test (test_attention_op_sync.py) requires every other field to map to a +# kwarg name, a @property on the dataclass, or a field that some @property +# transitively reads; entries here are exempt. +_THOP_EXCLUDED_FIELDS: frozenset = frozenset({ + "topk_indices", # DSA-only + "attention_mask_data", # custom-mask code path + "out_scale_sf", # promoted into ``out_scale`` in ``_run`` for NVFP4 path +}) + +# ``thop.attention`` kwargs hard-wired to a literal at the call site (no +# rich object owns them). Sync test enforces both the kwarg name and the +# literal value. +_THOP_LITERALS: dict = { + "sparse_mla_topk_lens": None, + "compressed_kv_cache_pool_ptr": None, +} + @functools.cache def generate_spec_decoding_position_offsets(max_num_requests: int, @@ -129,6 +147,48 @@ class TrtllmAttentionMetadata(AttentionMetadata): init=False, repr=False) + use_paged_context_fmha: bool = field(init=False, default=False, repr=False) + + # ``DSAtrtllmAttentionMetadata`` overrides this; the dense path keeps 0. + num_sparse_topk: int = 0 + + @property + def effective_workspace(self) -> Optional[torch.Tensor]: + """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" + return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace + + @property + def helix_tensor_params(self) -> List[Optional[torch.Tensor]]: + """``[helix_position_offsets, helix_is_inactive_rank]`` — the positional + helix tensor list expected by the C++ attention op.""" + return [self.helix_position_offsets, self.helix_is_inactive_rank] + + @property + def spec_decoding_bool_params(self) -> List[bool]: + """``[is_spec_decoding_enabled, use_spec_decoding, is_spec_dec_tree]`` — + the positional bool list expected by the C++ attention op.""" + return [ + self.is_spec_decoding_enabled, + self.use_spec_decoding, + self.is_spec_dec_tree, + ] + + @property + def spec_decoding_position_offsets_for_cpp(self) -> Optional[torch.Tensor]: + """``spec_decoding_position_offsets`` reshaped to the 2D layout the C++ + kernel expects. 1D inputs (dynamic-tree shorthand) are viewed as + ``(max_num_requests, -1)``.""" + offsets = self.spec_decoding_position_offsets + if offsets is not None and offsets.dim() == 1: + return offsets.view(self.max_num_requests, -1) + return offsets + + @property + def max_context_length(self) -> int: + """``min(max_seq_len - 1, max_num_tokens)`` — upper bound for a single + context window.""" + return min(self.max_seq_len - 1, self.max_num_tokens) + @property def max_seq_len(self) -> int: """ @@ -174,6 +234,11 @@ def __post_init__(self) -> None: super().__post_init__() self.enable_helix = self.mapping.has_cp_helix( ) if self.mapping is not None else False + self.use_paged_context_fmha = ( + self.runtime_features.chunked_prefill + or self.runtime_features.cache_reuse + or self.runtime_features.has_speculative_draft_tokens + ) if self.runtime_features is not None else False self._post_init_with_buffers(self.cuda_graph_buffers) def update_position_offsets_for_cpp(self, query_len: int) -> None: @@ -986,6 +1051,22 @@ def generate_spec_decoding_generation_length(self, runtime_draft_len): def is_sm_version_trtllm_gen_kernel(self, sm): return not (sm < 100 or sm in [120, 121]) + @property + def spec_decoding_tensor_params(self) -> List[Optional[torch.Tensor]]: + """Positional spec-decoding tensor list for the C++ attention op. + Includes three Blackwell-tree mask tensors on SM versions that take + the trtllm-gen kernel.""" + params = [ + self.spec_decoding_generation_lengths, + self.spec_decoding_position_offsets_for_cpp, + self.spec_decoding_packed_mask, + ] + if self.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()): + params.append(self.spec_decoding_bl_tree_mask_offset) + params.append(self.spec_decoding_bl_tree_mask) + params.append(self.spec_bl_tree_first_sparse_mask_offset_kv) + return params + class TrtllmAttention(AttentionBackend[TrtllmAttentionMetadata]): @@ -1062,11 +1143,22 @@ def __init__( self.print_skip_softmax_stat = os.environ.get( "TRTLLM_PRINT_SKIP_SOFTMAX_STAT", "0") == "1" + # Layer-level fp8 KV-cache scales. Stay at 1.0 (no-op) for the + # PyTorch backend, which never overrides them. They guarantee the + # kernel always receives a valid pointer, since several non-MLA + # XQA kernels (cpp/kernels/xqa/mha.cu, mha_sm90.cu) deref + # ``kvCacheScale[0]`` whenever ``isKVCacheQuantized`` is true and + # do not check for nullptr. ``modules/attention.py`` only assigns + # ``forward_args.kv_scale_*`` for fp4 KV cache, so without this + # fallback the kernel takes nullptr on fp8-KV models → illegal + # memory access. self.kv_cache_scaling_factor = torch.ones(1, dtype=torch.float32, device='cuda') self.kv_scale_quant_orig = self.kv_cache_scaling_factor self.kv_scale_orig_quant = 1.0 / self.kv_cache_scaling_factor + + self.local_layer_idx: Optional[int] = None if not skip_create_weights_in_init: self.update_quant_config(self.quant_config) @@ -1091,10 +1183,14 @@ def update_quant_config(self, new_quant_config: Optional[QuantConfig]): ) def get_local_layer_idx(self, metadata: TrtllmAttentionMetadata) -> int: + if self.local_layer_idx is not None: + return self.local_layer_idx if metadata.kv_cache_manager is None: + # Uncached: recomputed each call until a cache manager appears. return self.layer_idx - else: - return metadata.kv_cache_manager.layer_offsets[self.layer_idx] + self.local_layer_idx = metadata.kv_cache_manager.layer_offsets[ + self.layer_idx] + return self.local_layer_idx def use_nvfp4_output( self, @@ -1235,13 +1331,52 @@ def create_output(self, q, *, is_quantize_output: bool, dtype=out_dtype) ] - def _get_mask_type(self, - attention_mask: AttentionMask) -> AttentionMaskType: - if attention_mask == PredefinedAttentionMask.CAUSAL: - return AttentionMaskType.causal - if attention_mask == PredefinedAttentionMask.FULL: - return AttentionMaskType.padding - raise ValueError("Unexpected attention mask type") + @property + def rotary_embedding_dim(self) -> int: + return self.rope_params.dim + + @property + def rotary_embedding_base(self) -> float: + return self.rope_params.theta + + @property + def rotary_embedding_scale_type(self) -> int: + return int(self.rope_params.scale_type) + + @property + def rotary_embedding_scales(self) -> List[float]: + """``[scale, short_m_scale, long_m_scale]`` — the positional RoPE-scale + list expected by the C++ attention op.""" + return [ + self.rope_params.scale, + self.rope_params.short_m_scale, + self.rope_params.long_m_scale, + ] + + @property + def rotary_embedding_max_position_info(self) -> List[int]: + """``[max_positions, original_max_positions]`` — the positional + RoPE-positions list expected by the C++ attention op.""" + return [ + self.rope_params.max_positions, + self.rope_params.original_max_positions, + ] + + @property + def skip_softmax_threshold_scale_factor_prefill(self) -> Optional[float]: + """Prefill skip-softmax threshold; ``None`` unless + ``sparse_attention_config`` is a ``SkipSoftmaxAttentionConfig``.""" + if isinstance(self.sparse_attention_config, SkipSoftmaxAttentionConfig): + return self.sparse_attention_config.threshold_scale_factor_prefill + return None + + @property + def skip_softmax_threshold_scale_factor_decode(self) -> Optional[float]: + """Decode skip-softmax threshold; ``None`` unless + ``sparse_attention_config`` is a ``SkipSoftmaxAttentionConfig``.""" + if isinstance(self.sparse_attention_config, SkipSoftmaxAttentionConfig): + return self.sparse_attention_config.threshold_scale_factor_decode + return None def _get_trtllm_gen_backend( self) -> trtllm_gen.FlashInferTrtllmGenAttention: @@ -1257,38 +1392,19 @@ def _run( q: torch.Tensor, k: Optional[torch.Tensor], v: Optional[torch.Tensor], - output: torch.Tensor, - output_sf: Optional[torch.Tensor], metadata: TrtllmAttentionMetadata, forward_args: AttentionForwardArgs, - use_paged_context_fmha: bool, - sparse_kv_indices: Optional[torch.Tensor], - sparse_kv_offsets: Optional[torch.Tensor], - sparse_attn_indices: Optional[torch.Tensor], - sparse_attn_offsets: Optional[torch.Tensor], - sparse_attn_indices_block_size: int, - num_sparse_topk: int, - sparse_mla_topk_lens: Optional[torch.Tensor], - compressed_kv_cache_pool_ptr: Optional[int], - skip_softmax_threshold_scale_factor_prefill: Optional[float], - skip_softmax_threshold_scale_factor_decode: Optional[float], ) -> None: - is_fused_qkv = not metadata.is_cross and k is None - update_kv_cache = not metadata.is_cross or k is not None - assert (is_fused_qkv and k is None - and v is None) or (not is_fused_qkv and k is not None - and v is not None) - attention_input_type = forward_args.attention_input_type if not self.is_mla_enable: - if is_fused_qkv: + if forward_args.is_fused_qkv: qkv_hidden_size = (self.num_heads + 2 * self.num_kv_heads) * self.head_dim assert q.shape[1] == qkv_hidden_size else: q_hidden_size = self.num_heads * self.head_dim assert q.shape[1] == q_hidden_size - if update_kv_cache: + if forward_args.update_kv_cache: kv_hidden_size = self.num_kv_heads * self.head_dim assert k.shape[1] == kv_hidden_size assert v.shape[1] == kv_hidden_size @@ -1297,18 +1413,18 @@ def _run( assert k.shape[0] == num_tokens assert v.shape[0] == num_tokens else: - is_sparse_attn = sparse_attn_indices is not None and sparse_attn_indices.numel( + is_sparse_attn = forward_args.sparse.sparse_attn_indices is not None and forward_args.sparse.sparse_attn_indices.numel( ) > 0 if attention_input_type == AttentionInputType.context_only and is_sparse_attn: - assert is_fused_qkv + assert forward_args.is_fused_qkv qkv_hidden_size = self.num_heads * (self.kv_lora_rank + self.qk_rope_head_dim) elif attention_input_type == AttentionInputType.context_only: - assert not is_fused_qkv + assert not forward_args.is_fused_qkv qkv_hidden_size = self.num_heads * (self.qk_nope_head_dim + self.qk_rope_head_dim) elif attention_input_type == AttentionInputType.generation_only: - assert is_fused_qkv + assert forward_args.is_fused_qkv qkv_hidden_size = self.num_heads * (self.kv_lora_rank + self.qk_rope_head_dim) else: @@ -1324,67 +1440,40 @@ def _run( assert metadata.prompt_lens_cpu_runtime.shape[0] == batch_size assert metadata.host_request_types_runtime.shape[0] == batch_size - mask_type = self._get_mask_type(forward_args.attention_mask) self._ensure_rope_table_size(metadata.max_seq_len) - rotary_embedding_dim = self.rope_params.dim - rotary_embedding_base = self.rope_params.theta - rotary_embedding_scale_type = int(self.rope_params.scale_type) - rotary_embedding_scales = [ - self.rope_params.scale, self.rope_params.short_m_scale, - self.rope_params.long_m_scale - ] - rotary_embedding_max_position_info = [ - self.rope_params.max_positions, - self.rope_params.original_max_positions - ] - spec_decoding_bool_params = [ - metadata.is_spec_decoding_enabled, metadata.use_spec_decoding, - metadata.is_spec_dec_tree - ] - spec_decoding_tensor_params = [ - metadata.spec_decoding_generation_lengths, - metadata.spec_decoding_position_offsets_cpp, - metadata.spec_decoding_packed_mask - ] - if self.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()): - spec_decoding_tensor_params.append( - metadata.spec_decoding_bl_tree_mask_offset) - spec_decoding_tensor_params.append( - metadata.spec_decoding_bl_tree_mask) - spec_decoding_tensor_params.append( - metadata.spec_bl_tree_first_sparse_mask_offset_kv) - helix_tensor_params = [ - metadata.helix_position_offsets, metadata.helix_is_inactive_rank - ] - - layer_idx = self.get_local_layer_idx(metadata) - if metadata.spec_decoding_bl_tree_mask is not None and layer_idx == 0: + # Prime ``self.local_layer_idx`` so the ``thop.attention`` kwarg + # below reads a populated int rather than the ``None`` placeholder. + # The call is a fast cache hit after the first forward. + self.local_layer_idx = self.get_local_layer_idx(metadata) + if metadata.spec_decoding_bl_tree_mask is not None and self.local_layer_idx == 0: metadata.spec_decoding_bl_tree_mask.zero_() if self.print_skip_softmax_stat: self.skip_softmax_stat.zero_() - use_nvfp4_output = output_sf is not None - out_scale = (forward_args.out_scale_sf - if use_nvfp4_output else forward_args.out_scale) - kv_scale_orig_quant = (self.kv_scale_orig_quant - if forward_args.kv_scales_sf_inv is None else - forward_args.kv_scales_sf_inv) - kv_scale_quant_orig = (self.kv_scale_quant_orig - if forward_args.kv_scales_sf is None else - forward_args.kv_scales_sf) - mrope_rotary_cos_sin = forward_args.mrope_rotary_cos_sin - mrope_position_deltas = forward_args.mrope_position_deltas - workspace = metadata.workspace if not metadata.is_cuda_graph else metadata.cuda_graph_workspace - flash_mla_tile_scheduler_metadata = ( - metadata.flash_mla_tile_scheduler_metadata - if metadata.enable_flash_mla else None) - flash_mla_num_splits = metadata.flash_mla_num_splits if metadata.enable_flash_mla else None - attention_window_size = (forward_args.attention_window_size - or metadata.max_seq_len) - max_context_length = min(metadata.max_seq_len - 1, - metadata.max_num_tokens) + if forward_args.attention_window_size is None: + forward_args.attention_window_size = metadata.max_seq_len + + # Promote ``out_scale_sf`` -> ``out_scale`` for the NVFP4-output path + # (kernel reads a single ``out_scale`` and interprets it as the SF + # quant scale when ``output_sf`` is allocated). ``output_sf`` is + # populated by ``create_output`` in ``forward`` above, so the + # decision is correct only here, not at the modules/attention.py + # call site where ``output_sf`` is always ``None``. + if forward_args.output_sf is not None and forward_args.out_scale_sf is not None: + forward_args.out_scale = forward_args.out_scale_sf + + # Default ``forward_args.kv_scale_*`` to the layer-level mirrors when + # the caller didn't populate them. ``modules/attention.py`` only sets + # these for fp4 KV cache; fp8-KV models leave them ``None``. Several + # XQA kernels (mha.cu, mha_sm90.cu) deref ``kvCacheScale[0]`` when + # ``isKVCacheQuantized`` is true and don't check for nullptr, so + # passing ``None`` crashes with illegal memory access. + if forward_args.kv_scale_orig_quant is None: + forward_args.kv_scale_orig_quant = self.kv_scale_orig_quant + if forward_args.kv_scale_quant_orig is None: + forward_args.kv_scale_quant_orig = self.kv_scale_quant_orig helix_active = metadata.helix_position_offsets is not None use_sage_attn = (forward_args.sage_attn_num_elts_per_blk_q > 0 @@ -1394,14 +1483,11 @@ def _run( use_trtllm_gen = False if _TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION: trtllm_gen_backend = self._get_trtllm_gen_backend() - trtllm_gen_forward_args = replace(forward_args, - output=output, - output_sf=output_sf) use_trtllm_gen = trtllm_gen_backend.is_supported( q, metadata=metadata, - forward_args=trtllm_gen_forward_args, - mask_type=int(mask_type), + forward_args=forward_args, + mask_type=int(forward_args.mask_type), active_helix=helix_active, use_sage_attn=use_sage_attn, )[0] @@ -1410,100 +1496,127 @@ def _run( trtllm_gen_backend.attention( q, metadata=metadata, - forward_args=trtllm_gen_forward_args, - mask_type=int(mask_type), - use_paged_context_fmha=use_paged_context_fmha, + forward_args=forward_args, + mask_type=int(forward_args.mask_type), + use_paged_context_fmha=metadata.use_paged_context_fmha, ) else: + # Every kwarg sources from ``self`` / ``metadata`` / + # ``forward_args`` (with ``forward_args.sparse`` for sparse-attn + # inputs) or a literal allowlisted in ``_THOP_LITERALS``. + # ``test_attention_op_sync.py`` enforces this statically. thop.attention( - q, - k, - v, - output, - output_sf, - workspace, - metadata.kv_lens_cuda_runtime, - metadata.kv_lens_runtime, - metadata.host_total_kv_lens, - metadata.prompt_lens_cuda_runtime, - metadata.prompt_lens_cpu_runtime, - metadata.host_request_types_runtime, - metadata.kv_cache_block_offsets, - metadata.host_kv_cache_pool_pointers, - metadata.host_kv_cache_pool_mapping, - metadata.cache_indirection, - kv_scale_orig_quant, - kv_scale_quant_orig, - out_scale, - self.rotary_inv_freq, - self.rotary_cos_sin, - forward_args.latent_cache, - forward_args.q_pe, - metadata.block_ids_per_seq, - forward_args.attention_sinks, - is_fused_qkv, - update_kv_cache, - self.predicted_tokens_per_seq, - layer_idx, - self.num_heads, - self.num_kv_heads, - self.head_dim, - metadata.tokens_per_block, - metadata.max_num_requests, - max_context_length, - attention_window_size, - metadata.beam_width, - int(mask_type), - self.quant_mode, - self.q_scaling, - self.position_embedding_type, - rotary_embedding_dim, - rotary_embedding_base, - rotary_embedding_scale_type, - rotary_embedding_scales, - rotary_embedding_max_position_info, - use_paged_context_fmha, - int(attention_input_type), - self.is_mla_enable, - forward_args.chunked_prefill_buffer_batch_size, - self.q_lora_rank, - self.kv_lora_rank, - self.qk_nope_head_dim, - self.qk_rope_head_dim, - self.v_head_dim, - self.rope_append, - mrope_rotary_cos_sin, - mrope_position_deltas, - helix_tensor_params, - self.attention_chunk_size, - forward_args.softmax_stats_tensor, - spec_decoding_bool_params, + q=q, + k=k, + v=v, + output=forward_args.output, + output_sf=forward_args.output_sf, + workspace_=metadata.effective_workspace, + + # --- Per-step batch state (TrtllmAttentionMetadata) --- + sequence_length=metadata.kv_lens_cuda_runtime, + host_past_key_value_lengths=metadata.kv_lens_runtime, + host_total_kv_lens=metadata.host_total_kv_lens, + context_lengths=metadata.prompt_lens_cuda_runtime, + host_context_lengths=metadata.prompt_lens_cpu_runtime, + host_request_types=metadata.host_request_types_runtime, + kv_cache_block_offsets=metadata.kv_cache_block_offsets, + host_kv_cache_pool_pointers=metadata. + host_kv_cache_pool_pointers, + host_kv_cache_pool_mapping=metadata.host_kv_cache_pool_mapping, + cache_indirection=metadata.cache_indirection, + block_ids_per_seq=metadata.block_ids_per_seq, + tokens_per_block=metadata.tokens_per_block, + max_num_requests=metadata.max_num_requests, + beam_width=metadata.beam_width, + use_paged_context_fmha=metadata.use_paged_context_fmha, + helix_tensor_params=metadata.helix_tensor_params, + spec_decoding_bool_params=metadata.spec_decoding_bool_params, + spec_decoding_tensor_params=metadata. spec_decoding_tensor_params, - sparse_kv_indices, - sparse_kv_offsets, - sparse_attn_indices, - sparse_attn_offsets, - sparse_attn_indices_block_size, - num_sparse_topk, - sparse_mla_topk_lens, - skip_softmax_threshold_scale_factor_prefill, - skip_softmax_threshold_scale_factor_decode, - self.skip_softmax_stat, - forward_args.cu_q_seqlens, - forward_args.cu_kv_seqlens, - forward_args.fmha_scheduler_counter, - forward_args.mla_bmm1_scale, - forward_args.mla_bmm2_scale, - forward_args.quant_q_buffer, + num_sparse_topk=metadata.num_sparse_topk, + flash_mla_tile_scheduler_metadata=metadata. flash_mla_tile_scheduler_metadata, - flash_mla_num_splits, - forward_args.sage_attn_num_elts_per_blk_q, - forward_args.sage_attn_num_elts_per_blk_k, - forward_args.sage_attn_num_elts_per_blk_v, - forward_args.sage_attn_qk_int8, + flash_mla_num_splits=metadata.flash_mla_num_splits, num_contexts=metadata.num_contexts, num_ctx_tokens=metadata.num_ctx_tokens, - compressed_kv_cache_pool_ptr=compressed_kv_cache_pool_ptr, + max_context_length=metadata.max_context_length, + + # --- Per-call (AttentionForwardArgs) --- + out_scale=forward_args.out_scale, + kv_scale_orig_quant=forward_args.kv_scale_orig_quant, + kv_scale_quant_orig=forward_args.kv_scale_quant_orig, + latent_cache=forward_args.latent_cache, + q_pe=forward_args.q_pe, + attention_sinks=forward_args.attention_sinks, + mask_type=forward_args.mask_type, + attention_input_type=int(forward_args.attention_input_type), + attention_window_size=forward_args.attention_window_size, + chunked_prefill_buffer_batch_size=forward_args. + chunked_prefill_buffer_batch_size, + mrope_rotary_cos_sin=forward_args.mrope_rotary_cos_sin, + mrope_position_deltas=forward_args.mrope_position_deltas, + softmax_stats_tensor=forward_args.softmax_stats_tensor, + cu_q_seqlens=forward_args.cu_q_seqlens, + cu_kv_seqlens=forward_args.cu_kv_seqlens, + fmha_scheduler_counter=forward_args.fmha_scheduler_counter, + mla_bmm1_scale=forward_args.mla_bmm1_scale, + mla_bmm2_scale=forward_args.mla_bmm2_scale, + quant_q_buffer=forward_args.quant_q_buffer, + sage_attn_num_elts_per_blk_q=forward_args. + sage_attn_num_elts_per_blk_q, + sage_attn_num_elts_per_blk_k=forward_args. + sage_attn_num_elts_per_blk_k, + sage_attn_num_elts_per_blk_v=forward_args. + sage_attn_num_elts_per_blk_v, + sage_attn_qk_int8=forward_args.sage_attn_qk_int8, + is_fused_qkv=forward_args.is_fused_qkv, + update_kv_cache=forward_args.update_kv_cache, + + # --- Module config (TrtllmAttention) --- + rotary_inv_freq=self.rotary_inv_freq, + rotary_cos_sin=self.rotary_cos_sin, + predicted_tokens_per_seq=self.predicted_tokens_per_seq, + local_layer_idx=self.local_layer_idx, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + quant_mode=self.quant_mode, + q_scaling=self.q_scaling, + position_embedding_type=self.position_embedding_type, + rotary_embedding_dim=self.rotary_embedding_dim, + rotary_embedding_base=self.rotary_embedding_base, + rotary_embedding_scale_type=self.rotary_embedding_scale_type, + rotary_embedding_scales=self.rotary_embedding_scales, + rotary_embedding_max_position_info=self. + rotary_embedding_max_position_info, + is_mla_enable=self.is_mla_enable, + q_lora_rank=self.q_lora_rank, + kv_lora_rank=self.kv_lora_rank, + qk_nope_head_dim=self.qk_nope_head_dim, + qk_rope_head_dim=self.qk_rope_head_dim, + v_head_dim=self.v_head_dim, + rope_append=self.rope_append, + attention_chunk_size=self.attention_chunk_size, + skip_softmax_threshold_scale_factor_prefill=self. + skip_softmax_threshold_scale_factor_prefill, + skip_softmax_threshold_scale_factor_decode=self. + skip_softmax_threshold_scale_factor_decode, + skip_softmax_stat=self.skip_softmax_stat, + + # --- Sparse-specific (AttentionForwardArgs.sparse) --- + sparse_kv_indices=forward_args.sparse.sparse_kv_indices, + sparse_kv_offsets=forward_args.sparse.sparse_kv_offsets, + sparse_attn_indices=forward_args.sparse.sparse_attn_indices, + sparse_attn_offsets=forward_args.sparse.sparse_attn_offsets, + sparse_attn_indices_block_size=forward_args.sparse. + sparse_attn_indices_block_size, + + # --- Literals intentionally None (see _THOP_LITERALS) --- + # ``sparse_mla_topk_lens`` and ``compressed_kv_cache_pool_ptr`` + # stay as literal ``None`` until DeepSeek V4 sparse-MLA lands. + sparse_mla_topk_lens=None, + compressed_kv_cache_pool_ptr=None, ) if self.print_skip_softmax_stat: @@ -1530,31 +1643,22 @@ def forward( ) assert not metadata.is_cross, "TRT-LLM Attention does not support cross attention yet." - use_paged_context_fmha = ( - metadata.runtime_features.chunked_prefill - or metadata.runtime_features.cache_reuse - or metadata.runtime_features.has_speculative_draft_tokens - ) if metadata.runtime_features else False - - # This is a workaround for https://nvbugs/5624818 - # Paged context FMHA is forced on SM90 for correctness + # SM90 forces ``use_paged_context_fmha`` on for correctness + # (https://nvbugs/5624818). if get_sm_version() == 90: - use_paged_context_fmha = True + metadata.use_paged_context_fmha = True # Sparse mqa/gqa attention uses generation kernel which reads Q from qPtr (separate buffer). # Force paged context FMHA so QKV preprocessing writes Q to q_buf_2_. if (self.sparse_attention_config is not None and getattr( self.sparse_attention_config, 'algorithm', None) == 'mqa_gqa'): - use_paged_context_fmha = True + metadata.use_paged_context_fmha = True if self.is_mla_enable: # Context MLA uses separate qkv instead of paged_context_fmha - use_paged_context_fmha = False + metadata.use_paged_context_fmha = False - output = forward_args.output - output_sf = forward_args.output_sf - if output is None: - # Output is not provided. + if forward_args.output is None: is_gen_only = (forward_args.attention_input_type == AttentionInputType.generation_only) outputs = self.create_output( @@ -1562,34 +1666,36 @@ def forward( is_quantize_output=forward_args.out_scale is not None, metadata=metadata, attention_mask=forward_args.attention_mask, - use_paged_context_fmha=use_paged_context_fmha, + use_paged_context_fmha=metadata.use_paged_context_fmha, is_mla_enable=self.is_mla_enable, is_gen_only=is_gen_only, ) - - output = outputs[0] - output_sf = outputs[1] if len(outputs) == 2 else None - - sparse_kv_indices, sparse_kv_offsets, sparse_attn_indices, sparse_attn_offsets = None, None, None, None - sparse_attn_indices_block_size = 1 - skip_softmax_threshold_scale_factor_prefill = None - skip_softmax_threshold_scale_factor_decode = None - num_sparse_topk = getattr(metadata, 'num_sparse_topk', 0) - sparse_mla_topk_lens = None - compressed_kv_cache_pool_ptr = None - if self.sparse_attention_config is not None: - if isinstance(self.sparse_attention_config, - SkipSoftmaxAttentionConfig): - skip_softmax_threshold_scale_factor_prefill = self.sparse_attention_config.threshold_scale_factor_prefill - skip_softmax_threshold_scale_factor_decode = self.sparse_attention_config.threshold_scale_factor_decode - - else: - sparse_kv_indices, sparse_kv_offsets = self.sparse_kv_predict( - q, k, metadata, forward_args) - sparse_attn_indices, sparse_attn_offsets = self.sparse_attn_predict( - q, k, metadata, forward_args) - sparse_attn_indices_block_size = self.sparse_attention_config.get_indices_block_size( - ) + forward_args.output = outputs[0] + forward_args.output_sf = outputs[1] if len(outputs) == 2 else None + + forward_args.is_fused_qkv = not metadata.is_cross and k is None + forward_args.update_kv_cache = not metadata.is_cross or k is not None + assert (forward_args.is_fused_qkv and k is None + and v is None) or (not forward_args.is_fused_qkv + and k is not None and v is not None) + + # ``SkipSoftmax`` configs contribute nothing here — their thresholds + # are read via the ``skip_softmax_threshold_scale_factor_*`` + # @property accessors directly on ``self``. + if (self.sparse_attention_config is not None and not isinstance( + self.sparse_attention_config, SkipSoftmaxAttentionConfig)): + kv_idx, kv_off = self.sparse_kv_predict(q, k, metadata, + forward_args) + at_idx, at_off = self.sparse_attn_predict(q, k, metadata, + forward_args) + forward_args.sparse = AttentionSparseArgs( + sparse_kv_indices=kv_idx, + sparse_kv_offsets=kv_off, + sparse_attn_indices=at_idx, + sparse_attn_offsets=at_off, + sparse_attn_indices_block_size=self.sparse_attention_config. + get_indices_block_size(), + ) # Compute FlashMLA tile-scheduler metadata once per forward pass. # The flag is reset in prepare_flash_mla() and update_for_spec_dec() to trigger @@ -1603,24 +1709,17 @@ def forward( metadata._flash_mla_metadata_valid = True # Blackwell first_sparse: refresh at layer 0 before kernel launch. - layer_idx = self.get_local_layer_idx(metadata) - if layer_idx == 0 and (metadata.spec_bl_tree_first_sparse_mask_offset_kv - is not None - and metadata._seq_lens_cuda is not None): + if self.get_local_layer_idx(metadata) == 0 and ( + metadata.spec_bl_tree_first_sparse_mask_offset_kv is not None + and metadata._seq_lens_cuda is not None): metadata.update_blackwell_first_sparse_mask_offset() - self._run(q, k, v, output, output_sf, metadata, forward_args, - use_paged_context_fmha, sparse_kv_indices, sparse_kv_offsets, - sparse_attn_indices, sparse_attn_offsets, - sparse_attn_indices_block_size, num_sparse_topk, - sparse_mla_topk_lens, compressed_kv_cache_pool_ptr, - skip_softmax_threshold_scale_factor_prefill, - skip_softmax_threshold_scale_factor_decode) + self._run(q, k, v, metadata, forward_args) - if output_sf is None: - return output + if forward_args.output_sf is None: + return forward_args.output else: - return output, output_sf + return forward_args.output, forward_args.output_sf @classmethod def support_fused_rope(cls) -> bool: @@ -1684,8 +1783,7 @@ def load_paged_kv_cache_for_mla( metadata.kv_cache_block_offsets, metadata.kv_cache_manager.kv_cache_pool_pointers, metadata.kv_cache_manager.kv_cache_pool_mapping, - self.kv_scale_orig_quant, - self.kv_scale_quant_orig, + None, # kv_scale_quant_orig self.get_local_layer_idx(metadata), self.mla_params.kv_lora_rank, self.mla_params.qk_rope_head_dim, @@ -1730,8 +1828,7 @@ def load_chunked_kv_cache_for_mla( metadata.kv_cache_block_offsets, metadata.kv_cache_manager.kv_cache_pool_pointers, metadata.kv_cache_manager.kv_cache_pool_mapping, - self.kv_scale_orig_quant, - self.kv_scale_quant_orig, + None, # kv_scale_quant_orig self.get_local_layer_idx(metadata), self.mla_params.kv_lora_rank, self.mla_params.qk_rope_head_dim, @@ -1774,8 +1871,7 @@ def mla_rope_append_paged_kv_assign_q( metadata.kv_cache_block_offsets, metadata.kv_cache_manager.kv_cache_pool_pointers, metadata.kv_cache_manager.kv_cache_pool_mapping, - self.kv_scale_orig_quant, - self.kv_scale_quant_orig, + None, # kv_scale_orig_quant self.get_local_layer_idx(metadata), metadata.kv_cache_manager.tokens_per_block, metadata.kv_cache_manager.max_seq_len, @@ -1889,8 +1985,8 @@ def mla_rope_generation( metadata.kv_cache_block_offsets, metadata.kv_cache_manager.kv_cache_pool_pointers, metadata.kv_cache_manager.kv_cache_pool_mapping, - self.kv_scale_orig_quant, - self.kv_scale_quant_orig, + None, # kv_scale_orig_quant + None, # kv_scale_quant_orig out_scale, metadata.block_ids_per_seq, helix_tensor_params, diff --git a/tensorrt_llm/_torch/attention_backend/trtllm_gen.py b/tensorrt_llm/_torch/attention_backend/trtllm_gen.py index 4fd3bc0d0143..16ce98ef876a 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm_gen.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm_gen.py @@ -590,18 +590,9 @@ def _get_kv_scale_params( forward_args: AttentionForwardArgs, quant_mode: int, ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: - attention_layer = self._get_attention_layer() kv_cache_quant_mode = QuantMode(quant_mode) - kv_scale_orig_quant = ( - attention_layer.kv_scale_orig_quant - if forward_args.kv_scales_sf_inv is None - else forward_args.kv_scales_sf_inv - ) - kv_scale_quant_orig = ( - attention_layer.kv_scale_quant_orig - if forward_args.kv_scales_sf is None - else forward_args.kv_scales_sf - ) + kv_scale_orig_quant = forward_args.kv_scale_orig_quant + kv_scale_quant_orig = forward_args.kv_scale_quant_orig if ( not kv_cache_quant_mode.has_kv_cache_quant() or kv_scale_orig_quant is None @@ -619,23 +610,11 @@ def _get_kv_scale_params( return kv_scale_orig_quant, kv_scale_quant_orig - @staticmethod - def _get_attention_output_orig_quant( - forward_args: AttentionForwardArgs, - ) -> Optional[torch.Tensor]: - return ( - forward_args.out_scale_sf - if forward_args.output_sf is not None - else forward_args.out_scale - ) - @staticmethod def _get_mrope_rotary_cos_sin( forward_args: AttentionForwardArgs, ) -> Optional[torch.Tensor]: - if forward_args.mrope_config is None: - return None - return forward_args.mrope_config.get("mrope_rotary_cos_sin") + return forward_args.mrope_rotary_cos_sin def is_supported( self, @@ -989,7 +968,7 @@ def run_context( kv_scale_orig_quant, kv_scale_quant_orig = self._get_kv_scale_params( params.forward, params.kv_cache_quant_mode ) - attention_output_orig_quant = self._get_attention_output_orig_quant(params.forward) + attention_output_orig_quant = params.forward.out_scale mrope_rotary_cos_sin = self._get_mrope_rotary_cos_sin(params.forward) ( @@ -1098,7 +1077,7 @@ def run_generation( kv_scale_orig_quant, kv_scale_quant_orig = self._get_kv_scale_params( params.forward, params.kv_cache_quant_mode ) - attention_output_orig_quant = self._get_attention_output_orig_quant(params.forward) + attention_output_orig_quant = params.forward.out_scale ( q_processed, kv_pool, diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py index 37017dd5e946..d36d030cb402 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py @@ -1208,7 +1208,6 @@ def _handle_prefill_thop_cached_kv( host_kv_cache_pool_pointers, planner.host_pool_mapping, planner.kv_scale_orig_quant, - planner.kv_scale_quant_orig, _CONTEXT_LAYER_OFFSET, tokens_per_block, max_context_length, @@ -1277,7 +1276,6 @@ def _handle_prefill_thop_cached_kv( kv_cache_block_offsets, host_kv_cache_pool_pointers, planner.host_pool_mapping, - planner.kv_scale_orig_quant, planner.kv_scale_quant_orig, _CONTEXT_LAYER_OFFSET, kv_lora_rank, diff --git a/tensorrt_llm/_torch/modules/attention.py b/tensorrt_llm/_torch/modules/attention.py index 8f44c008ee9c..616724550497 100644 --- a/tensorrt_llm/_torch/modules/attention.py +++ b/tensorrt_llm/_torch/modules/attention.py @@ -712,7 +712,7 @@ def _attn_impl( # then exchange and combine across CP ranks. # NOTE: The helix post-process combine step works on unquantized # (BF16/FP16) partial outputs and softmax stats from each rank. - # We intentionally skip passing out_scale/out_scale_sf to FMHA here + # We intentionally skip passing out_scale to FMHA here # so it produces BF16 output. After combining, the downstream o_proj # linear layer handles quantization (FP8/NVFP4) in its apply() method. if self.mapping.has_cp_helix() and attn_metadata.num_contexts == 0: @@ -741,21 +741,27 @@ def _attn_impl( attn_output = self._helix_post_process(attn_output, softmax_stats) return attn_output, None + # Don't set out_scale if o_proj has pre_quant_scale — this prevents + # FP8/FP4 output and keeps attention output in BF16 for better + # precision when applying pre_quant_scale. Also don't set out_scale + # if LoRA is active — LoRA grouped_gemm doesn't support FP8. + # Pass both scales; the backend selects ``out_scale_sf`` when the + # kernel writes NVFP4 output (``forward_args.output_sf`` is + # allocated downstream by ``create_output``) and ``out_scale`` + # otherwise. Deciding here would be premature — ``output_sf`` is + # not populated yet at this call site. out_scale = None out_scale_sf = None - # Don't set out_scale if o_proj has pre_quant_scale - this prevents FP8/FP4 output - # and keeps attention output in BF16 for better precision when applying pre_quant_scale - # Also don't set out_scale if LoRA is active - LoRA grouped_gemm doesn't support FP8 if self._use_quantize_output() and not has_lora: out_scale = self.o_proj.inv_input_scale out_scale_sf = self.o_proj.input_scale - kv_scales_sf = None - kv_scales_sf_inv = None + kv_scale_orig_quant = None + kv_scale_quant_orig = None if self.quant_config is not None and self.quant_config.layer_quant_mode.has_fp4_kv_cache( ): - kv_scales_sf = self.qkv_proj.kv_scales - kv_scales_sf_inv = self.qkv_proj.inv_kv_scales + kv_scale_orig_quant = self.qkv_proj.inv_kv_scales + kv_scale_quant_orig = self.qkv_proj.kv_scales attn_output = self.attn.forward( q, @@ -765,8 +771,8 @@ def _attn_impl( forward_args=AttentionForwardArgs( out_scale=out_scale, out_scale_sf=out_scale_sf, - kv_scales_sf=kv_scales_sf, - kv_scales_sf_inv=kv_scales_sf_inv, + kv_scale_orig_quant=kv_scale_orig_quant, + kv_scale_quant_orig=kv_scale_quant_orig, attention_mask=attention_mask, mrope_rotary_cos_sin=mrope_rotary_cos_sin, mrope_position_deltas=mrope_position_deltas, diff --git a/tests/unittest/_torch/attention_backend/test_attention_op_sync.py b/tests/unittest/_torch/attention_backend/test_attention_op_sync.py new file mode 100644 index 000000000000..b664eda11ab5 --- /dev/null +++ b/tests/unittest/_torch/attention_backend/test_attention_op_sync.py @@ -0,0 +1,548 @@ +# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Static sync test for the inline ``thop.attention(...)`` call in +``TrtllmAttention._run``. + +That call is the single explicit-kwarg call site for the C++ ``thop.attention`` +binding. This test parses both the call site (Python AST) and the C++ +function declaration in ``attentionOp.h`` (text/regex), and enforces: + +1. Every C++ parameter name appears at the call site (and nothing extra). +2. Every call-site kwarg sourced as ``root.attr[.attr...]`` resolves on + exactly one of ``self`` / ``metadata`` / ``forward_args``, and its + declared C++ type matches the source attribute's Python type at a + coarse-category level (tensor / int / bool / float / list-of-X). +3. Every dataclass field reachable from ``AttentionForwardArgs`` (including + nested dataclass sub-bags like ``AttentionSparseArgs``) is consumed at + the call site — directly, transitively via a @property of the + containing class, or listed in ``_THOP_EXCLUDED_FIELDS``. +4. Every kwarg passed as a literal constant matches an entry in + ``_THOP_LITERALS`` (both name and value). + +The test is AST-only (no kernel run) so it fails fast and runs without a GPU. +""" + +import ast +import dataclasses +import inspect +import pathlib +import re +import textwrap +import typing +from dataclasses import fields + +from tensorrt_llm._torch.attention_backend.interface import AttentionForwardArgs +from tensorrt_llm._torch.attention_backend.trtllm import ( + _THOP_EXCLUDED_FIELDS, + _THOP_LITERALS, + TrtllmAttention, + TrtllmAttentionMetadata, +) + +# Roots used as the LHS of attribute chains at the call site. Match the +# parameter names inside ``TrtllmAttention._run``. +_SOURCE_CLASSES = { + "self": TrtllmAttention, + "metadata": TrtllmAttentionMetadata, + "forward_args": AttentionForwardArgs, +} + +# The C++ attention() declaration is the single source of truth for kwarg +# names, ordering, and types. +_HEADER = pathlib.Path(__file__).resolve().parents[4] / ("cpp/tensorrt_llm/thop/attentionOp.h") + + +# ---- C++ declaration parser ------------------------------------------------- + + +_TENSOR_RE = re.compile(r"\btorch::Tensor\b") +_QUANT_MODE_RE = re.compile(r"\bcommon::QuantMode\b") +_INT64_RE = re.compile(r"\bint64_t\b") +_DOUBLE_RE = re.compile(r"\bdouble\b") +_BOOL_RE = re.compile(r"\bbool\b") + + +def _split_top_level(s: str, sep: str = ",") -> list[str]: + """Split ``s`` by ``sep`` at angle-bracket depth 0.""" + out: list[str] = [] + depth = 0 + buf: list[str] = [] + for ch in s: + if ch == "<": + depth += 1 + buf.append(ch) + elif ch == ">": + depth -= 1 + buf.append(ch) + elif ch == sep and depth == 0: + out.append("".join(buf).strip()) + buf = [] + else: + buf.append(ch) + if buf: + out.append("".join(buf).strip()) + return out + + +def _strip_inner_type(cpp_type: str) -> str: + """Strip ``std::optional<...>`` and ``const`` / references to expose the + inner element type. Idempotent.""" + t = cpp_type.replace("const", "").replace("&", "").strip() + m = re.fullmatch(r"std::optional<\s*(.+)\s*>", t) + if m: + t = m.group(1).strip() + return t + + +def _cpp_category(cpp_type: str) -> str: + """Coarse Python-side category for a C++ parameter type. + + Returns one of ``tensor`` / ``int`` / ``bool`` / ``float`` / + ``list_tensor`` / ``list_int`` / ``list_bool`` / ``list_float`` / + ``unknown``. Optional/const wrappers and references are ignored. + """ + bare = _strip_inner_type(cpp_type) + if bare.startswith("std::vector<"): + inner_match = re.fullmatch(r"std::vector<\s*(.+)\s*>", bare) + if inner_match: + inner_cat = _cpp_category(inner_match.group(1)) + return f"list_{inner_cat}" + if _TENSOR_RE.search(bare): + return "tensor" + if _BOOL_RE.fullmatch(bare): + return "bool" + if _INT64_RE.fullmatch(bare) or _QUANT_MODE_RE.fullmatch(bare): + return "int" + if _DOUBLE_RE.fullmatch(bare): + return "float" + return "unknown" + + +def _parse_attention_decl() -> list[tuple[str, str]]: + """Parse the ``void attention(...)`` declaration in ``attentionOp.h`` + and return ``[(name, cpp_type), ...]`` in declaration order.""" + src = _HEADER.read_text() + # Strip line comments to keep the regex tidy. + src = re.sub(r"//[^\n]*", "", src) + m = re.search(r"void\s+attention\s*\(", src) + if not m: + raise AssertionError(f"Could not find void attention(...) in {_HEADER}") + # Walk forward, matching the opening paren at m.end()-1 to its close. + start = m.end() - 1 + depth = 0 + end = None + for i in range(start, len(src)): + if src[i] == "(": + depth += 1 + elif src[i] == ")": + depth -= 1 + if depth == 0: + end = i + break + if end is None: + raise AssertionError("Unbalanced parens in attention() declaration") + body = src[start + 1 : end] + + params: list[tuple[str, str]] = [] + for raw in _split_top_level(body): + # Drop ``= default`` suffix at top level only. + param = _split_top_level(raw, sep="=")[0].strip() + # The name is the trailing identifier. Strip trailing '&' or '*' + # attached to the type, not the name. + m = re.match(r"(.*?)([A-Za-z_]\w*)\s*$", param, re.DOTALL) + if not m: + raise AssertionError(f"Could not parse param: {param!r}") + cpp_type = m.group(1).strip() + name = m.group(2) + params.append((name, cpp_type)) + return params + + +def _binding_kwargs() -> set[str]: + """Set of parameter names declared on ``void attention(...)``.""" + return {name for name, _ in _parse_attention_decl()} + + +def _binding_types() -> dict[str, str]: + """Map parameter name → declared C++ type.""" + return dict(_parse_attention_decl()) + + +# ---- Call-site AST helpers -------------------------------------------------- + + +def _parse_thop_attention_call() -> ast.Call: + """Locate the single ``thop.attention(...)`` call inside ``_run``.""" + src = textwrap.dedent(inspect.getsource(TrtllmAttention._run)) + tree = ast.parse(src) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "attention" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "thop" + ): + return node + raise AssertionError("Could not find thop.attention(...) call in TrtllmAttention._run") + + +def _attribute_path(node: ast.AST) -> tuple[str, tuple[str, ...]] | None: + """If ``node`` is a pure attribute chain ``Name.attr1.attr2...``, return + ``(root_name_id, (attr1, attr2, ...))``. Otherwise return ``None``. + """ + if not isinstance(node, ast.Attribute): + return None + attrs: list[str] = [] + current: ast.AST = node + while isinstance(current, ast.Attribute): + attrs.append(current.attr) + current = current.value + if not isinstance(current, ast.Name): + return None + return current.id, tuple(reversed(attrs)) + + +def _classify_kwargs() -> tuple[ + dict[str, tuple[str, tuple[str, ...]]], dict[str, object], set[str] +]: + """Split the call site's kwargs into three buckets: + + - ``attr_kwargs``: ``kwarg=source.attr[...]`` (or + ``kwarg=int(source.attr)``) → ``{kwarg: (root, path)}``. + - ``literal_kwargs``: ``kwarg=`` → ``{kwarg: value}``. + - ``other_kwargs``: kwargs whose value is anything else (e.g. a bare + Name like ``q``). + """ + call = _parse_thop_attention_call() + attr_kwargs: dict[str, tuple[str, tuple[str, ...]]] = {} + literal_kwargs: dict[str, object] = {} + other_kwargs: set[str] = set() + for kw in call.keywords: + v = kw.value + # int(source.attr) → equivalent to source.attr for sync purposes. + if ( + isinstance(v, ast.Call) + and isinstance(v.func, ast.Name) + and v.func.id == "int" + and len(v.args) == 1 + ): + path = _attribute_path(v.args[0]) + if path is not None: + attr_kwargs[kw.arg] = path + continue + other_kwargs.add(kw.arg) + continue + if isinstance(v, ast.Constant): + literal_kwargs[kw.arg] = v.value + continue + path = _attribute_path(v) + if path is not None: + attr_kwargs[kw.arg] = path + continue + other_kwargs.add(kw.arg) + return attr_kwargs, literal_kwargs, other_kwargs + + +# ---- Python-side attribute & type resolution -------------------------------- + + +def _runtime_instance_attrs(cls) -> set[str]: + """Names assigned as ``self. = ...`` anywhere in ``cls`` or any of + its base classes.""" + cache = _runtime_instance_attrs._cache # type: ignore[attr-defined] + if cls in cache: + return cache[cls] + + def _walk_target(tgt: ast.AST, names: set[str]) -> None: + if isinstance(tgt, (ast.Tuple, ast.List)): + for elt in tgt.elts: + _walk_target(elt, names) + elif ( + isinstance(tgt, ast.Attribute) + and isinstance(tgt.value, ast.Name) + and tgt.value.id == "self" + ): + names.add(tgt.attr) + + names: set[str] = set() + for base in cls.__mro__: + if base is object: + continue + try: + src = textwrap.dedent(inspect.getsource(base)) + except (OSError, TypeError): + continue + for node in ast.walk(ast.parse(src)): + if isinstance(node, ast.Assign): + for tgt in node.targets: + _walk_target(tgt, names) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + _walk_target(node.target, names) + cache[cls] = names + return names + + +_runtime_instance_attrs._cache = {} # type: ignore[attr-defined] + + +def _has_attr_or_field(cls, name: str) -> bool: + if hasattr(cls, name): + return True + try: + if name in {f.name for f in fields(cls)}: + return True + except TypeError: + pass + return name in _runtime_instance_attrs(cls) + + +def _dataclass_field_type(cls, name: str): + try: + f = cls.__dataclass_fields__.get(name) # type: ignore[attr-defined] + except AttributeError: + return None + if f is None: + return None + return f.type if not isinstance(f.type, str) else None + + +def _resolve_path(root_cls, path: tuple[str, ...]): + """Walk ``path[:-1]`` from ``root_cls`` and return ``(leaf_cls, + leaf_attr)``. Returns ``(None, leaf_attr)`` if an intermediate step is + not a dataclass field.""" + cls = root_cls + for step in path[:-1]: + nxt = _dataclass_field_type(cls, step) + if nxt is None: + return None, path[-1] + cls = nxt + return cls, path[-1] + + +def _python_category(py_type) -> str: + """Coarse classification of a Python annotation, mirroring + ``_cpp_category``. Returns ``unknown`` for anything we can't classify + confidently (the type check is then skipped for that kwarg).""" + # Unwrap Optional[X] / Union[X, None]. + origin = typing.get_origin(py_type) + if origin is typing.Union: + args = [a for a in typing.get_args(py_type) if a is not type(None)] + if len(args) == 1: + return _python_category(args[0]) + return "unknown" + if origin in (list, typing.List): + args = typing.get_args(py_type) + if args: + return f"list_{_python_category(args[0])}" + return "list_unknown" + if py_type is bool: + return "bool" + if py_type is int: + return "int" + if py_type is float: + return "float" + if isinstance(py_type, type): + if py_type.__name__ == "Tensor": + return "tensor" + return "unknown" + + +# ---- Tests ------------------------------------------------------------------ + + +def test_call_site_kwargs_match_binding_kwargs(): + """The call site's keyword set must equal the C++ binding's keyword set.""" + cpp_kwargs = _binding_kwargs() + attr_kwargs, literal_kwargs, other_kwargs = _classify_kwargs() + call_site_kwargs = set(attr_kwargs) | set(literal_kwargs) | other_kwargs + assert call_site_kwargs == cpp_kwargs, ( + f"missing in call site: {cpp_kwargs - call_site_kwargs}, " + f"unknown to C++ binding: {call_site_kwargs - cpp_kwargs}" + ) + + +def test_each_source_attr_kwarg_resolves_uniquely(): + """``root.attr[.attr...]`` kwargs must resolve along the chain to an + attribute that exists on exactly one of the source classes, and the + leaf's Python-category must match the C++ kwarg's category (where + both are unambiguously known).""" + attr_kwargs, _, _ = _classify_kwargs() + cpp_types = _binding_types() + for thop_kwarg, (root, path) in attr_kwargs.items(): + assert root in _SOURCE_CLASSES, ( + f"thop kwarg `{thop_kwarg}` sources from unknown root `{root}` — " + f"call site must use one of {set(_SOURCE_CLASSES)}." + ) + leaf_cls, leaf_attr = _resolve_path(_SOURCE_CLASSES[root], path) + chain = ".".join((root, *path)) + assert leaf_cls is not None, ( + f"thop kwarg `{thop_kwarg}` reads `{chain}` but an intermediate " + f"step is not a dataclass field of its parent." + ) + assert _has_attr_or_field(leaf_cls, leaf_attr), ( + f"thop kwarg `{thop_kwarg}` reads `{chain}` but `{leaf_attr}` " + f"is not exposed on {leaf_cls.__name__}." + ) + # Coarse type cross-check (skipped when either side is unknown). + cpp_cat = _cpp_category(cpp_types[thop_kwarg]) + leaf_py_type = _dataclass_field_type(leaf_cls, leaf_attr) + if cpp_cat != "unknown" and leaf_py_type is not None: + py_cat = _python_category(leaf_py_type) + if py_cat != "unknown": + assert cpp_cat == py_cat, ( + f"thop kwarg `{thop_kwarg}` (C++ {cpp_types[thop_kwarg]!r}" + f" → {cpp_cat}) bound to `{chain}` " + f"({leaf_py_type!r} → {py_cat})." + ) + + +def test_literal_kwargs_match_allowlist(): + """Every literal-constant kwarg at the call site must appear in + ``_THOP_LITERALS`` with the matching value, and every entry in + ``_THOP_LITERALS`` must be used at the call site.""" + _, literal_kwargs, _ = _classify_kwargs() + unknown = set(literal_kwargs) - set(_THOP_LITERALS) + assert not unknown, ( + f"kwargs passed as literals but not in _THOP_LITERALS: " + f"{sorted(unknown)}. Source from one of {set(_SOURCE_CLASSES)} or " + f"add to the allowlist." + ) + stale = set(_THOP_LITERALS) - set(literal_kwargs) + assert not stale, ( + f"_THOP_LITERALS entries no longer passed as literals: " + f"{sorted(stale)}. Drop them or restore the call-site literal." + ) + for kwarg, value in literal_kwargs.items(): + assert value == _THOP_LITERALS[kwarg], ( + f"thop kwarg `{kwarg}` passed as literal {value!r} but " + f"_THOP_LITERALS expects {_THOP_LITERALS[kwarg]!r}." + ) + + +def _self_attrs_in_property(prop: property) -> set[str]: + """``self.`` names read inside ``prop``'s getter body.""" + src = textwrap.dedent(inspect.getsource(prop.fget)) + return { + node.attr + for node in ast.walk(ast.parse(src)) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + } + + +def _collect_chains(root: str) -> set[tuple[str, ...]]: + """All attribute paths in ``_run`` that start with ``Name(root).``.""" + src = textwrap.dedent(inspect.getsource(TrtllmAttention._run)) + chains: set[tuple[str, ...]] = set() + for node in ast.walk(ast.parse(src)): + if not isinstance(node, ast.Attribute): + continue + path: list[str] = [] + cur: ast.AST = node + while isinstance(cur, ast.Attribute): + path.insert(0, cur.attr) + cur = cur.value + if isinstance(cur, ast.Name) and cur.id == root: + chains.add(tuple(path)) + return chains + + +def _verify_consumed(cls, chains: set[tuple[str, ...]], excluded=frozenset()): + """Recursively assert that every field on ``cls`` is consumed by some + chain in ``chains`` (or excluded). For nested dataclass fields, recurse + into the sub-bag with chain tails. Properties on ``cls`` accessed at + the call site transitively consume the fields they read on ``self``. + """ + direct = {p[0] for p in chains if p} + transitive: set[str] = set() + for name in direct: + obj = vars(cls).get(name) + if isinstance(obj, property): + transitive |= _self_attrs_in_property(obj) + consumed = direct | transitive + for f in fields(cls): + if f.name in excluded: + continue + ftype = f.type if not isinstance(f.type, str) else None + if ftype is not None and dataclasses.is_dataclass(ftype): + sub = {p[1:] for p in chains if len(p) >= 2 and p[0] == f.name} + assert sub, ( + f"Nested dataclass field `{f.name}` on {cls.__name__} is " + f"declared but `{f.name}.` is never read at the " + f"call site." + ) + _verify_consumed(ftype, sub) + else: + assert f.name in consumed, ( + f"Field `{f.name}` on {cls.__name__} not consumed by the " + f"thop call site (directly or via @property). Source it at " + f"the call site or add it to _THOP_EXCLUDED_FIELDS." + ) + + +def test_every_forward_args_field_is_consumed(): + """Recursively check that every dataclass field reachable from + ``AttentionForwardArgs`` (including nested sub-bags such as + ``AttentionSparseArgs``) is consumed at the call site, transitively + via @property where applicable, or listed in ``_THOP_EXCLUDED_FIELDS``. + """ + _verify_consumed( + AttentionForwardArgs, + _collect_chains("forward_args"), + excluded=_THOP_EXCLUDED_FIELDS, + ) + + +def test_no_unexpected_other_kwargs(): + """The only call-site kwargs that aren't ``source.attr`` chains or + allowlisted literals are the ``_run`` positional parameters.""" + _, _, other_kwargs = _classify_kwargs() + expected = {"q", "k", "v"} + unexpected = other_kwargs - expected + assert not unexpected, ( + f"thop kwargs with unexpected expression form: {sorted(unexpected)}. " + f"Allowed: source.attr[.attr...], int(source.attr), literal " + f"constant, or one of {sorted(expected)}." + ) + + +def _all_forward_args_field_names() -> set[str]: + """All dataclass field names reachable from ``AttentionForwardArgs``, + recursively descending into nested dataclass sub-bags.""" + seen: set[str] = set() + + def _walk(cls) -> None: + for f in fields(cls): + seen.add(f.name) + ftype = f.type if not isinstance(f.type, str) else None + if ftype is not None and dataclasses.is_dataclass(ftype): + _walk(ftype) + + _walk(AttentionForwardArgs) + return seen + + +def test_excluded_fields_match_real_fields(): + """Every entry in ``_THOP_EXCLUDED_FIELDS`` must name a real field on + ``AttentionForwardArgs`` (or a nested sub-bag). Dead entries (typos, + fields removed in a refactor) silently allow newly-added real fields + to slip past ``test_every_forward_args_field_is_consumed``.""" + stale = set(_THOP_EXCLUDED_FIELDS) - _all_forward_args_field_names() + assert not stale, ( + f"_THOP_EXCLUDED_FIELDS entries that don't match any " + f"AttentionForwardArgs field: {sorted(stale)}. Drop them or " + f"restore the field." + ) From e73d06808d1fce6f8227e6a1be126a6dfd3d0226 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Thu, 28 May 2026 16:57:36 +0800 Subject: [PATCH 083/308] [https://nvbugs/6160085][fix] At `tensorrt_llm/tokenizer/tokenizer.py` import time, re-export `bytes_to_unicod (#14116) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tensorrt_llm/tokenizer/tokenizer.py | 13 +++++++++++++ tests/integration/test_lists/waives.txt | 2 -- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index d08cb9ed5a42..f681b5496527 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -10,6 +10,19 @@ from .._utils import nvtx_range_debug from ..logger import logger +# Transformers 5.x moved ``bytes_to_unicode`` out of +# ``transformers.models.gpt2.tokenization_gpt2`` into +# ``transformers.convert_slow_tokenizer``. Some ``trust_remote_code=True`` +# checkpoints (e.g. Kimi-K2's ``tokenization_kimi.py``) still import it from +# the legacy location; re-export the symbol so those tokenizers keep loading. +try: + from transformers.models.gpt2 import tokenization_gpt2 as _gpt2_mod + if not hasattr(_gpt2_mod, "bytes_to_unicode"): + from transformers.convert_slow_tokenizer import bytes_to_unicode + _gpt2_mod.bytes_to_unicode = bytes_to_unicode +except ImportError: + pass + # Aliases for built-in custom tokenizers. TOKENIZER_ALIASES = { "deepseek_v32": "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 7be6c4df5a2c..f8f7f54ec599 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -88,8 +88,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) From 89bcd802bf7536f265ffc5e7ada5ea7cf4f3a486 Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Thu, 28 May 2026 18:33:46 +0800 Subject: [PATCH 084/308] [None][infra] Waive 1 failed cases for main in post-merge 2740 (#14688) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f8f7f54ec599..3b284cb55d62 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -287,6 +287,7 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwel perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k] SKIP (https://nvbugs/6190071) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6189928) perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k] SKIP (https://nvbugs/6227472) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236108) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6192201) From 6484b713c21b311dbb5df401bb6a53039ef356e0 Mon Sep 17 00:00:00 2001 From: Grzegorz Kwasniewski <213329731+greg-kwasniewski1@users.noreply.github.com> Date: Thu, 28 May 2026 13:50:11 +0200 Subject: [PATCH 085/308] [https://nvbugs/6221483][fix] Revert auto_deploy _mamba_ssm_prepare_metadata to pre-#13566 state (#14640) Signed-off-by: greg-kwasniewski1 <213329731+greg-kwasniewski1@users.noreply.github.com> Signed-off-by: Gal Hubara-Agam <96368689+galagam@users.noreply.github.com> Co-authored-by: Gal Hubara-Agam <96368689+galagam@users.noreply.github.com> --- .../custom_ops/mamba/mamba_backend_common.py | 26 +++---------------- tests/integration/test_lists/waives.txt | 2 -- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py index 45feb2d4ea4b..a91095fe7aa1 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/mamba/mamba_backend_common.py @@ -19,10 +19,7 @@ from torch._ops import OpOverloadPacket from torch.fx import Node -from tensorrt_llm._torch.modules.mamba.mamba2_metadata import ( - compute_extra_chunks_cpu, - cu_seqlens_to_chunk_indices_offsets_triton, -) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import cu_seqlens_to_chunk_indices_offsets from tensorrt_llm._torch.modules.mamba.ssd_combined import mamba_chunk_scan_combined from ..._compat import KvCacheConfig @@ -44,7 +41,6 @@ def _mamba_ssm_prepare_metadata( position_ids: torch.Tensor, batch_info_host: torch.Tensor, seq_len: torch.Tensor, - seq_len_host: torch.Tensor, cu_seqlen: torch.Tensor, # EXTRA METADATA PROVIDED BY THE DESCRIPTOR chunk_size: int, @@ -52,10 +48,6 @@ def _mamba_ssm_prepare_metadata( """Prepare metadata for cached SSM transform. Returns a tuple of (chunk_indices, chunk_offsets, seq_idx_prefill). - - Uses seq_len_host (CPU tensor) and batch_info_host to derive total_seqlens - and extra_chunks without GPU->CPU synchronization, preventing deadlocks - when NCCL collectives from MoE expert-parallel layers are pending. """ device = cu_seqlen.device batch_info = BatchInfo(batch_info_host) @@ -63,20 +55,11 @@ def _mamba_ssm_prepare_metadata( num_prefill, _, _ = batch_info.get_num_sequences() if num_prefill > 0: - num_prefill_tokens, _, _ = batch_info.get_num_tokens() - - _extra = compute_extra_chunks_cpu(seq_len_host, num_prefill, chunk_size) - - chunk_indices, chunk_offsets = cu_seqlens_to_chunk_indices_offsets_triton( - cu_seqlen[: num_prefill + 1], - chunk_size, - total_seqlens=num_prefill_tokens, - extra_chunks=_extra, + chunk_indices, chunk_offsets = cu_seqlens_to_chunk_indices_offsets( + cu_seqlen[: num_prefill + 1], chunk_size ) seq_idx_prefill = torch.repeat_interleave( - torch.arange(num_prefill, device=device, dtype=torch.int32), - seq_len[:num_prefill], - output_size=num_prefill_tokens, + torch.arange(num_prefill, device=device, dtype=torch.int32), seq_len[:num_prefill] ).view(1, -1) else: chunk_indices = torch.empty(0, dtype=torch.int32, device=device) @@ -92,7 +75,6 @@ def _mamba_ssm_prepare_metadata_fake( position_ids: torch.Tensor, batch_info_host: torch.Tensor, seq_len: torch.Tensor, - seq_len_host: torch.Tensor, cu_seqlen: torch.Tensor, # EXTRA METADATA PROVIDED BY THE DESCRIPTOR chunk_size: int, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3b284cb55d62..3fbc93110ee6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -15,8 +15,6 @@ accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbu accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) -accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[fp8_ws4_80gb-trtllm] SKIP (https://nvbugs/6221483) -accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] SKIP (https://nvbugs/6221483) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) From 82679b58ab6ce83bcd6f1d5d1744c715c851fca6 Mon Sep 17 00:00:00 2001 From: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Date: Thu, 28 May 2026 07:56:01 -0400 Subject: [PATCH 086/308] [TRTLLM-12762][fix] Enable multi-node TP for MiniMax-M2 (#14314) Signed-off-by: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> --- .../_torch/models/modeling_minimaxm2.py | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm2.py b/tensorrt_llm/_torch/models/modeling_minimaxm2.py index 944f20ec77f3..754962ab5ebc 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm2.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm2.py @@ -19,6 +19,7 @@ from torch import nn from transformers import PretrainedConfig +from tensorrt_llm._ipc_utils import can_access_peer from tensorrt_llm.functional import AllReduceStrategy, PositionEmbeddingType from tensorrt_llm.mapping import Mapping @@ -119,7 +120,13 @@ def forward( # We use all_reduce across all tp gpus to get the rms norm variance sum class MiniMaxRMSNorm(nn.Module): def __init__( - self, *, hidden_size: int, eps: float, mapping: Mapping, dtype: torch.dtype = torch.bfloat16 + self, + *, + hidden_size: int, + eps: float, + mapping: Mapping, + dtype: torch.dtype = torch.bfloat16, + head_dim: Optional[int] = None, ): super().__init__() self.mapping = mapping @@ -128,14 +135,37 @@ def __init__( self.hidden_size = hidden_size self.eps = eps self.dtype = dtype + self.head_dim = head_dim + self.is_p2p_supported = can_access_peer(mapping) self.all_reduce = AllReduce(mapping=self.mapping, strategy=AllReduceStrategy.NCCL) self.minimax_all_reduce_rms = MiniMaxAllReduceRMS(mapping=self.mapping) def load_weights(self, weights: List[Dict]): assert len(weights) == 1 + src = weights[0]["weight"] + # When num_total_heads < tp_size (e.g. 8 KV heads, tp=16), the checkpoint weight + # [num_total_heads * head_dim] is smaller than what TP sharding expects + # [tp_size * local_hidden_size]. Replicate at the head level before sharding, + # consistent with how duplicate_kv_weight handles k_proj/v_proj. + full_size = self.mapping.tp_size * self.hidden_size + if src.shape[0] < full_size and self.head_dim is not None: + assert src.shape[0] % self.head_dim == 0, ( + f"checkpoint weight size {src.shape[0]} is not divisible by head_dim {self.head_dim}" + ) + num_total_heads = src.shape[0] // self.head_dim + assert self.mapping.tp_size % num_total_heads == 0, ( + f"tp_size {self.mapping.tp_size} must be divisible by num_total_heads {num_total_heads} " + f"for head-level weight replication" + ) + reps = self.mapping.tp_size // num_total_heads + src = ( + src.reshape(num_total_heads, self.head_dim) + .repeat_interleave(reps, dim=0) + .reshape(-1) + ) weight = load_weight_shard( - weights[0]["weight"], + src, tensor_parallel_size=self.mapping.tp_size, tensor_parallel_rank=self.mapping.tp_rank, tensor_parallel_mode=TensorParallelMode.COLUMN, @@ -144,6 +174,15 @@ def load_weights(self, weights: List[Dict]): def forward(self, hidden_states: torch.Tensor): hidden_states = hidden_states.contiguous() + if not self.is_p2p_supported: + # Inter-node TP: IPC is unavailable, fall back to NCCL all-reduce of + # partial sum-of-squares followed by local RMS normalization. + hidden_f32 = hidden_states.float() + local_sum_sq = hidden_f32.pow(2).sum(-1, keepdim=True) + total_sum_sq = self.all_reduce(local_sum_sq) + total_hidden = self.hidden_size * self.mapping.tp_size + rms_inv = torch.rsqrt(total_sum_sq / total_hidden + self.eps) + return (hidden_f32 * rms_inv).to(hidden_states.dtype) * self.weight rms_norm_out = self.minimax_all_reduce_rms(hidden_states, self.weight, self.eps) return rms_norm_out @@ -188,12 +227,14 @@ def __init__( eps=config.rms_norm_eps, mapping=self.qkv_proj.mapping, dtype=config.torch_dtype, + head_dim=self.head_dim, ) self.k_norm = MiniMaxRMSNorm( hidden_size=self.kv_size, eps=config.rms_norm_eps, mapping=self.qkv_proj.mapping, dtype=config.torch_dtype, + head_dim=self.head_dim, ) else: self.q_norm = RMSNorm( @@ -209,6 +250,9 @@ def __init__( def apply_qk_norm(self, q, k): if self.qkv_proj.mapping.tp_size > 1: + if not self.q_norm.is_p2p_supported: + # Inter-node TP: fall back to separate per-tensor NCCL-based norm. + return self.q_norm(q), self.k_norm(k) q = q.contiguous() k = k.contiguous() q, k = self.q_norm.minimax_all_reduce_rms.forward_qk( From 0432a81ccb0c11b944856e007b2bdc8c5eef13b8 Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Thu, 28 May 2026 10:05:57 -0400 Subject: [PATCH 087/308] [https://nvbugs/6043248][fix] Validate tensor payload size on deserialization (#14648) Signed-off-by: Yibin-Li <109242046+yibinl-nvidia@users.noreply.github.com> --- cpp/tensorrt_llm/executor/serialization.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 59b0389f4e90..e4c325423126 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -933,6 +933,11 @@ Tensor Serialization::deserializeTensor(std::istream& is) // Size in bytes size_t sizeInBytes{0}; is.read(reinterpret_cast(&sizeInBytes), sizeof(size_t)); + auto checkTensorSize = [sizeInBytes](Tensor const& tensor) + { + TLLM_CHECK_WITH_INFO(sizeInBytes == tensor.getSizeInBytes(), + "Serialized tensor byte size does not match size implied by tensor data type and shape."); + }; Tensor tensor; switch (memoryType) @@ -940,18 +945,21 @@ Tensor Serialization::deserializeTensor(std::istream& is) case MemoryType::kCPU: { tensor = Tensor::cpu(dataType, shape); + checkTensorSize(tensor); is.read(reinterpret_cast(tensor.getData()), static_cast(sizeInBytes)); break; } case MemoryType::kCPU_PINNED: { tensor = Tensor::pinned(dataType, shape); + checkTensorSize(tensor); is.read(reinterpret_cast(tensor.getData()), static_cast(sizeInBytes)); break; } case MemoryType::kUVM: { tensor = Tensor::managed(dataType, shape); + checkTensorSize(tensor); is.read(reinterpret_cast(tensor.getData()), static_cast(sizeInBytes)); break; } @@ -960,6 +968,7 @@ Tensor Serialization::deserializeTensor(std::istream& is) // TODO: Eventually we might want to support serialization/deserialization in GPU memory // Until then created Pinned tensor and move to GPU auto pinnedTensor = Tensor::pinned(dataType, shape); + checkTensorSize(pinnedTensor); is.read(reinterpret_cast(pinnedTensor.getData()), static_cast(sizeInBytes)); auto stream = std::make_shared(); tensor = pinnedTensor.copyToGpu(stream); From 56be62529c6467eb46fa2b1c13f6b9f48f3723af Mon Sep 17 00:00:00 2001 From: tcherckez-nvidia <127761168+tcherckez-nvidia@users.noreply.github.com> Date: Thu, 28 May 2026 17:09:55 +0300 Subject: [PATCH 088/308] [None][perf] Add AutoDeploy NVFP4 RMSNorm quant fusion (#14361) Signed-off-by: Tal Cherckez <127761168+tcherckez-nvidia@users.noreply.github.com> Signed-off-by: Tal Cherckez --- .../_torch/auto_deploy/config/default.yaml | 3 + .../custom_ops/distributed/trtllm_dist.py | 94 +++ .../custom_ops/normalization/rms_norm.py | 169 +++++ .../library/fuse_rmsnorm_quant_fp8.py | 30 +- .../library/fuse_rmsnorm_quant_nvfp4.py | 611 ++++++++++++++++++ .../auto_deploy/utils/multi_stream_utils.py | 23 +- .../_torch/auto_deploy/utils/node_utils.py | 85 ++- .../custom_ops/test_multi_stream_moe.py | 57 ++ .../library/test_quant_fusion.py | 482 +++++++++++++- 9 files changed, 1513 insertions(+), 41 deletions(-) create mode 100644 tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_nvfp4.py diff --git a/tensorrt_llm/_torch/auto_deploy/config/default.yaml b/tensorrt_llm/_torch/auto_deploy/config/default.yaml index 337a9f4ef85f..831a9cc1e74e 100644 --- a/tensorrt_llm/_torch/auto_deploy/config/default.yaml +++ b/tensorrt_llm/_torch/auto_deploy/config/default.yaml @@ -217,6 +217,9 @@ transforms: rmsnorm_backend: flashinfer gated_rmsnorm_backend: triton requires_shape_prop: true + fuse_rmsnorm_quant_nvfp4: + stage: post_load_fusion + enabled: true fuse_gdn_gating: stage: post_load_fusion fuse_l2norm: diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/distributed/trtllm_dist.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/distributed/trtllm_dist.py index 136cebaf7c8f..a3573fca1281 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/distributed/trtllm_dist.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/distributed/trtllm_dist.py @@ -23,6 +23,7 @@ import torch +import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils from tensorrt_llm._torch.distributed import AllReduce, allgather from tensorrt_llm._torch.distributed.symm_mem_allgather import SymmetricMemoryAllGather from tensorrt_llm._torch.modules.linear import AllReduceFusionOp, AllReduceParams, AllReduceStrategy @@ -200,6 +201,99 @@ def trtllm_fused_allreduce_residual_rmsnorm_fake( return torch.empty_like(tensor), torch.empty_like(tensor) +@torch.library.custom_op( + "dist::trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4", + mutates_args=(), + device_types="cuda", +) +def trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4( + tensor: torch.Tensor, + residual: torch.Tensor, + norm_weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + strategy: str, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused allreduce + residual + RMSNorm + NVFP4 quantization.""" + all_reduce_params = AllReduceParams( + fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM_QUANT_NVFP4, + bias=None, + residual=residual, + norm_weight=norm_weight, + scale=scale, + eps=eps, + ) + quant_fp4, scale_factor, residual_out = trtllm_allreduce( + tensor, ReduceOp.SUM, strategy=strategy, all_reduce_params=all_reduce_params + ) + return quant_fp4, scale_factor, residual_out + + +@trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4.register_fake +def trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4_fake( + tensor: torch.Tensor, + residual: torch.Tensor, + norm_weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + strategy: str, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del norm_weight, scale, eps, strategy + fp4_shape, scale_shape = fp4_utils.get_fp4_shape(tensor.shape, 16) + return ( + tensor.new_empty(fp4_shape, dtype=torch.uint8), + tensor.new_empty((scale_shape,), dtype=torch.uint8), + torch.empty_like(residual), + ) + + +@torch.library.custom_op( + "dist::trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4", + mutates_args=(), + device_types="cuda", +) +def trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4( + tensor: torch.Tensor, + residual: torch.Tensor, + norm_weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + strategy: str, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused allreduce + residual + RMSNorm with both BF16 and NVFP4 norm outputs.""" + all_reduce_params = AllReduceParams( + fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM_OUT_QUANT_NVFP4, + bias=None, + residual=residual, + norm_weight=norm_weight, + scale=scale, + eps=eps, + ) + norm_out, quant_fp4, scale_factor, residual_out = trtllm_allreduce( + tensor, ReduceOp.SUM, strategy=strategy, all_reduce_params=all_reduce_params + ) + return norm_out, quant_fp4, scale_factor, residual_out + + +@trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4.register_fake +def trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4_fake( + tensor: torch.Tensor, + residual: torch.Tensor, + norm_weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + strategy: str, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + del norm_weight, scale, eps, strategy + fp4_shape, scale_shape = fp4_utils.get_fp4_shape(tensor.shape, 16) + return ( + torch.empty_like(tensor), + tensor.new_empty(fp4_shape, dtype=torch.uint8), + tensor.new_empty((scale_shape,), dtype=torch.uint8), + torch.empty_like(residual), + ) + + def is_trtllm_op_available(): """Check if TRT-LLM ops are available and running with MPI.""" return is_ompi() diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py index 6c750c46eee0..2d042705423f 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py @@ -21,6 +21,10 @@ import torch.nn.functional as F from einops import rearrange +import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils + +from ..quantization.quant import TRTLLM_NVFP4_SCALING_VECTOR_SIZE + try: from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl except (ModuleNotFoundError, ImportError): @@ -37,6 +41,11 @@ def get_env_enable_pdl() -> bool: from .triton_rms_norm import rms_norm +def _get_nvfp4_fake_shapes(x: torch.Tensor) -> tuple[tuple[int, ...], int]: + output_shape, sf_size = fp4_utils.get_fp4_shape(x.shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + return tuple(output_shape), sf_size + + @torch.library.custom_op("auto_deploy::flashinfer_rms_norm", mutates_args=()) def flashinfer_rmsnorm(input: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: """Custom operator for FlashInfer RMSNorm implementation. @@ -259,6 +268,166 @@ def _triton_rmsnorm_gated_meta( return x.new_empty(x.shape, dtype=x.dtype) +@torch.library.custom_op("auto_deploy::trtllm_fused_gated_rmsnorm_quant_nvfp4", mutates_args=()) +def trtllm_fused_gated_rmsnorm_quant_nvfp4( + x: torch.Tensor, + gate: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fuse gated RMSNorm and NVFP4 quantization using the TRT-LLM Torch kernel.""" + if weight.dtype in (torch.float16, torch.bfloat16): + kernel_dtype = weight.dtype + elif gate.dtype in (torch.float16, torch.bfloat16): + kernel_dtype = gate.dtype + else: + kernel_dtype = x.dtype + + if x.dtype != kernel_dtype: + x = x.to(kernel_dtype) + if gate.dtype != kernel_dtype: + gate = gate.to(kernel_dtype) + if weight.dtype != kernel_dtype: + weight = weight.to(kernel_dtype) + + x_shape = x.shape + hidden_size = x_shape[-1] + x_2d = x.reshape(-1, hidden_size) + if x_2d.stride(-1) != 1: + x_2d = x_2d.contiguous() + + gate_2d = gate.reshape(-1, hidden_size) + if gate_2d.stride(-1) != 1: + gate_2d = gate_2d.contiguous() + + fp4_i32, scale_factors = torch.ops.trtllm.fused_gated_rmsnorm_quant( + x_2d, gate_2d, weight.contiguous(), group_size, eps, scale.contiguous() + ) + fp4_u8 = fp4_i32.view(torch.uint8) + return fp4_u8.reshape(*x_shape[:-1], hidden_size // 2), scale_factors + + +@trtllm_fused_gated_rmsnorm_quant_nvfp4.register_fake +def _trtllm_fused_gated_rmsnorm_quant_nvfp4_fake( + x: torch.Tensor, + gate: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + del gate, weight, scale, eps, group_size + output_shape, sf_size = _get_nvfp4_fake_shapes(x) + return x.new_empty(output_shape, dtype=torch.uint8), x.new_empty((sf_size,), dtype=torch.uint8) + + +def _run_trtllm_fused_add_rmsnorm_quant_nvfp4( + x: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, + output_hp_norm: bool, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: + x_shape = x.shape + hidden_size = x_shape[-1] + x_2d = x.reshape(-1, hidden_size) + if x_2d.stride(-1) != 1: + x_2d = x_2d.contiguous() + + residual_2d = residual.reshape(-1, hidden_size) + if residual_2d.dtype != x_2d.dtype: + residual_2d = residual_2d.to(x_2d.dtype) + if residual_2d.stride(-1) != 1: + residual_2d = residual_2d.contiguous() + + if weight.dtype != x_2d.dtype: + weight = weight.to(x_2d.dtype) + + fp4_i32, residual_out, scale_factors, norm_out = torch.ops.trtllm.fused_add_rms_norm_quant( + x_2d, + residual_2d, + weight.contiguous(), + scale.contiguous(), + True, + eps, + output_hp_norm, + ) + fp4_u8 = fp4_i32.view(torch.uint8).reshape(*x_shape[:-1], hidden_size // 2) + residual_out = residual_out.reshape(x_shape) + if norm_out is not None: + norm_out = norm_out.reshape(x_shape) + return fp4_u8, residual_out, scale_factors, norm_out + + +@torch.library.custom_op("auto_deploy::trtllm_fused_add_rmsnorm_quant_nvfp4", mutates_args=()) +def trtllm_fused_add_rmsnorm_quant_nvfp4( + x: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Fuse residual add, RMSNorm, and NVFP4 quantization using a TRT-LLM kernel.""" + fp4_out, residual_out, scale_factors, _ = _run_trtllm_fused_add_rmsnorm_quant_nvfp4( + x, residual, weight, scale, eps, False + ) + return fp4_out, residual_out, scale_factors + + +@trtllm_fused_add_rmsnorm_quant_nvfp4.register_fake +def _trtllm_fused_add_rmsnorm_quant_nvfp4_fake( + x: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del residual, weight, scale, eps + output_shape, sf_size = _get_nvfp4_fake_shapes(x) + return ( + x.new_empty(output_shape, dtype=torch.uint8), + torch.empty_like(x), + x.new_empty((sf_size,), dtype=torch.uint8), + ) + + +@torch.library.custom_op("auto_deploy::trtllm_fused_add_rmsnorm_out_quant_nvfp4", mutates_args=()) +def trtllm_fused_add_rmsnorm_out_quant_nvfp4( + x: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fuse residual add, RMSNorm, and NVFP4 quantization while keeping BF16 norm output.""" + fp4_out, residual_out, scale_factors, norm_out = _run_trtllm_fused_add_rmsnorm_quant_nvfp4( + x, residual, weight, scale, eps, True + ) + assert norm_out is not None + return norm_out, fp4_out, residual_out, scale_factors + + +@trtllm_fused_add_rmsnorm_out_quant_nvfp4.register_fake +def _trtllm_fused_add_rmsnorm_out_quant_nvfp4_fake( + x: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + del residual, weight, scale, eps + output_shape, sf_size = _get_nvfp4_fake_shapes(x) + return ( + torch.empty_like(x), + x.new_empty(output_shape, dtype=torch.uint8), + torch.empty_like(x), + x.new_empty((sf_size,), dtype=torch.uint8), + ) + + # Forked from: # https://github.com/state-spaces/mamba/blob/6b32be06d026e170b3fdaf3ae6282c5a6ff57b06/mamba_ssm/ops/triton/layernorm_gated.py # NOTES: diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_fp8.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_fp8.py index 435e80579da0..7f0a32b811dc 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_fp8.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_fp8.py @@ -29,9 +29,9 @@ collect_terminal_users_through_passthrough, extract_op_args, extract_output_tuple, - is_any_view_op, is_op, is_trivial_passthrough_user, + unwrap_input_through_passthrough, ) from ..interface import ( BaseTransform, @@ -91,26 +91,24 @@ def _collect_grouped_fp8_linear_users( return grouped_users -def _is_view_like(node: Node) -> bool: - return is_any_view_op(node) - - def _unwrap_post_norm_nodes(node: Node) -> Tuple[Node, list[Node]]: - current = node - post_nodes: list[Node] = [] - while isinstance(current, Node) and _is_view_like(current): - post_nodes.append(current) - current = current.args[0] - return current, post_nodes + return unwrap_input_through_passthrough(node) def _reapply_post_norm_nodes(graph, current: Node, post_nodes: list[Node]) -> Node: for post_node in reversed(post_nodes): - current = graph.call_function( - post_node.target, - args=(current, *post_node.args[1:]), - kwargs=post_node.kwargs, - ) + if post_node.op == "call_method": + current = graph.call_method( + post_node.target, + args=(current, *post_node.args[1:]), + kwargs=post_node.kwargs, + ) + else: + current = graph.call_function( + post_node.target, + args=(current, *post_node.args[1:]), + kwargs=post_node.kwargs, + ) current.meta.update(post_node.meta) return current diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_nvfp4.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_nvfp4.py new file mode 100644 index 000000000000..2858adf90191 --- /dev/null +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rmsnorm_quant_nvfp4.py @@ -0,0 +1,611 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fuse Torch-backend NVFP4 norm-quant kernels into AutoDeploy graphs.""" + +import operator +from typing import List, Tuple, Type + +import torch +from torch.fx import GraphModule, Node + +import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils + +from ...custom_ops.quantization.quant import TRTLLM_NVFP4_SCALING_VECTOR_SIZE +from ...models.factory import ModelFactory +from ...shim.interface import CachedSequenceInterface +from ...utils._graph import eliminate_dead_code +from ...utils.logger import ad_logger +from ...utils.node_utils import ( + collect_terminal_users_through_passthrough, + extract_op_args, + extract_output_tuple, + is_dtype_cast_op, + is_op, + unwrap_input_through_passthrough, +) +from ..interface import ( + BaseTransform, + SharedConfig, + TransformConfig, + TransformInfo, + TransformRegistry, +) + + +def _is_supported_nvfp4_linear(node: Node) -> bool: + return is_op(node, torch.ops.auto_deploy.torch_quant_nvfp4_linear) + + +def _extract_nvfp4_linear_args(node: Node): + return extract_op_args( + node, "input", "weight_fp4", "bias", "input_scale", "weight_scale", "alpha" + ) + + +def _same_scale_node(lhs: Node, rhs: Node) -> bool: + if lhs is rhs: + return True + return lhs.op == "get_attr" and rhs.op == "get_attr" and lhs.target == rhs.target + + +def _unwrap_post_norm_nodes(node: Node) -> Tuple[Node, list[Node]]: + return unwrap_input_through_passthrough(node, allow_dtype_cast=True) + + +def _has_unsupported_post_norm_nodes(post_nodes: list[Node]) -> bool: + return any(not is_dtype_cast_op(node) for node in post_nodes) + + +def _extract_dtype_from_meta(node: Node) -> torch.dtype | None: + val = node.meta.get("val") + if hasattr(val, "dtype"): + return val.dtype + + tensor_meta = node.meta.get("tensor_meta") + if hasattr(tensor_meta, "dtype"): + return tensor_meta.dtype + + return None + + +def _get_out_dtype(linear_node: Node, source_node: Node) -> torch.dtype: + input_arg, _, _, _, _, _ = _extract_nvfp4_linear_args(linear_node) + input_dtype = _extract_dtype_from_meta(input_arg) if isinstance(input_arg, Node) else None + return ( + _extract_dtype_from_meta(linear_node) + or input_dtype + or _extract_dtype_from_meta(source_node) + or torch.bfloat16 + ) + + +def _get_last_dim_from_meta(node: Node) -> int | None: + val = node.meta.get("val") + if hasattr(val, "shape") and len(val.shape) > 0: + return int(val.shape[-1]) + + tensor_meta = node.meta.get("tensor_meta") + if hasattr(tensor_meta, "shape") and len(tensor_meta.shape) > 0: + return int(tensor_meta.shape[-1]) + + return None + + +def _get_shape_from_meta(node: Node) -> tuple | None: + val = node.meta.get("val") + if hasattr(val, "shape"): + return tuple(val.shape) + + tensor_meta = node.meta.get("tensor_meta") + if hasattr(tensor_meta, "shape"): + return tuple(tensor_meta.shape) + + return None + + +def _new_empty_from_meta(source_node: Node, shape: tuple, dtype: torch.dtype) -> torch.Tensor: + val = source_node.meta.get("val") + if hasattr(val, "new_empty"): + return val.new_empty(shape, dtype=dtype) + return torch.empty(shape, dtype=dtype, device="meta") + + +def _set_tensor_val_meta( + node: Node, + source_node: Node, + shape: tuple, + dtype: torch.dtype, +) -> None: + node.meta["val"] = _new_empty_from_meta(source_node, shape, dtype) + node.meta.pop("tensor_meta", None) + + +def _set_nvfp4_quant_meta(fp4_node: Node, scale_node: Node, source_node: Node) -> None: + source_shape = _get_shape_from_meta(source_node) + if source_shape is None: + return + + fp4_shape, scale_shape = fp4_utils.get_fp4_shape(source_shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + _set_tensor_val_meta(fp4_node, source_node, tuple(fp4_shape), torch.uint8) + _set_tensor_val_meta(scale_node, source_node, (scale_shape,), torch.uint8) + + +def _supports_trtllm_fused_add_rmsnorm_quant_nvfp4(node: Node) -> bool: + hidden_size = _get_last_dim_from_meta(node) + if hidden_size is None: + return False + return 2048 <= hidden_size <= 16384 and hidden_size % 16 == 0 + + +def _get_arg_defined_before( + graph, + arg: Node, + insertion_node: Node, + node_order: dict[Node, int], +) -> Node | None: + if node_order.get(arg, -1) < node_order.get(insertion_node, float("inf")): + return arg + if arg.op != "get_attr": + return None + + # Model parameters/buffers are order-independent logically, but FX lint requires + # the get_attr node to appear before every consumer. + with graph.inserting_before(insertion_node): + cloned_arg = graph.get_attr(arg.target) + cloned_arg.meta.update(arg.meta) + return cloned_arg + + +def _collect_grouped_nvfp4_linear_users( + source_node: Node, + seed_user: Node, + seed_scale: Node, + processed_users: set[int], +) -> List[Node]: + terminal_users, traversal_ok = collect_terminal_users_through_passthrough( + source_node, allow_dtype_cast=True + ) + if not traversal_ok: + return [] + + grouped_users: List[Node] = [] + for user in terminal_users: + if not _is_supported_nvfp4_linear(user) or id(user) in processed_users: + continue + + input_arg, _, _, input_scale, _, _ = _extract_nvfp4_linear_args(user) + if not isinstance(input_arg, Node) or not isinstance(input_scale, Node): + continue + + user_source, post_nodes = _unwrap_post_norm_nodes(input_arg) + if user_source is not source_node or _has_unsupported_post_norm_nodes(post_nodes): + continue + if not _same_scale_node(seed_scale, input_scale): + continue + + grouped_users.append(user) + + if seed_user not in grouped_users: + grouped_users.append(seed_user) + + return grouped_users + + +def _all_terminal_users_are_grouped(source_node: Node, grouped_users: List[Node]) -> bool: + terminal_users, traversal_ok = collect_terminal_users_through_passthrough( + source_node, allow_dtype_cast=True + ) + if not traversal_ok: + return False + grouped_user_set = set(grouped_users) + return all(user in grouped_user_set for user in terminal_users) + + +def _has_terminal_users_outside_group(source_node: Node, grouped_users: List[Node]) -> bool: + terminal_users, traversal_ok = collect_terminal_users_through_passthrough( + source_node, allow_dtype_cast=True + ) + if not traversal_ok: + return True + grouped_user_set = set(grouped_users) + return any(user not in grouped_user_set for user in terminal_users) + + +def _is_getitem(node: Node, idx: int) -> bool: + return node.op == "call_function" and node.target == operator.getitem and node.args[1] == idx + + +def _extract_nonquant_allreduce_norm(node: Node): + if not _is_getitem(node, 0): + return None + + source_node = node.args[0] + if not isinstance(source_node, Node) or not is_op( + source_node, torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm + ): + return None + + tensor, residual, norm_weight, eps, strategy = source_node.args + _, residual_out = extract_output_tuple(source_node, count=2) + return source_node, tensor, residual, norm_weight, eps, strategy, residual_out + + +def _extract_add_rmsnorm(node: Node): + if not is_op( + node, + [ + torch.ops.auto_deploy.flashinfer_rms_norm, + torch.ops.auto_deploy.torch_rmsnorm, + torch.ops.auto_deploy.triton_rms_norm, + ], + ): + return None + + norm_input, norm_weight, eps = node.args + pre_norm_cast = None + if isinstance(norm_input, Node) and is_dtype_cast_op(norm_input): + pre_norm_cast = norm_input + norm_input = norm_input.args[0] + + if not isinstance(norm_input, Node) or not is_op(norm_input, torch.ops.aten.add.Tensor): + return None + + add_lhs, add_rhs = norm_input.args[:2] + if not isinstance(add_lhs, Node) or not isinstance(add_rhs, Node): + return None + + return norm_input, pre_norm_cast, add_lhs, add_rhs, norm_weight, eps + + +def _extract_gated_rmsnorm(node: Node): + if not is_op( + node, + [ + torch.ops.auto_deploy.torch_rmsnorm_gated, + torch.ops.auto_deploy.triton_rmsnorm_gated, + ], + ): + return None + + x, weight, gate, eps, group_size, norm_before_gate = extract_op_args( + node, "x", "weight", "gate", "eps", "group_size", "norm_before_gate" + ) + if gate is None or norm_before_gate: + return None + return x, weight, gate, eps, group_size + + +def _insert_prequant_linear( + graph, + linear_node: Node, + fp4_input: Node, + scale_factors: Node, + out_dtype: torch.dtype, +) -> Node: + _, weight_fp4, bias, _, weight_scale, alpha = _extract_nvfp4_linear_args(linear_node) + return graph.call_function( + torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear.default, + args=(fp4_input, weight_fp4, scale_factors, weight_scale, alpha), + kwargs={"bias": bias, "out_dtype": out_dtype}, + ) + + +@TransformRegistry.register("fuse_rmsnorm_quant_nvfp4") +class FuseRMSNormQuantNVFP4(BaseTransform): + """Fuse NVFP4 quantization into RMSNorm producers where TRT-LLM kernels exist.""" + + config: TransformConfig + + @classmethod + def get_config_class(cls) -> Type[TransformConfig]: + return TransformConfig + + def _apply( + self, + gm: GraphModule, + cm: CachedSequenceInterface, + factory: ModelFactory, + shared_config: SharedConfig, + ) -> Tuple[GraphModule, TransformInfo]: + graph = gm.graph + cnt = 0 + processed_nvfp4_users: set[int] = set() + original_nodes = list(graph.nodes) + node_order = {n: i for i, n in enumerate(original_nodes)} + + for node in original_nodes: + if not _is_supported_nvfp4_linear(node) or id(node) in processed_nvfp4_users: + continue + + input_arg, _, _, input_scale, _, _ = _extract_nvfp4_linear_args(node) + if not isinstance(input_arg, Node) or not isinstance(input_scale, Node): + continue + + norm_node, post_nodes = _unwrap_post_norm_nodes(input_arg) + if _has_unsupported_post_norm_nodes(post_nodes): + continue + + nvfp4_linear_users = _collect_grouped_nvfp4_linear_users( + norm_node, node, input_scale, processed_nvfp4_users + ) + if not nvfp4_linear_users: + continue + + earliest_user = min(nvfp4_linear_users, key=lambda n: node_order.get(n, float("inf"))) + + num_matches = self._try_fuse_allreduce_rmsnorm_quant( + norm_node, + input_scale, + nvfp4_linear_users, + earliest_user, + node_order, + processed_nvfp4_users, + ) + if num_matches is not None: + cnt += num_matches + continue + + num_matches = self._try_fuse_add_rmsnorm_quant( + norm_node, + input_scale, + nvfp4_linear_users, + node_order, + processed_nvfp4_users, + ) + if num_matches is not None: + cnt += num_matches + continue + + num_matches = self._try_fuse_gated_rmsnorm_quant( + norm_node, + input_scale, + nvfp4_linear_users, + earliest_user, + processed_nvfp4_users, + ) + if num_matches is None: + continue + cnt += num_matches + + if cnt > 0: + eliminate_dead_code(gm) + gm.graph.lint() + gm.recompile() + + info = TransformInfo( + skipped=(cnt == 0), + num_matches=cnt, + is_clean=True, + has_valid_shapes=True, + ) + return gm, info + + def _try_fuse_allreduce_rmsnorm_quant( + self, + norm_node: Node, + input_scale: Node, + nvfp4_linear_users: List[Node], + earliest_user: Node, + node_order: dict[Node, int], + processed_nvfp4_users: set[int], + ) -> int | None: + allreduce_info = _extract_nonquant_allreduce_norm(norm_node) + if allreduce_info is None: + return None + + ( + allreduce_node, + tensor, + residual, + norm_weight, + eps, + strategy, + residual_out_node, + ) = allreduce_info + graph = norm_node.graph + needs_norm_output = _has_terminal_users_outside_group(norm_node, nvfp4_linear_users) + insertion_node = earliest_user + if needs_norm_output: + insertion_node = min( + list(norm_node.users), key=lambda n: node_order.get(n, float("inf")) + ) + + fused_input_scale = _get_arg_defined_before(graph, input_scale, insertion_node, node_order) + if fused_input_scale is None: + return 0 + + with graph.inserting_before(insertion_node): + if needs_norm_output: + fused_quant = graph.call_function( + torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4.default, + args=(tensor, residual, norm_weight, fused_input_scale, eps, strategy), + ) + new_norm_out = graph.call_function(operator.getitem, args=(fused_quant, 0)) + fp4_node = graph.call_function(operator.getitem, args=(fused_quant, 1)) + scale_node = graph.call_function(operator.getitem, args=(fused_quant, 2)) + new_residual_out = graph.call_function(operator.getitem, args=(fused_quant, 3)) + new_norm_out.meta.update(norm_node.meta) + else: + fused_quant = graph.call_function( + torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4.default, + args=(tensor, residual, norm_weight, fused_input_scale, eps, strategy), + ) + fp4_node = graph.call_function(operator.getitem, args=(fused_quant, 0)) + scale_node = graph.call_function(operator.getitem, args=(fused_quant, 1)) + new_residual_out = graph.call_function(operator.getitem, args=(fused_quant, 2)) + if residual_out_node is not None: + new_residual_out.meta.update(residual_out_node.meta) + _set_nvfp4_quant_meta(fp4_node, scale_node, norm_node) + + if needs_norm_output: + norm_node.replace_all_uses_with(new_norm_out) + if residual_out_node is not None: + residual_out_node.replace_all_uses_with(new_residual_out) + + num_matches = self._replace_nvfp4_linears( + nvfp4_linear_users, + fp4_node, + scale_node, + norm_node, + processed_nvfp4_users, + ) + + if residual_out_node is not None and len(residual_out_node.users) == 0: + graph.erase_node(residual_out_node) + if len(norm_node.users) == 0: + graph.erase_node(norm_node) + if len(allreduce_node.users) == 0: + graph.erase_node(allreduce_node) + return num_matches + + def _try_fuse_add_rmsnorm_quant( + self, + norm_node: Node, + input_scale: Node, + nvfp4_linear_users: List[Node], + node_order: dict[Node, int], + processed_nvfp4_users: set[int], + ) -> int | None: + add_norm_info = _extract_add_rmsnorm(norm_node) + if add_norm_info is None: + return None + + if not _supports_trtllm_fused_add_rmsnorm_quant_nvfp4(norm_node): + hidden_size = _get_last_dim_from_meta(norm_node) + ad_logger.debug( + "fuse_rmsnorm_quant_nvfp4: skipping add+norm at " + f"{norm_node.name}, hidden_size={hidden_size} outside supported " + "[2048, 16384] range or not divisible by 16" + ) + return 0 + + add_node, pre_norm_cast, add_lhs, add_rhs, norm_weight, eps = add_norm_info + graph = norm_node.graph + needs_norm_output = _has_terminal_users_outside_group(norm_node, nvfp4_linear_users) + insertion_candidates = list(add_node.users) + list(norm_node.users) + if pre_norm_cast is not None: + insertion_candidates.extend(list(pre_norm_cast.users)) + insertion_node = min( + insertion_candidates, + key=lambda n: node_order.get(n, float("inf")), + ) + fused_input_scale = _get_arg_defined_before(graph, input_scale, insertion_node, node_order) + fused_norm_weight = _get_arg_defined_before(graph, norm_weight, insertion_node, node_order) + if fused_input_scale is None or fused_norm_weight is None: + return 0 + + with graph.inserting_before(insertion_node): + if needs_norm_output: + fused_quant = graph.call_function( + torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_out_quant_nvfp4.default, + args=(add_lhs, add_rhs, fused_norm_weight, fused_input_scale, eps), + ) + new_norm_out = graph.call_function(operator.getitem, args=(fused_quant, 0)) + fp4_node = graph.call_function(operator.getitem, args=(fused_quant, 1)) + new_residual_out = graph.call_function(operator.getitem, args=(fused_quant, 2)) + scale_node = graph.call_function(operator.getitem, args=(fused_quant, 3)) + new_norm_out.meta.update(norm_node.meta) + else: + fused_quant = graph.call_function( + torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_quant_nvfp4.default, + args=(add_lhs, add_rhs, fused_norm_weight, fused_input_scale, eps), + ) + fp4_node = graph.call_function(operator.getitem, args=(fused_quant, 0)) + new_residual_out = graph.call_function(operator.getitem, args=(fused_quant, 1)) + scale_node = graph.call_function(operator.getitem, args=(fused_quant, 2)) + new_residual_out.meta.update(add_node.meta) + _set_nvfp4_quant_meta(fp4_node, scale_node, norm_node) + + if needs_norm_output: + norm_node.replace_all_uses_with(new_norm_out) + + num_matches = self._replace_nvfp4_linears( + nvfp4_linear_users, + fp4_node, + scale_node, + norm_node, + processed_nvfp4_users, + ) + + add_node.replace_all_uses_with(new_residual_out) + + if len(norm_node.users) == 0: + graph.erase_node(norm_node) + if pre_norm_cast is not None and len(pre_norm_cast.users) == 0: + graph.erase_node(pre_norm_cast) + if len(add_node.users) == 0: + graph.erase_node(add_node) + return num_matches + + def _try_fuse_gated_rmsnorm_quant( + self, + norm_node: Node, + input_scale: Node, + nvfp4_linear_users: List[Node], + earliest_user: Node, + processed_nvfp4_users: set[int], + ) -> int | None: + gated_info = _extract_gated_rmsnorm(norm_node) + if gated_info is None: + return None + if not _all_terminal_users_are_grouped(norm_node, nvfp4_linear_users): + return 0 + + x, weight, gate, eps, group_size = gated_info + graph = norm_node.graph + with graph.inserting_before(earliest_user): + fused_quant = graph.call_function( + torch.ops.auto_deploy.trtllm_fused_gated_rmsnorm_quant_nvfp4.default, + args=(x, gate, weight, input_scale, eps, group_size), + ) + fp4_node = graph.call_function(operator.getitem, args=(fused_quant, 0)) + scale_node = graph.call_function(operator.getitem, args=(fused_quant, 1)) + _set_nvfp4_quant_meta(fp4_node, scale_node, norm_node) + + num_matches = self._replace_nvfp4_linears( + nvfp4_linear_users, + fp4_node, + scale_node, + norm_node, + processed_nvfp4_users, + ) + + if len(norm_node.users) == 0: + graph.erase_node(norm_node) + return num_matches + + def _replace_nvfp4_linears( + self, + nvfp4_linear_users: List[Node], + fp4_node: Node, + scale_node: Node, + source_node: Node, + processed_nvfp4_users: set[int], + ) -> int: + cnt = 0 + graph = fp4_node.graph + for nvfp4_user in nvfp4_linear_users: + out_dtype = _get_out_dtype(nvfp4_user, source_node) + with graph.inserting_before(nvfp4_user): + gemm_node = _insert_prequant_linear( + graph, nvfp4_user, fp4_node, scale_node, out_dtype + ) + gemm_node.meta.update(nvfp4_user.meta) + nvfp4_user.replace_all_uses_with(gemm_node) + graph.erase_node(nvfp4_user) + processed_nvfp4_users.add(id(nvfp4_user)) + cnt += 1 + return cnt diff --git a/tensorrt_llm/_torch/auto_deploy/utils/multi_stream_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/multi_stream_utils.py index be55a586ee65..02f32565a6ba 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/multi_stream_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/multi_stream_utils.py @@ -31,12 +31,14 @@ """ from threading import RLock -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, TypeVar import torch from .logger import ad_logger +T = TypeVar("T") + # --------------------------------------------------------------------------- # Runtime enable/disable flag # --------------------------------------------------------------------------- @@ -168,6 +170,19 @@ def wait_event(device: int, stream_name: str) -> None: # --------------------------------------------------------------------------- +def _record_stream_for_tensor_outputs(x: object, stream: torch.cuda.Stream) -> None: + if isinstance(x, torch.Tensor): + x.record_stream(stream) + return + if isinstance(x, (list, tuple)): + for item in x: + _record_stream_for_tensor_outputs(item, stream) + return + if isinstance(x, dict): + for item in x.values(): + _record_stream_for_tensor_outputs(item, stream) + + @torch._dynamo.disable def record_event_passthrough( x: torch.Tensor, @@ -191,10 +206,10 @@ def record_event_passthrough( @torch._dynamo.disable def begin_aux_stream_passthrough( - x: torch.Tensor, + x: T, *, device: int = -1, -) -> torch.Tensor: +) -> T: """Record a CUDA event on the main stream, switch to aux, and wait for it. After this function returns the thread-local current stream is the @@ -226,7 +241,7 @@ def begin_aux_stream_passthrough( # NOTE: skip during CUDA graph capture — passthrough partitions are # reclassified as dynamic and won't be captured anyway. if not torch.cuda.is_current_stream_capturing(): - x.record_stream(aux_stream) + _record_stream_for_tensor_outputs(x, aux_stream) torch.cuda.set_stream(aux_stream) # Make aux wait for the main-stream event before executing any work. aux_stream.wait_event(main_event) diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index 90937d355efd..f89634df51af 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -439,27 +439,48 @@ def is_op(node: Node, ops: Union[OperatorLike, Iterable[OperatorLike]]) -> bool: return is_match -def is_trivial_passthrough_user(node: Node) -> bool: +_VIEW_METHOD_TARGETS = {"view", "reshape", "transpose", "permute", "contiguous"} + + +def is_call_method_view_op(node: Node) -> bool: + """Check whether a node is a method-style tensor view/layout op.""" + return ( + node.op == "call_method" + and node.target in _VIEW_METHOD_TARGETS + and bool(node.args) + and isinstance(node.args[0], Node) + ) + + +def is_view_like_op(node: Node) -> bool: + """Check whether a node is a tensor view/layout passthrough op.""" + if is_call_method_view_op(node): + return True + return ( + is_op(node, torch.ops.aten.view) + or is_op(node, torch.ops.aten.reshape) + or is_op(node, torch.ops.aten.transpose) + or is_op(node, torch.ops.aten.permute) + or is_op(node, torch.ops.aten.contiguous) + or is_op(node, torch.ops.auto_deploy.view) + ) + + +def is_dtype_cast_op(node: Node) -> bool: + """Check whether a node casts only tensor dtype.""" + return is_op(node, torch.ops.aten.to.dtype) + + +def is_trivial_passthrough_user(node: Node, *, allow_dtype_cast: bool = False) -> bool: """Check whether a node is a trivial layout/index passthrough op.""" - if node.op == "call_method": - return node.target in { - "view", - "reshape", - "transpose", - "permute", - "contiguous", - "__getitem__", - } + if allow_dtype_cast and is_dtype_cast_op(node): + return True + if is_view_like_op(node): + return True + if node.op == "call_method" and node.target == "__getitem__": + return True if node.op == "call_function": - if node.target is operator.getitem: - return True - return ( - is_op(node, torch.ops.aten.view) - or is_op(node, torch.ops.aten.reshape) - or is_op(node, torch.ops.aten.transpose) - or is_op(node, torch.ops.aten.permute) - or is_op(node, torch.ops.aten.contiguous) - ) + return node.target is operator.getitem return False @@ -467,6 +488,7 @@ def collect_terminal_users_through_passthrough( source_node: Node, *, max_traversal_nodes: int = 256, + allow_dtype_cast: bool = False, ) -> Tuple[List[Node], bool]: """Collect terminal users while traversing trivial passthrough users. @@ -489,7 +511,7 @@ def collect_terminal_users_through_passthrough( seen.add(user) if len(seen) > max_traversal_nodes: return [], False - if is_trivial_passthrough_user(user): + if is_trivial_passthrough_user(user, allow_dtype_cast=allow_dtype_cast): if user.args and isinstance(user.args[0], Node) and user.args[0] in data_nodes: data_nodes.add(user) stack.extend(list(user.users)) @@ -498,6 +520,29 @@ def collect_terminal_users_through_passthrough( return terminal_users, True +def unwrap_input_through_passthrough( + node: Node, + *, + allow_dtype_cast: bool = False, +) -> Tuple[Node, List[Node]]: + """Walk backward through view-like passthrough ops from a consumer input. + + The returned post_nodes are ordered from consumer input back toward the + source producer. Transforms that can absorb dtype casts may opt in with + allow_dtype_cast=True. + """ + current = node + post_nodes: List[Node] = [] + while isinstance(current, Node) and ( + is_view_like_op(current) or (allow_dtype_cast and is_dtype_cast_op(current)) + ): + if not current.args or not isinstance(current.args[0], Node): + break + post_nodes.append(current) + current = current.args[0] + return current, post_nodes + + def get_shared_input_scale_for_fp8_linears( nodes: Iterable[Node], ) -> Tuple[List[Node], Optional[Node]]: diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/test_multi_stream_moe.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/test_multi_stream_moe.py index 569713af2c4f..41c0214ac977 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/test_multi_stream_moe.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/test_multi_stream_moe.py @@ -111,6 +111,17 @@ def _mock_fused_add_res_norm_fake(shared_out, routed_out, residual, weight, eps) return torch.nn.functional.layer_norm(combined, (combined.shape[-1],), weight=weight, eps=eps) +@torch.library.custom_op("auto_deploy::mock_tuple_fork_moe_test", mutates_args=()) +def mock_tuple_fork(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Mock tuple-producing op used to simulate fused RMSNorm + quant outputs.""" + return x + 1, x + 2 + + +@mock_tuple_fork.register_fake +def _mock_tuple_fork_fake(x): + return torch.empty_like(x), torch.empty_like(x) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -589,6 +600,29 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) +class MockTupleForkNemotronHMoELayer(nn.Module): + """Nemotron-H pattern where shared and routed paths fork from tuple outputs.""" + + def __init__(self, hidden_dim: int, intermediate_dim: int, num_experts: int = 8): + super().__init__() + self.gate = nn.Linear(hidden_dim, num_experts, bias=False) + self.shared_experts = _SimpleMLP(hidden_dim, intermediate_dim) + self.expert_weight = nn.Parameter(torch.randn(hidden_dim, hidden_dim)) + self.layernorm = nn.LayerNorm(hidden_dim) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + shared_input, routed_input = torch.ops.auto_deploy.mock_tuple_fork_moe_test(hidden_states) + logits = self.gate(routed_input) + routing_weights, selected_experts = torch.topk(logits, k=2, dim=-1) + + shared_out = self.shared_experts(shared_input) + moe_out = torch.ops.auto_deploy.mock_fused_moe_moe_test( + routed_input, selected_experts, routing_weights, self.expert_weight + ) + + return self.layernorm(shared_out + moe_out) + + def test_fused_merge_pattern_and_correctness(): """Fused merge node (MLIR-like): pattern + graph + correctness.""" hidden_dim, intermediate_dim = 128, 256 @@ -639,3 +673,26 @@ def test_fused_merge_multi_layer(): assert num == 2, f"Expected 2 replacements, got {num}" _assert_numerical_correctness(gm, model, torch.randn(4, hidden_dim, device="cuda")) + + +def test_tuple_fork_pattern_and_correctness(): + """Tuple fork point: begin_aux must preserve and record all tuple tensors.""" + hidden_dim, intermediate_dim = 128, 256 + cuda_stream_manager.add_device(torch.cuda.current_device()) + + model = MockTupleForkNemotronHMoELayer(hidden_dim, intermediate_dim).eval().to("cuda") + example = torch.randn(4, hidden_dim, device="cuda") + gm = _build_gm(model, example) + + gm, num = _execute_shared_expert_in_aux_stream(gm, _MOE_OPS) + + assert num == 1, f"Expected 1 replacement, got {num}" + _assert_stream_nodes_present(gm) + begin_nodes = [ + n + for n in gm.graph.nodes + if n.op == "call_function" and n.target is begin_aux_stream_passthrough + ] + assert len(begin_nodes) == 1, f"Expected exactly one begin_aux node, got {len(begin_nodes)}" + assert begin_nodes[0].args[0].target is torch.ops.auto_deploy.mock_tuple_fork_moe_test.default + _assert_numerical_correctness(gm, model, torch.randn(4, hidden_dim, device="cuda")) diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py index 3be43e8c70cd..a80c3eaaa386 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py @@ -31,8 +31,15 @@ FuseRMSNormQuantFP8, _get_out_dtype_str, ) +from tensorrt_llm._torch.auto_deploy.transform.library.fuse_rmsnorm_quant_nvfp4 import ( + FuseRMSNormQuantNVFP4, +) from tensorrt_llm._torch.auto_deploy.transform.optimizer import InferenceOptimizer -from tensorrt_llm._torch.auto_deploy.utils.node_utils import is_op +from tensorrt_llm._torch.auto_deploy.utils.node_utils import ( + collect_terminal_users_through_passthrough, + is_op, + unwrap_input_through_passthrough, +) from tensorrt_llm._torch.auto_deploy.utils.quantization_utils import fp4_global_scale, fp8_scale @@ -748,3 +755,476 @@ def test_get_out_dtype_str_returns_none_when_norm_meta_missing(): norm.meta.pop("val", None) assert _get_out_dtype_str(norm) is None + + +def test_passthrough_helpers_handle_method_views_and_optional_dtype_cast(): + graph = torch.fx.Graph() + x = graph.placeholder("x") + view = graph.call_method("reshape", args=(x, (2, 4))) + cast = graph.call_function(torch.ops.aten.to.dtype, args=(view, torch.bfloat16)) + user = graph.call_function(torch.ops.aten.neg.default, args=(cast,)) + graph.output(user) + + source, post_nodes = unwrap_input_through_passthrough(view) + assert source is x + assert post_nodes == [view] + + source, post_nodes = unwrap_input_through_passthrough(cast, allow_dtype_cast=True) + assert source is x + assert post_nodes == [cast, view] + + terminal_users, traversal_ok = collect_terminal_users_through_passthrough(x) + assert traversal_ok + assert terminal_users == [cast] + + terminal_users, traversal_ok = collect_terminal_users_through_passthrough( + x, allow_dtype_cast=True + ) + assert traversal_ok + assert terminal_users == [user] + + +def _make_nvfp4_graph_root(hidden_size=64): + root = nn.Module() + root.register_buffer("norm_weight", torch.empty(hidden_size, dtype=torch.bfloat16)) + root.register_buffer("weight_fp4", torch.empty(32, hidden_size // 2, dtype=torch.uint8)) + root.register_buffer("input_scale", torch.empty(1, dtype=torch.float32)) + root.register_buffer("weight_scale", torch.empty(128, 4, dtype=torch.uint8)) + root.register_buffer("alpha", torch.empty(1, dtype=torch.float32)) + return root + + +def _get_fused_getitem(gm, fused_op, index): + matches = [ + n + for n in gm.graph.nodes + if n.op == "call_function" + and n.target is operator.getitem + and len(n.args) == 2 + and isinstance(n.args[0], torch.fx.Node) + and is_op(n.args[0], fused_op) + and n.args[1] == index + ] + assert len(matches) == 1 + return matches[0] + + +def test_fuse_gated_rmsnorm_quant_nvfp4_rewrites_graph(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + gate = graph.placeholder("gate") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + norm_out = graph.call_function( + torch.ops.auto_deploy.triton_rmsnorm_gated.default, + args=(x, norm_weight, gate, 1e-5, 256, False), + ) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output(gemm_out) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_gated_rmsnorm_quant_nvfp4) + for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.triton_rmsnorm_gated) for n in gm.graph.nodes) + fp4_node = _get_fused_getitem( + gm, torch.ops.auto_deploy.trtllm_fused_gated_rmsnorm_quant_nvfp4, 0 + ) + scale_node = _get_fused_getitem( + gm, torch.ops.auto_deploy.trtllm_fused_gated_rmsnorm_quant_nvfp4, 1 + ) + assert tuple(fp4_node.meta["val"].shape) == (2, 32) + assert fp4_node.meta["val"].dtype == torch.uint8 + assert tuple(scale_node.meta["val"].shape) == (512,) + assert scale_node.meta["val"].dtype == torch.uint8 + + +def test_fuse_gated_rmsnorm_quant_nvfp4_accepts_dtype_cast(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + gate = graph.placeholder("gate") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + norm_out = graph.call_function( + torch.ops.auto_deploy.triton_rmsnorm_gated.default, + args=(x, norm_weight, gate, 1e-5, 256, False), + ) + cast_out = graph.call_function(torch.ops.aten.to.dtype, args=(norm_out, torch.bfloat16)) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(cast_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output(gemm_out) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.float32) + cast_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_gated_rmsnorm_quant_nvfp4) + for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.aten.to.dtype) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + + +def test_fuse_gated_rmsnorm_quant_nvfp4_preserves_mixed_consumer_dtypes(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + gate = graph.placeholder("gate") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + norm_out = graph.call_function( + torch.ops.auto_deploy.triton_rmsnorm_gated.default, + args=(x, norm_weight, gate, 1e-5, 256, False), + ) + direct_gemm = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + cast_out = graph.call_function(torch.ops.aten.to.dtype, args=(norm_out, torch.bfloat16)) + casted_gemm = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(cast_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((direct_gemm, casted_gemm)) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.float32) + cast_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + direct_gemm.meta["val"] = torch.empty((2, 32), dtype=torch.float32) + casted_gemm.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + out_dtypes = { + n.kwargs["out_dtype"] + for n in gm.graph.nodes + if is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) + } + + assert info.num_matches == 2 + assert out_dtypes == {torch.float32, torch.bfloat16} + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + + +def test_fuse_allreduce_rmsnorm_quant_nvfp4_rewrites_graph(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + fused_allreduce = graph.call_function( + torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm.default, + args=(x, residual, norm_weight, 1e-5, "AUTO"), + ) + norm_out = graph.call_function(operator.getitem, args=(fused_allreduce, 0)) + residual_out = graph.call_function(operator.getitem, args=(fused_allreduce, 1)) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, residual_out)) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + residual_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm_quant_nvfp4) + for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert not any( + is_op(n, torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm) for n in gm.graph.nodes + ) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + + +def test_fuse_allreduce_rmsnorm_quant_nvfp4_keeps_norm_for_mixed_consumers(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + fused_allreduce = graph.call_function( + torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm.default, + args=(x, residual, norm_weight, 1e-5, "AUTO"), + ) + norm_out = graph.call_function(operator.getitem, args=(fused_allreduce, 0)) + residual_out = graph.call_function(operator.getitem, args=(fused_allreduce, 1)) + extra_consumer = graph.call_function(torch.ops.aten.add.Tensor, args=(norm_out, 1.0)) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, extra_consumer, residual_out)) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + residual_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + extra_consumer.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4) + for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert any(is_op(n, torch.ops.aten.add.Tensor) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + + +def test_fuse_allreduce_rmsnorm_quant_nvfp4_clones_late_input_scale(): + root = _make_nvfp4_graph_root() + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + + fused_allreduce = graph.call_function( + torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm.default, + args=(x, residual, norm_weight, 1e-5, "AUTO"), + ) + norm_out = graph.call_function(operator.getitem, args=(fused_allreduce, 0)) + residual_out = graph.call_function(operator.getitem, args=(fused_allreduce, 1)) + extra_consumer = graph.call_function(torch.ops.aten.add.Tensor, args=(norm_out, 1.0)) + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, extra_consumer, residual_out)) + norm_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + residual_out.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + extra_consumer.meta["val"] = torch.empty((2, 64), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + gm.graph.lint() + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.dist.trtllm_fused_allreduce_residual_rmsnorm_out_quant_nvfp4) + for n in gm.graph.nodes + ) + + +def test_fuse_add_rmsnorm_quant_nvfp4_rewrites_graph(): + hidden_size = 2048 + root = _make_nvfp4_graph_root(hidden_size) + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + add_out = graph.call_function(torch.ops.aten.add.Tensor, args=(x, residual)) + norm_out = graph.call_function( + torch.ops.auto_deploy.flashinfer_rms_norm.default, + args=(add_out, norm_weight, 1e-5), + ) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, add_out)) + add_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + norm_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_quant_nvfp4) for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.flashinfer_rms_norm) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.aten.add.Tensor) for n in gm.graph.nodes) + + +def test_fuse_add_cast_rmsnorm_quant_nvfp4_rewrites_graph(): + hidden_size = 2048 + root = _make_nvfp4_graph_root(hidden_size) + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + add_out = graph.call_function(torch.ops.aten.add.Tensor, args=(x, residual)) + cast_out = graph.call_function(torch.ops.aten.to.dtype, args=(add_out, torch.bfloat16)) + norm_out = graph.call_function( + torch.ops.auto_deploy.flashinfer_rms_norm.default, + args=(cast_out, norm_weight, 1e-5), + ) + input_scale = graph.get_attr("input_scale") + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, add_out)) + add_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + cast_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + norm_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + gm.graph.lint() + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_quant_nvfp4) for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.flashinfer_rms_norm) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.aten.to.dtype) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.aten.add.Tensor) for n in gm.graph.nodes) + + +def test_fuse_add_cast_rmsnorm_quant_nvfp4_clones_late_norm_weight(): + hidden_size = 2048 + root = _make_nvfp4_graph_root(hidden_size) + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + + add_out = graph.call_function(torch.ops.aten.add.Tensor, args=(x, residual)) + cast_out = graph.call_function(torch.ops.aten.to.dtype, args=(add_out, torch.bfloat16)) + norm_weight = graph.get_attr("norm_weight") + norm_out = graph.call_function( + torch.ops.auto_deploy.flashinfer_rms_norm.default, + args=(cast_out, norm_weight, 1e-5), + ) + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, add_out)) + add_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + cast_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + norm_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + gm.graph.lint() + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_quant_nvfp4) for n in gm.graph.nodes + ) + + +def test_fuse_add_rmsnorm_quant_nvfp4_keeps_norm_for_mixed_consumers(): + hidden_size = 2048 + root = _make_nvfp4_graph_root(hidden_size) + graph = torch.fx.Graph() + x = graph.placeholder("x") + residual = graph.placeholder("residual") + norm_weight = graph.get_attr("norm_weight") + weight_fp4 = graph.get_attr("weight_fp4") + input_scale = graph.get_attr("input_scale") + weight_scale = graph.get_attr("weight_scale") + alpha = graph.get_attr("alpha") + + add_out = graph.call_function(torch.ops.aten.add.Tensor, args=(x, residual)) + norm_out = graph.call_function( + torch.ops.auto_deploy.flashinfer_rms_norm.default, + args=(add_out, norm_weight, 1e-5), + ) + extra_consumer = graph.call_function(torch.ops.aten.mul.Tensor, args=(norm_out, 2.0)) + gemm_out = graph.call_function( + torch.ops.auto_deploy.torch_quant_nvfp4_linear.default, + args=(norm_out, weight_fp4, None, input_scale, weight_scale, alpha), + ) + graph.output((gemm_out, extra_consumer, add_out)) + add_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + norm_out.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + extra_consumer.meta["val"] = torch.empty((2, hidden_size), dtype=torch.bfloat16) + gemm_out.meta["val"] = torch.empty((2, 32), dtype=torch.bfloat16) + + gm = torch.fx.GraphModule(root, graph) + transform = FuseRMSNormQuantNVFP4(TransformConfig(stage="post_load_fusion")) + gm, info = transform._apply(gm, None, None, None) + + assert info.num_matches == 1 + assert any( + is_op(n, torch.ops.auto_deploy.trtllm_fused_add_rmsnorm_out_quant_nvfp4) + for n in gm.graph.nodes + ) + assert any(is_op(n, torch.ops.auto_deploy.trtllm_nvfp4_prequant_linear) for n in gm.graph.nodes) + assert any(is_op(n, torch.ops.aten.mul.Tensor) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.flashinfer_rms_norm) for n in gm.graph.nodes) + assert not any(is_op(n, torch.ops.auto_deploy.torch_quant_nvfp4_linear) for n in gm.graph.nodes) From 13ca44a11765139dc40872e7fd8ff3b83a4be2de Mon Sep 17 00:00:00 2001 From: "Yueh-Ting (eop) Chen" Date: Thu, 28 May 2026 23:07:37 +0800 Subject: [PATCH 089/308] [None][feat] Support Gemma4 multi-head_dim pools and host-side slicing to provide local view to Triton kernels for SWA (#13745) Signed-off-by: Yueh-Ting Chen --- .../batch_manager/kvCacheManager.h | 153 ++++++- .../batch_manager/cacheFormatter.cpp | 5 + .../batch_manager/cacheTransBuffer.cpp | 22 + .../batch_manager/kvCacheManager.cpp | 115 +++-- .../nanobind/batch_manager/kvCacheManager.cpp | 33 +- .../batch_manager/kvCacheManagerTest.cpp | 405 ++++++++++++++++++ .../compile/backends/torch_cudagraph.py | 4 +- .../attention/flashinfer_attention.py | 11 + .../attention/triton_paged_attention.py | 6 + .../custom_ops/attention/trtllm_attention.py | 7 +- .../custom_ops/attention_interface.py | 313 ++++++++++++-- .../models/custom/modeling_gemma4.py | 11 +- .../_torch/auto_deploy/shim/ad_executor.py | 192 ++++++++- .../_torch/auto_deploy/shim/demollm.py | 8 +- .../_torch/auto_deploy/shim/interface.py | 352 ++++++++++++--- .../auto_deploy/transform/library/kvcache.py | 313 +++++++++++++- .../_torch/pyexecutor/mamba_cache_manager.py | 9 +- .../_torch/pyexecutor/resource_manager.py | 369 +++++++++++++--- .../test_triton_paged_attention_swa.py | 360 ++++++++++++++++ .../attention/test_trtllm_attention_op.py | 8 +- .../test_switch_to_generate_inplace.py | 36 +- .../shim/test_ad_executor_swa_eviction.py | 316 ++++++++++++++ .../shim/test_cached_sequence_interface.py | 162 +++++-- .../auto_deploy/singlegpu/shim/test_engine.py | 4 +- .../transformations/library/test_kv_cache.py | 380 +++++++++++++++- .../library/test_kvcache_vswa_metadata.py | 213 +++++++++ 26 files changed, 3498 insertions(+), 309 deletions(-) create mode 100644 tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py create mode 100644 tests/unittest/auto_deploy/singlegpu/shim/test_ad_executor_swa_eviction.py create mode 100644 tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py diff --git a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h index 4200e690755e..514e6f6e9155 100644 --- a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h @@ -115,6 +115,23 @@ std::list> chopVectorIntoBlocks( return blockedVectors; } +//! \brief Configuration of a single KV pool. +//! +//! A pool is uniquely described by its attention @c windowSize, the per-layer +//! @c sizePerHead of the layers it serves, and the KV-cache element @c dtype. +//! A KVCacheManager / BlockManager is constructed from a +//! @c std::vector -- one entry per pool the manager hosts. +//! Multiple entries with the same @c windowSize are legal and reserved for +//! future multi-pool-per-window cases (e.g. mixed head_dim within a single +//! window); call sites that key by window today must be revisited when that +//! lands. +struct PoolConfiguration +{ + SizeType32 windowSize; + SizeType32 sizePerHead; + nvinfer1::DataType dtype; +}; + struct LinearAttentionMetadata { enum LinearCacheType : WindowSizeType @@ -554,7 +571,6 @@ class GenerationRequest , mNumTokens(numTokens) , mBeamWidth(beamWidth) , mKvCacheRetentionConfig(std::move(kvCacheRetentionConfig)) - , mNumFrontBlocksRemoved(0) , mCurrentPrepopulatedPromptLen(std::numeric_limits::max()) { auto const numWindowSizes = windowSizeToMetadata.size(); @@ -563,6 +579,8 @@ class GenerationRequest for (auto const [windowSize, metadata] : windowSizeToMetadata) { mCacheBlockIds[windowSize] = std::vector>(beamWidth); + // Per-window front-eviction counter; SWA windows evict, full-attention windows stay at 0. + mNumFrontBlocksRemovedPerWindow[windowSize] = 0; auto const numPools = metadata.numPools; auto const maxBlocks = metadata.maxBlocksPerSeq; mCacheBlockIndices[windowSize] @@ -598,9 +616,14 @@ class GenerationRequest return mNumTokens; } - [[nodiscard]] SizeType32 getNumFrontBlocksRemoved() const + //! \brief Per-window front-eviction count; full-attention windows always return 0. + [[nodiscard]] SizeType32 getNumFrontBlocksRemoved(SizeType32 windowSize) const { - return mNumFrontBlocksRemoved; + auto it = mNumFrontBlocksRemovedPerWindow.find(windowSize); + TLLM_CHECK_WITH_INFO(it != mNumFrontBlocksRemovedPerWindow.end(), + "GenerationRequest::getNumFrontBlocksRemoved: windowSize=%d not registered for request %lu", windowSize, + static_cast(mRequestId)); + return it->second; } [[nodiscard]] SizeType32 getBeamWidth() const @@ -645,12 +668,18 @@ class GenerationRequest { beamBlockIds.clear(); } - mNumFrontBlocksRemoved = 0; + // Reset only this window's counter, not the others — clearing one pool's blocks + // must not invalidate the eviction state of co-existing pools. + auto it = mNumFrontBlocksRemovedPerWindow.find(windowSize); + if (it != mNumFrontBlocksRemovedPerWindow.end()) + { + it->second = 0; + } } void removeFrontBlock(SizeType32 windowSize) { - ++mNumFrontBlocksRemoved; + ++mNumFrontBlocksRemovedPerWindow.at(windowSize); } void removeLastBlock(SizeType32 windowSize) @@ -708,8 +737,10 @@ class GenerationRequest std::unordered_map mCacheBlockIndices; // The retention priority to assign to decode blocks executor::KvCacheRetentionConfig mKvCacheRetentionConfig; - // Number of front blocks removed from the sequence - SizeType32 mNumFrontBlocksRemoved; + // Number of front blocks evicted from the sequence, tracked per window size. + // Each WindowBlockManager increments only its own entry on detachFrontBlock; full-attn + // windows always stay at 0 since they never trigger SWA eviction. + std::map mNumFrontBlocksRemovedPerWindow; // Set of used blocks by the sequence std::set mUsedBlocks; // Current prepopulated prompt length @@ -968,6 +999,31 @@ class WindowBlockManager return mWindowSize; } + //! \brief Return this manager's KV-cache element data type. + //! \details Each WindowBlockManager carries its own dtype, which lets BlockManager + //! host pools with mixed precisions when constructed with a per-window + //! dtype map. Empty pools or NVFP4-scale pools are routed through the + //! per-pool tensor metadata instead. + [[nodiscard]] nvinfer1::DataType getDataType() const noexcept + { + return mDataType; + } + + //! \brief Return the head_dim shared by all KV pools in this window manager. + //! \details All pools constructed under a WindowBlockManager share a single + //! head_dim (the constructor's @c sizePerHead), so we read it from the + //! first pool. Returns 0 if no pools are present, or if the pool was + //! built with @c sizePerHead<=0 (recurrent-state pools use -1 as a + //! sentinel and do not have a meaningful head dimension). + [[nodiscard]] SizeType32 getSizePerHead() const noexcept + { + if (mPools.empty() || mPools.front().sizePerHead <= 0) + { + return 0; + } + return mPools.front().sizePerHead; + } + [[nodiscard]] std::string const& getLogPrefix() const noexcept { return mLogPrefix; @@ -1377,6 +1433,16 @@ class BlockManager using SizeType32 = tensorrt_llm::runtime::SizeType32; using BaseEvictionPolicy = tensorrt_llm::batch_manager::eviction_policy::BaseEvictionPolicy; + //! \brief Construct a BlockManager. + //! \param sizePerHead Default head size (used for pools whose window_size has no entry in + //! @p poolConfigurations). + //! \param dtype Default KV-cache data type (used for pools whose window_size has no entry in + //! @p poolConfigurations). + //! \param poolConfigurations One PoolConfiguration per pool the manager will host. Each + //! entry pins (windowSize, sizePerHead, dtype) for one pool, letting a single + //! BlockManager host pools with mixed shapes (e.g. Gemma4 SWA head_dim=256 alongside + //! full-attention head_dim=512). Empty vector = uniform @p sizePerHead / @p dtype + //! across all windows. explicit BlockManager(std::vector const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth, @@ -1388,8 +1454,8 @@ class BlockManager std::shared_ptr kvCacheConnectorManager = nullptr, std::optional agentConfig = std::nullopt, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, SizeType32 indexerKCacheIndexHeadDim = 0, - bool indexerKCacheUseFp4 = false, - std::optional linearAttentionMetadata = std::nullopt); + bool indexerKCacheUseFp4 = false, std::optional linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); [[nodiscard]] bool isEnableIndexerKCache() const { @@ -1502,6 +1568,44 @@ class BlockManager return numFreeBlocksPerWindowSize; } + //! \brief Per-pool configuration view across all WindowBlockManagers. + //! \details One PoolConfiguration per WindowBlockManager, exposing (windowSize, + //! sizePerHead, dtype) for the pool. Lets callers (e.g. Python-side tensor + //! reshaping or kernel dispatch) discover per-pool shape information without + //! reaching into KVCacheBlockPool. + [[nodiscard]] std::vector getPoolConfigurations() const + { + std::vector result; + result.reserve(mWindowBlockManagers.size()); + for (auto const& [windowSize, manager] : mWindowBlockManagers) + { + result.push_back(PoolConfiguration{windowSize, manager.getSizePerHead(), manager.getDataType()}); + } + return result; + } + + //! \brief Convenience: window_size -> dataType, derived from getPoolConfigurations(). + //! For one-pool-per-window managers only; multi-pool-per-window will collide. + [[nodiscard]] std::map getDataTypePerWindow() const + { + std::map result; + for (auto const& [windowSize, manager] : mWindowBlockManagers) + { + result[windowSize] = manager.getDataType(); + } + return result; + } + + [[nodiscard]] SizeType32 getSizePerHeadForWindow(SizeType32 windowSize) const + { + return mWindowBlockManagers.at(windowSize).getSizePerHead(); + } + + [[nodiscard]] nvinfer1::DataType getDataTypeForWindow(SizeType32 windowSize) const + { + return mWindowBlockManagers.at(windowSize).getDataType(); + } + [[nodiscard]] SizeType32 getNumFreeBlocks() const { return sumWindows([](WindowBlockManager const& manager) { return manager.getNumFreeBlocks(); }); @@ -2032,9 +2136,9 @@ class BaseKVCacheManager /// of memory requirements. The weighting considers both the window size and the number of /// layers using each window size, as well as the sum of cache sizes per token for each window. /// @param config KV cache configuration parameters - /// @param dtype Data type used for KV cache values + /// @param dtype Default KV-cache data type (used for windows without a matching pool entry) /// @param numKvHeadsPerLayer Number of KV heads for each local layer (caller selects self/cross attention heads) - /// @param sizePerHead Size of each attention head + /// @param sizePerHead Default head size (used for windows without a matching pool entry) /// @param tokensPerBlock Number of tokens per KV cache block /// @param worldConfig World configuration for multi-GPU setups /// @param windowSizeToLayers Map from attention window size to vector of layer indices using that window size @@ -2043,13 +2147,19 @@ class BaseKVCacheManager /// @param extraCostMemory Additional memory cost to account for CacheTransBufferManager::preAllocBufferSize /// @param kvFactor Factor for KV cache size calculation (typically 2 for key+value) /// @param maxBatchSize Maximum batch size + /// @param linearAttentionMetadata Optional metadata for linear / recurrent state pools. + /// @param poolConfigurations One PoolConfiguration per pool the manager will host. Each entry + /// pins (windowSize, sizePerHead, dtype) for one pool, letting a single BlockManager + /// budget pools with mixed shapes (e.g. Gemma4 SWA head_dim=256 + full-attention + /// head_dim=512). Empty vector = uniform @p sizePerHead / @p dtype across all windows. /// @return Map from window size to tuple of (primary blocks, secondary blocks) [[nodiscard]] static BlocksPerWindow calculateMaxNumBlocks(executor::KvCacheConfig const& config, nvinfer1::DataType dtype, std::vector const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, tensorrt_llm::runtime::WorldConfig const& worldConfig, std::map> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, - std::optional const& linearAttentionMetadata = std::nullopt); + std::optional const& linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); /// @brief Calculates the maximum batch size that can fit the kv-cache, given that all sequences in the batch have /// the provided input and output length. @@ -2097,6 +2207,13 @@ class KVCacheManager : public BaseKVCacheManager using CudaStreamPtr = std::shared_ptr; using CacheType = tensorrt_llm::batch_manager::kv_cache_manager::CacheType; + //! \brief Construct a KVCacheManager. + //! \param sizePerHead Default head size used for windows without a matching pool entry. + //! \param dtype Default KV-cache data type used for windows without a matching pool entry. + //! \param poolConfigurations One PoolConfiguration per pool the manager will host. + //! Lets a single manager host mixed-shape pools (e.g. Gemma4 SWA head_dim=256 + //! + full head_dim=512) so the existing block reuse, scheduling, MPI reduction, + //! and disagg transfer machinery applies natively. Empty vector = uniform. KVCacheManager(std::vector const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, std::vector const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, @@ -2108,7 +2225,8 @@ class KVCacheManager : public BaseKVCacheManager std::shared_ptr kvCacheConnectorManager = nullptr, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, SizeType32 indexerKCacheIndexHeadDim = 0, bool indexerKCacheUseFp4 = false, - std::optional linearAttentionMetadata = std::nullopt); + std::optional linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); KVCacheManager(std::vector const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, @@ -2121,7 +2239,8 @@ class KVCacheManager : public BaseKVCacheManager std::shared_ptr kvCacheConnectorManager = nullptr, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, SizeType32 indexerKCacheIndexHeadDim = 0, bool indexerKCacheUseFp4 = false, - std::optional linearAttentionMetadata = std::nullopt); + std::optional linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, @@ -2134,7 +2253,8 @@ class KVCacheManager : public BaseKVCacheManager std::shared_ptr kvCacheConnectorManager = nullptr, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, SizeType32 indexerKCacheIndexHeadDim = 0, bool indexerKCacheUseFp4 = false, - std::optional linearAttentionMetadata = std::nullopt); + std::optional linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, @@ -2143,7 +2263,8 @@ class KVCacheManager : public BaseKVCacheManager CacheType cacheType = CacheType::kSELF, bool enablePartialReuse = true, bool copyOnpartialReuse = true, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, SizeType32 indexerKCacheIndexHeadDim = 0, bool indexerKCacheUseFp4 = false, - std::optional linearAttentionMetadata = std::nullopt); + std::optional linearAttentionMetadata = std::nullopt, + std::vector const& poolConfigurations = {}); ~KVCacheManager() override = default; diff --git a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp index 3daf2735db96..17dd557be1a3 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp @@ -696,6 +696,11 @@ void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& sess { NVTX3_SCOPED_RANGE(formatInputRecvBuffer); + // TODO(disagg-multi-dtype): pool 0's dtype is treated as canonical for the wire + // transport here. Pools with differing dtypes are rejected up-front in + // CacheTransBufferManager's constructor (see cacheTransBuffer.cpp). When + // per-pool dtype dispatch lands, this single dataType variable must be replaced + // with a per-pool lookup keyed by the source pool of each block. auto dataType = mCacheManager->getPrimaryPool(0)->getDataType(); bool layerWise = common::getEnvDisaggLayerwise() && numKvPools == 1; if (layerWise) diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp index 875e3c7e3bee..b6f1e4e8df15 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp @@ -253,6 +253,28 @@ CacheTransBufferManager::CacheTransBufferManager( { // TODO: FP4 dataSize TLLM_CHECK(mCacheManager); + // TODO(disagg-multi-dtype): Per-pool dtype dispatch in formatter / transfer buffer + // not yet implemented. Disagg currently picks pool 0's dtype as the canonical + // transport type (above), so any KV pool with a different dtype would be silently + // miscoerced on the wire. Fail loudly until per-pool dispatch lands. We restrict + // the comparison to KV pools (getNumPools(false, false)) since block-scale and + // indexer-K pools legitimately have their own dtypes and travel through their own + // code paths. + if (!transferIndexerKCache) + { + auto const numKvPools = mCacheManager->getBlockManager().getNumPools( + /*includeBlockScalePools=*/false, /*includeIndexerKCachePools=*/false); + auto const dtype0 = mCacheManager->getPrimaryPool(0)->getDataType(); + for (SizeType32 i = 1; i < numKvPools; ++i) + { + auto const dtypeI = mCacheManager->getPrimaryPool(i)->getDataType(); + TLLM_CHECK_WITH_INFO(dtypeI == dtype0, + "Disaggregated KV cache transfer does not yet support pools with differing dtypes " + "(pool 0 dtype=%d, pool %d dtype=%d). TODO(disagg-multi-dtype): per-pool dtype " + "dispatch in formatter.", + static_cast(dtype0), i, static_cast(dtypeI)); + } + } TLLM_LOG_INFO("CacheTransBufferManager created for KV cache"); } diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp index c129a3646214..03a4908e7c27 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp @@ -582,7 +582,8 @@ BlockManager::BlockManager(std::vector const& numKvHeadsPerLayer, Si std::shared_ptr kvCacheConnectorManager, std::optional agentConfig, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, bool indexerKCacheUseFp4, - std::optional linearAttentionMetadata) + std::optional linearAttentionMetadata, + std::vector const& poolConfigurations) : mNumLayers{static_cast(numKvHeadsPerLayer.size())} , mTokensPerBlock{tokensPerBlock} , mEventManager{std::move(eventManager)} @@ -624,6 +625,18 @@ BlockManager::BlockManager(std::vector const& numKvHeadsPerLayer, Si mIsVariableWindow = numUniqueWindowSizes > 1; mIsVariableGQA = std::unordered_set(numKvHeadsPerLayer.begin(), numKvHeadsPerLayer.end()).size() > 1; + // Build a window -> PoolConfiguration index for fast lookup. Today, one pool per window + // is the only case in use; multi-pool-per-window would replace this with an explicit + // layer->pool mapping from the caller. + std::map poolByWindow; + for (auto const& pc : poolConfigurations) + { + TLLM_CHECK_WITH_INFO(poolByWindow.emplace(pc.windowSize, &pc).second, + "Multiple PoolConfigurations share windowSize=%d. Multi-pool-per-window requires an " + "explicit layer->pool mapping, which is not yet wired through BlockManager.", + pc.windowSize); + } + mLayerToWindowSize.resize(mNumLayers); for (auto const& [windowSize, layersWithWindowSize] : uniqueWindowSizeToLayers) { @@ -658,8 +671,15 @@ BlockManager::BlockManager(std::vector const& numKvHeadsPerLayer, Si } } - mWindowBlockManagers.try_emplace(SizeType32(windowSize), dtype, windowSize, layersWithWindowSize, - numKvHeadsPerLayer, sizePerHead, tokensPerBlock, + // Pull per-pool head_dim and dtype from poolConfigurations when available; fall back to + // the manager-level scalars otherwise. Lets a single BlockManager host pools with + // distinct shapes (Gemma4-style mixed attention types) without giving up the existing + // radix-tree / scheduler / disagg machinery that already iterates pools internally. + auto const poolIt = poolByWindow.find(windowSize); + auto const windowSizePerHead = (poolIt != poolByWindow.end()) ? poolIt->second->sizePerHead : sizePerHead; + auto const windowDtype = (poolIt != poolByWindow.end()) ? poolIt->second->dtype : dtype; + mWindowBlockManagers.try_emplace(SizeType32(windowSize), windowDtype, windowSize, layersWithWindowSize, + numKvHeadsPerLayer, windowSizePerHead, tokensPerBlock, /*isSWA=*/(windowSize < maxSequenceLength) && (windowSize >= 0), allottedPrimaryBlocks, allottedSecondaryBlocks, maxNumSequences, stream, cacheType, secondaryOffloadMinPriority, mEventManager, enablePartialReuse, copyOnPartialReuse, kvCacheConnectorManager, mLookupTree, mLoopbackAgent, @@ -2173,7 +2193,7 @@ void WindowBlockManager::adjustBlocksIfNeeded(GenerationRequest& sequence) { auto const minTokensForBlockDetach = mWindowSize + mTokensPerBlock; while (mIsSWA && // A block only go out-of-window in SWA - (sequence.getNumTokens() - sequence.getNumFrontBlocksRemoved() * getTokensPerBlock() + (sequence.getNumTokens() - sequence.getNumFrontBlocksRemoved(mWindowSize) * getTokensPerBlock() >= minTokensForBlockDetach)) { // Detaching block for SWA is non-trivial due to the radix tree structure. @@ -2874,11 +2894,18 @@ void BlockManager::pinBlocks(GenerationRequest& sequence) void BlockManager::unpinBlocksById(std::vector const& blockIds) { - // Use the first window size if (mWindowBlockManagers.empty()) { return; } + // Block IDs are window-scoped (each WindowBlockManager owns its own + // mAllBlocksById vector), so silently dispatching to the first window + // would corrupt refcounts whenever the caller's IDs belong to a different + // window. Multi-window block-ID-keyed retention pinning needs + // window-qualified handles; until that lands, refuse the multi-window case. + TLLM_CHECK_WITH_INFO(mWindowBlockManagers.size() == 1, + "BlockManager::unpinBlocksById currently only supports single-window managers; " + "multi-window block-ID-keyed retention pinning requires window-qualified handles (TODO)."); auto& firstManager = mWindowBlockManagers.begin()->second; firstManager.unpinBlocksById(blockIds); } @@ -3132,13 +3159,14 @@ KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, Size SizeType32 sinkTokenLength, int64_t stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, bool enablePartialReuse, bool copyOnPartialReuse, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, - bool indexerKCacheUseFp4, std::optional linearAttentionMetadata) + bool indexerKCacheUseFp4, std::optional linearAttentionMetadata, + std::vector const& poolConfigurations) : KVCacheManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, dtype, sinkTokenLength, std::make_shared(reinterpret_cast(stream)), maxSequenceLength, chunkSize, enableBlockReuse, cacheType, std::nullopt, nullptr, enablePartialReuse, copyOnPartialReuse, nullptr, enableIndexerKCache, indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, indexerKCacheUseFp4, - linearAttentionMetadata) + linearAttentionMetadata, poolConfigurations) { } @@ -3150,13 +3178,14 @@ KVCacheManager::KVCacheManager(std::vector const& numKvHeadsPerLayer std::shared_ptr eventManager, bool enablePartialReuse, bool copyOnPartialReuse, std::shared_ptr kvCacheConnectorManager, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, bool indexerKCacheUseFp4, - std::optional linearAttentionMetadata) + std::optional linearAttentionMetadata, + std::vector const& poolConfigurations) : KVCacheManager(numKvHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, dtype, sinkTokenLength, std::make_shared(reinterpret_cast(stream)), maxSequenceLength, chunkSize, enableBlockReuse, cacheType, secondaryOffloadMinPriority, eventManager, enablePartialReuse, copyOnPartialReuse, kvCacheConnectorManager, enableIndexerKCache, indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, - indexerKCacheUseFp4, linearAttentionMetadata) + indexerKCacheUseFp4, linearAttentionMetadata, poolConfigurations) { } @@ -3168,7 +3197,8 @@ KVCacheManager::KVCacheManager(std::vector const& numKvHeadsPerLayer std::shared_ptr eventManager, bool enablePartialReuse, bool copyOnPartialReuse, std::shared_ptr kvCacheConnectorManager, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, bool indexerKCacheUseFp4, - std::optional linearAttentionMetadata) + std::optional linearAttentionMetadata, + std::vector const& poolConfigurations) : mMaxBeamWidth(maxBeamWidth) , mDataType(dtype) , mMaxAttentionWindow(*std::max_element(maxAttentionWindowVec.begin(), maxAttentionWindowVec.end())) @@ -3180,7 +3210,8 @@ KVCacheManager::KVCacheManager(std::vector const& numKvHeadsPerLayer std::move(stream), maxSequenceLength, maxBeamWidth, maxAttentionWindowVec, dtype, mSinkBubbleLength, mChunkSize, cacheType, secondaryOffloadMinPriority, std::move(eventManager), enablePartialReuse, copyOnPartialReuse, std::move(kvCacheConnectorManager), std::nullopt, enableIndexerKCache, - indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, indexerKCacheUseFp4, linearAttentionMetadata) + indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, indexerKCacheUseFp4, linearAttentionMetadata, + poolConfigurations) // disable block reuse for sink bubble since chopVectorIntoBlocks does not match KV cache blocks in this case , mEnableBlockReuse{mSinkBubbleLength > 0 ? false : enableBlockReuse} { @@ -3208,12 +3239,14 @@ KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, Size std::shared_ptr eventManager, bool enablePartialReuse, bool copyOnPartialReuse, std::shared_ptr kvCacheConnectorManager, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, bool indexerKCacheUseFp4, - std::optional linearAttentionMetadata) + std::optional linearAttentionMetadata, + std::vector const& poolConfigurations) : KVCacheManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, dtype, sinkTokenLength, std::move(stream), maxSequenceLength, chunkSize, enableBlockReuse, cacheType, secondaryOffloadMinPriority, std::move(eventManager), enablePartialReuse, copyOnPartialReuse, std::move(kvCacheConnectorManager), enableIndexerKCache, - indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, indexerKCacheUseFp4, linearAttentionMetadata) + indexerKCacheQuantBlockSize, indexerKCacheIndexHeadDim, indexerKCacheUseFp4, linearAttentionMetadata, + poolConfigurations) { } @@ -3225,16 +3258,21 @@ void KVCacheManager::allocatePools(bool useUvm) uint64_t cacheSizeBytes = 0; for (SizeType32 poolIdx = 0; poolIdx < numPools; poolIdx++) { - auto const cacheShape = mBlockManager.getPrimaryPool(poolIdx)->getShape(); + auto const& primaryPool = mBlockManager.getPrimaryPool(poolIdx); + auto const cacheShape = primaryPool->getShape(); auto const cacheVolume = ITensor::volume(cacheShape); + // Query the per-pool dtype from the tensor itself rather than the manager-level + // mDataType. Each pool may carry its own dtype: NVFP4 element / scale pools, or + // a future per-window override map can mix precisions inside a single manager. + auto const poolDataType = primaryPool->getDataType(); #ifdef ENABLE_FP4 - auto const isFp4 = mDataType == nvinfer1::DataType::kFP4; + auto const isFp4 = poolDataType == nvinfer1::DataType::kFP4; #else auto const isFp4 = false; #endif if (!isFp4) { - cacheSizeBytes += cacheVolume * BufferDataType(mDataType).getSize(); + cacheSizeBytes += cacheVolume * BufferDataType(poolDataType).getSize(); } else { @@ -3442,7 +3480,8 @@ SizeType32 KVCacheManager::getRemainingBlocksToCompletion( auto const& seq = seqIt->second; // Subtract detached front blocks (from SWA sliding) which are still // in the cache block ID list but no longer held by the sequence. - numAllocBlocksPerBeam = seq.getCacheBlockIds(windowSize).at(0).size() - seq.getNumFrontBlocksRemoved(); + numAllocBlocksPerBeam + = seq.getCacheBlockIds(windowSize).at(0).size() - seq.getNumFrontBlocksRemoved(windowSize); } } @@ -3599,7 +3638,7 @@ void WindowBlockManager::detachFrontBlock(GenerationRequest& sequence) auto const requestId = sequence.getRequestId(); auto const beamWidth = sequence.getBeamWidth(); auto& allocatedBlocks = mAllocatedBlocksPerSeq.at(requestId); - SizeType32 outOfWindowBlockIdx = sequence.getNumFrontBlocksRemoved(); + SizeType32 outOfWindowBlockIdx = sequence.getNumFrontBlocksRemoved(mWindowSize); for (auto beamIdx = 0; beamIdx < beamWidth; ++beamIdx) { @@ -3642,6 +3681,12 @@ void WindowBlockManager::detachFrontBlock(GenerationRequest& sequence) // Disconnect first block from sequence and remove it from allocated blocks sequence.removeFrontBlock(mWindowSize); + + // Surface the per-window eviction count post-increment so logs show the + // running counter for *this* window (the existing block-id log above only + // identifies which physical block was detached). + TLLM_LOG_DEBUG("%s::detachFrontBlock window=%d count=%d", mLogPrefix.c_str(), mWindowSize, + sequence.getNumFrontBlocksRemoved(mWindowSize)); } PrefixReuseSummary KVCacheManager::analyzePrefixReuse( @@ -4139,7 +4184,8 @@ BlocksPerWindow BaseKVCacheManager::calculateMaxNumBlocks(executor::KvCacheConfi SizeType32 tokensPerBlock, WorldConfig const& worldConfig, std::map> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, - std::optional const& linearAttentionMetadata) + std::optional const& linearAttentionMetadata, + std::vector const& poolConfigurations) { TLLM_LOG_DEBUG("Calculating max num blocks: {.allottedPrimaryMemBytes=%" PRIu64 ", .allottedSecondaryMemBytes=%" PRIu64 "}", @@ -4154,15 +4200,34 @@ BlocksPerWindow BaseKVCacheManager::calculateMaxNumBlocks(executor::KvCacheConfi "configuration you meant to use."); } + // Build a windowSize -> PoolConfiguration lookup so the per-pool loop below stays terse. + // Multi-pool-per-window would replace this with an explicit layer->pool mapping; today + // the 1:1 mapping is the only case in use. + std::map poolByWindow; + for (auto const& pc : poolConfigurations) + { + TLLM_CHECK_WITH_INFO(poolByWindow.emplace(pc.windowSize, &pc).second, + "Multiple PoolConfigurations share windowSize=%d. Multi-pool-per-window requires an " + "explicit layer->pool mapping, which is not yet wired through calculateMaxNumBlocks.", + pc.windowSize); + } + std::map cacheSizeBytesPerTokenPerWindow; for (auto const& [windowSize, managedLayers] : windowSizeToLayers) { - auto const cacheSizePerToken = LinearAttentionMetadata::hasLinearCache(windowSize) - ? 1 - : BaseKVCacheManager::calculateCacheSizePerTokenForSingleWindowSize( - numKvHeadsPerLayer, sizePerHead, managedLayers, kvFactor); - auto const cacheSizeBytesPerToken = cacheSizePerToken * BufferDataType(dtype).getSize(); - cacheSizeBytesPerTokenPerWindow[windowSize] = cacheSizeBytesPerToken; + if (LinearAttentionMetadata::hasLinearCache(windowSize)) + { + cacheSizeBytesPerTokenPerWindow[windowSize] = 1; + continue; + } + // Resolve the pool serving this window: use its (sizePerHead, dtype) when the caller + // supplied a PoolConfiguration; otherwise fall back to the manager-level scalars. + auto const poolIt = poolByWindow.find(windowSize); + auto const windowSizePerHead = (poolIt != poolByWindow.end()) ? poolIt->second->sizePerHead : sizePerHead; + auto const windowDtype = (poolIt != poolByWindow.end()) ? poolIt->second->dtype : dtype; + auto const cacheSizePerToken = BaseKVCacheManager::calculateCacheSizePerTokenForSingleWindowSize( + numKvHeadsPerLayer, windowSizePerHead, managedLayers, kvFactor); + cacheSizeBytesPerTokenPerWindow[windowSize] = cacheSizePerToken * BufferDataType(windowDtype).getSize(); } // When the model is mamba hybrid (i.e. has only 2 windows sizes: max_seq_len and diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp index 325fbbe17a0c..63c07b4f2de2 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp @@ -346,6 +346,14 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::enum_(m, "LinearCacheType") .value("RECURRENT_STATES", tbk::LinearAttentionMetadata::LinearCacheType::kRecurrentStates); + nb::class_(m, "PoolConfiguration") + .def(nb::init<>()) + .def(nb::init(), nb::arg("window_size"), nb::arg("size_per_head"), + nb::arg("dtype")) + .def_rw("window_size", &tbk::PoolConfiguration::windowSize) + .def_rw("size_per_head", &tbk::PoolConfiguration::sizePerHead) + .def_rw("dtype", &tbk::PoolConfiguration::dtype); + nb::class_(m, "PrefixReuseSummary") .def(nb::init<>()) .def_ro("reusable_blocks_allocated", &tbk::PrefixReuseSummary::reusableBlocksAllocated) @@ -415,6 +423,7 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::arg("world_config"), nb::arg("window_size_to_layers"), nb::arg("allotted_primary_mem_bytes"), nb::arg("allotted_secondary_mem_bytes"), nb::arg("extra_cost_memory"), nb::arg("kv_factor"), nb::arg("max_batch_size"), nb::arg("linear_attention_metadata") = std::nullopt, + nb::arg("pool_configurations") = std::vector{}, nb::call_guard()) .def("allocate_pools", &BaseKVCacheManager::allocatePools, nb::call_guard()) .def("release_pools", &BaseKVCacheManager::releasePools, nb::call_guard()) @@ -597,6 +606,18 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) .def("analyze_prefix_reuse", &BaseKVCacheManager::analyzePrefixReuse, nb::arg("unique_tokens"), nb::arg("llm_request"), nb::call_guard()) .def("get_cache_block_ids", &BaseKVCacheManager::getCacheBlockIds, nb::call_guard()) + .def( + "get_num_front_blocks_removed", + [](BaseKVCacheManager const& self, tb::LlmRequest::RequestIdType requestId, SizeType32 windowSize) + { + auto const& seq = self.getSequence(requestId); + // Per-window query. windowSize is required (no aggregation): the Python + // wrapper in resource_manager.py provides a "default to first window" + // convenience layer; the C++/nanobind boundary stays explicit so that + // every caller is forced to think about which pool's counter it wants. + return seq.getNumFrontBlocksRemoved(windowSize); + }, + nb::arg("request_id"), nb::arg("window_size"), nb::call_guard()) .def("get_batch_cache_block_ids", &BaseKVCacheManager::getBatchCacheBlockIds, nb::call_guard()) .def("flush_iteration_events", &BaseKVCacheManager::flushIterationEvents, @@ -631,7 +652,8 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) std::vector const&, nvinfer1::DataType, SizeType32, int64_t, SizeType32, SizeType32, bool, tbk::CacheType, std::optional, std::shared_ptr, bool, bool, std::shared_ptr, - bool, SizeType32, SizeType32, bool, std::optional>(), + bool, SizeType32, SizeType32, bool, std::optional, + std::vector const&>(), nb::arg("num_kv_heads_per_layer"), nb::arg("size_per_head"), nb::arg("tokens_per_block"), nb::arg("blocks_per_window"), nb::arg("max_num_sequences"), nb::arg("max_beam_width"), nb::arg("max_attention_window_vec"), nb::arg("dtype"), nb::arg("sink_token_length"), nb::arg("stream"), @@ -641,7 +663,9 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::arg("copy_on_partial_reuse") = true, nb::arg("kv_connector_manager") = nullptr, nb::arg("enable_indexer_k_cache") = false, nb::arg("indexer_k_cache_quant_block_size") = 128, nb::arg("indexer_k_cache_index_head_dim") = 0, nb::arg("indexer_k_cache_use_fp4") = false, - nb::arg("linear_attention_metadata").none() = std::nullopt, nb::call_guard()) + nb::arg("linear_attention_metadata").none() = std::nullopt, + nb::arg("pool_configurations") = std::vector{}, + nb::call_guard()) .def( "scheduling_has_free_blocks", [](tbk::KVCacheManager& self, SizeType32 numRequired, SizeType32 windowSize) @@ -649,6 +673,11 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::arg("num_required"), nb::arg("window_size"), nb::call_guard()) .def_prop_ro( "is_variable_window", [](tbk::KVCacheManager& self) { return self.getBlockManager().isVariableWindow(); }) + // Per-pool introspection: lets Python discover (windowSize, sizePerHead, dtype) per + // hosted pool so a single KVCacheManager can host mixed-shape pools without a + // Python-side wrapper duplicating the layer->pool routing. + .def_prop_ro("pool_configurations", + [](tbk::KVCacheManager& self) { return self.getBlockManager().getPoolConfigurations(); }) .def("copy_linear_attention_block", &tbk::KVCacheManager::copyLinearAttentionBlock, nb::arg("llm_request"), nb::call_guard()) .def("copy_linear_attention_block_batch", &tbk::KVCacheManager::copyLinearAttentionBlockBatch, diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp index fd6011e807ec..8b15651e2c97 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @@ -16,6 +16,7 @@ */ #include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/cacheTransBuffer.h" #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/batch_manager/kvCacheConnector.h" #include "tensorrt_llm/batch_manager/kvCacheEventManager.h" @@ -9798,3 +9799,407 @@ TEST_F(KVCacheManagerTest, KvCacheConnector_DecodeBlockBoundary_ParityWithBaseli << ", withConnector=" << withConnector[i] << "); see issue #13320"; } } + +// ============================================================================= +// Per-window head_dim / dtype overrides on BlockManager and KVCacheManager, +// plus per-window front-eviction bookkeeping on GenerationRequest. +// ============================================================================= + +TEST_F(KVCacheManagerTest, BlockManagerTestPerWindowFallback) +{ + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr scalarSizePerHead = 64; + auto constexpr tokensPerBlock = 4; + auto constexpr blocksInPrimary = 4; + auto constexpr blocksInSecondary = 0; + auto constexpr maxNumSequences = 4; + auto constexpr maxBeamWidth = 1; + auto constexpr smallWindow = 1024; + auto constexpr largeWindow = 4096; + auto constexpr scalarDtype = nvinfer1::DataType::kHALF; + auto const stream = std::make_shared(); + auto const maxAttentionWindowVec = std::vector{smallWindow, largeWindow}; + auto const blocksPerWindow = BlocksPerWindow{ + {smallWindow, {blocksInPrimary, blocksInSecondary}}, {largeWindow, {blocksInPrimary, blocksInSecondary}}}; + + // No per-pool configurations — expect both windows to fall back to scalars. + BlockManager blockManager(std::vector(numLayers, numKvHeads), scalarSizePerHead, + tokensPerBlock, blocksPerWindow, maxNumSequences, stream, /*maxSequenceLength=*/largeWindow, maxBeamWidth, + maxAttentionWindowVec, scalarDtype, /*sinkBubbleLength=*/0, /*chunkSize=*/0); + blockManager.allocatePools(/*useUvm=*/false); + + auto const pools = blockManager.getPoolConfigurations(); + ASSERT_EQ(pools.size(), 2u); + std::map poolByWindow; + for (auto const& pc : pools) + { + poolByWindow.emplace(pc.windowSize, pc); + } + EXPECT_EQ(poolByWindow.at(smallWindow).sizePerHead, scalarSizePerHead); + EXPECT_EQ(poolByWindow.at(largeWindow).sizePerHead, scalarSizePerHead); + EXPECT_EQ(poolByWindow.at(smallWindow).dtype, scalarDtype); + EXPECT_EQ(poolByWindow.at(largeWindow).dtype, scalarDtype); + EXPECT_EQ(blockManager.getSizePerHeadForWindow(smallWindow), blockManager.getSizePerHeadForWindow(largeWindow)); + EXPECT_EQ(blockManager.getDataTypeForWindow(smallWindow), blockManager.getDataTypeForWindow(largeWindow)); +} + +// Static; does not require a GPU device. +TEST(BaseKVCacheManagerCalculateMaxNumBlocks, PerWindowOverrideDivergesByteBudget) +{ + using SizeType32 = tensorrt_llm::runtime::SizeType32; + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr scalarSizePerHead = 64; + auto constexpr largeSizePerHead = 128; + auto constexpr tokensPerBlock = 64; + auto constexpr smallWindow = 1024; + auto constexpr largeWindow = 4096; + auto constexpr kvFactor = 2; + auto constexpr maxBatchSize = 1; + + auto const numKvHeadsPerLayer = std::vector(numLayers, numKvHeads); + // Layer 0 → smallWindow, layer 1 → largeWindow. + auto const windowSizeToLayers = std::map>{ + {smallWindow, std::vector{0}}, {largeWindow, std::vector{1}}}; + uint64_t const allottedPrimaryMemBytes = static_cast(1) << 30; // 1 GiB + uint64_t const allottedSecondaryMemBytes = static_cast(1) << 30; + size_t const extraCostMemory = 0; + auto const dtype = nvinfer1::DataType::kHALF; + tensorrt_llm::executor::KvCacheConfig const config{}; + tensorrt_llm::runtime::WorldConfig const worldConfig{}; + + auto const baseline = BaseKVCacheManager::calculateMaxNumBlocks(config, dtype, numKvHeadsPerLayer, + scalarSizePerHead, tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, + allottedSecondaryMemBytes, extraCostMemory, kvFactor, maxBatchSize); + + auto const poolConfigurations = std::vector{ + {smallWindow, scalarSizePerHead, dtype}, {largeWindow, largeSizePerHead, dtype}}; + auto const overridden + = BaseKVCacheManager::calculateMaxNumBlocks(config, dtype, numKvHeadsPerLayer, scalarSizePerHead, + tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, allottedSecondaryMemBytes, + extraCostMemory, kvFactor, maxBatchSize, /*linearAttentionMetadata=*/std::nullopt, poolConfigurations); + + ASSERT_NE(baseline.find(smallWindow), baseline.end()); + ASSERT_NE(baseline.find(largeWindow), baseline.end()); + ASSERT_NE(overridden.find(smallWindow), overridden.end()); + ASSERT_NE(overridden.find(largeWindow), overridden.end()); + + auto const& [baselineSmallPrim, baselineSmallSec] = baseline.at(smallWindow); + auto const& [baselineLargePrim, baselineLargeSec] = baseline.at(largeWindow); + auto const& [overSmallPrim, overSmallSec] = overridden.at(smallWindow); + auto const& [overLargePrim, overLargeSec] = overridden.at(largeWindow); + + // Sanity: every count is positive. + EXPECT_GT(baselineSmallPrim, 0); + EXPECT_GT(baselineLargePrim, 0); + EXPECT_GT(overSmallPrim, 0); + EXPECT_GT(overLargePrim, 0); + + // Default share allocation gives equal share (1/N) to each window, so baseline gives the + // same number of blocks per window when both windows have identical bytes/token. + EXPECT_EQ(baselineSmallPrim, baselineLargePrim); + + // Override only doubles bytes/token for the *large* window. Equal share is unchanged, + // so the small window's block count stays the same and the large window's halves. + EXPECT_EQ(overSmallPrim, baselineSmallPrim); + EXPECT_EQ(overSmallSec, baselineSmallSec); + EXPECT_LT(overLargePrim, baselineLargePrim); + EXPECT_LT(overLargeSec, baselineLargeSec); + // Large window pays double bytes/token under override → roughly half the blocks. + EXPECT_NEAR(static_cast(overLargePrim) / baselineLargePrim, 0.5, 0.01); +} + +TEST_F(KVCacheManagerTest, KVCacheManagerSWAEvictionCountPerWindow) +{ + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr sizePerHead = 64; + auto constexpr tokensPerBlock = 4; + auto constexpr maxNumSequences = 4; + auto constexpr maxBeamWidth = 1; + auto constexpr sinkTokenLength = 0; + auto constexpr dtype = nvinfer1::DataType::kHALF; + auto const stream = std::make_shared(); + auto constexpr maxSequenceLength = 128; + auto constexpr maxNewTokens = 40; + + // Layer 0 lives in the SWA pool; layer 1 lives in the full-attention pool. + auto constexpr swaWindow = 8; + auto constexpr fullWindow = 128; + auto const maxAttentionWindowVec = std::vector{swaWindow, fullWindow}; + + auto constexpr blocksInPrimary = 8; + auto constexpr blocksInSecondary = 0; + auto const blocksPerWindow = BlocksPerWindow{ + {swaWindow, {blocksInPrimary, blocksInSecondary}}, {fullWindow, {blocksInPrimary, blocksInSecondary}}}; + + KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, + maxBeamWidth, maxAttentionWindowVec, dtype, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/0, + /*enableBlockReuse=*/false); + kvCacheManager.allocatePools(/*useUvm=*/false); + + auto constexpr beamWidth = maxBeamWidth; + auto constexpr beamIdx = 0; + tr::SamplingConfig const samplingConfig{beamWidth}; + bool constexpr isStreaming{false}; + TokenIdType constexpr firstToken = 1000; + + int inputLength = 11; // > swaWindow (8); < fullWindow (128) + auto inputTokens = std::make_shared(inputLength); + std::iota(inputTokens->begin(), inputTokens->end(), firstToken); + auto llmRequest + = std::make_shared(/*requestId=*/0, maxNewTokens, inputTokens, samplingConfig, isStreaming); + + kvCacheManager.addSequenceBatch({{{/*requestId=*/0, inputLength, beamWidth}}}, {std::ref(*llmRequest)}); + GenerationRequest const& seq = kvCacheManager.getSequence(/*requestId=*/0); + + // No tokens added past the SWA window yet — both counters are zero. + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), 0); + + // Add one token past the window. SWA pool detaches block 0; full pool does not. + llmRequest->addNewToken(firstToken + inputLength, beamIdx); + kvCacheManager.addToken(/*requestId=*/0); + + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow), 1) << "SWA pool should have evicted exactly one front block"; + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 0) << "Full-attention pool must never front-evict"; + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), 1) + << "Aggregate sum must equal the SWA-only count"; + + // Drive a second SWA eviction; full-attn count stays at zero. + llmRequest->addNewToken(firstToken + inputLength + 1, beamIdx); + kvCacheManager.addToken(/*requestId=*/0); + llmRequest->addNewToken(firstToken + inputLength + 2, beamIdx); + kvCacheManager.addToken(/*requestId=*/0); + llmRequest->addNewToken(firstToken + inputLength + 3, beamIdx); + kvCacheManager.addToken(/*requestId=*/0); + llmRequest->addNewToken(firstToken + inputLength + 4, beamIdx); + kvCacheManager.addToken(/*requestId=*/0); + + auto const swaEvictedAfter = seq.getNumFrontBlocksRemoved(swaWindow); + EXPECT_GT(swaEvictedAfter, 1) << "Continued generation should keep evicting SWA blocks"; + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), swaEvictedAfter); + + tensorrt_llm::testing::KvCacheManagerTestUtil::simulatePrefillCompletion(*llmRequest); + EXPECT_NO_THROW(static_cast(kvCacheManager.removeSequence(/*requestId=*/0, llmRequest))); +} + +TEST_F(KVCacheManagerTest, GenerationRequestClearCacheBlocksPerWindowResetsOnlyThatWindow) +{ + // Use a real BlockManager solely to obtain a valid windowSizeToMetadata map; the + // bookkeeping under test (removeFrontBlock / clearCacheBlocks / getNumFrontBlocksRemoved) + // lives entirely on GenerationRequest. + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr sizePerHead = 64; + auto constexpr tokensPerBlock = 4; + auto constexpr blocksInPrimary = 4; + auto constexpr blocksInSecondary = 0; + auto constexpr maxNumSequences = 4; + auto constexpr maxBeamWidth = 1; + auto constexpr swaWindow = 8; + auto constexpr fullWindow = 128; + auto const stream = std::make_shared(); + auto const maxAttentionWindowVec = std::vector{swaWindow, fullWindow}; + auto const blocksPerWindow = BlocksPerWindow{ + {swaWindow, {blocksInPrimary, blocksInSecondary}}, {fullWindow, {blocksInPrimary, blocksInSecondary}}}; + + BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, + blocksPerWindow, maxNumSequences, stream, /*maxSequenceLength=*/fullWindow, maxBeamWidth, maxAttentionWindowVec, + nvinfer1::DataType::kHALF, /*sinkBubbleLength=*/0, /*chunkSize=*/0); + blockManager.allocatePools(/*useUvm=*/false); + + auto constexpr requestId = 7; + auto constexpr numTokens = 1; + GenerationRequest seq{requestId, numTokens, maxBeamWidth, blockManager.getWindowSizesMetadata()}; + + // Drive each window's counter to a distinct non-zero value. + seq.removeFrontBlock(swaWindow); + seq.removeFrontBlock(swaWindow); + seq.removeFrontBlock(fullWindow); + + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow), 2); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 1); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), 3); + + // Clearing one window's blocks must not touch the other window's counter. + seq.clearCacheBlocks(swaWindow); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 1); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), 1); + + // Symmetrically: clearing the other window resets only that one. + seq.clearCacheBlocks(fullWindow); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(fullWindow), 0); + EXPECT_EQ(seq.getNumFrontBlocksRemoved(swaWindow) + seq.getNumFrontBlocksRemoved(fullWindow), 0); +} + +// A5: Smoke-test VSWA + radix prefix reuse with mixed head_dim. +// +// Coverage: constructs a KVCacheManager whose two windows have different head_dim +// (poolConfigurations supplying smallWindow head_dim=256, largeWindow head_dim=512) with reuse + +// partial reuse enabled, runs addSequenceBatch / storeContextBlocks / removeSequence +// for two sequences sharing a prefix, and asserts pool routing is not cross- +// contaminated and refcounts are balanced. This is the smoke variant called for in +// the spec: a complete prefix-reuse hit-rate scenario across mixed head_dim is more +// brittle to set up and is left as a follow-up. +TEST_F(KVCacheManagerTest, VswaMixedHeadDimReuseSmoke) +{ + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr scalarSizePerHead = 64; + auto constexpr tokensPerBlock = 4; + auto constexpr blocksInPrimary = 16; + auto constexpr blocksInSecondary = 0; + auto constexpr maxNumSequences = 4; + auto constexpr maxBeamWidth = 1; + auto constexpr smallWindow = 32; + auto constexpr largeWindow = 64; + auto constexpr smallSizePerHead = 256; + auto constexpr largeSizePerHead = 512; + auto constexpr sinkTokenLength = 0; + auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr maxSequenceLength = 64; + auto constexpr maxNewTokens = 0; + auto const stream = std::make_shared(); + auto const maxAttentionWindowVec = std::vector{smallWindow, largeWindow}; + auto const blocksPerWindow = BlocksPerWindow{ + {smallWindow, {blocksInPrimary, blocksInSecondary}}, {largeWindow, {blocksInPrimary, blocksInSecondary}}}; + auto const poolConfigurations = std::vector{ + {smallWindow, smallSizePerHead, dtype}, {largeWindow, largeSizePerHead, dtype}}; + + KVCacheManager kvCacheManager(numLayers, numKvHeads, scalarSizePerHead, tokensPerBlock, blocksPerWindow, + maxNumSequences, maxBeamWidth, maxAttentionWindowVec, dtype, sinkTokenLength, stream, maxSequenceLength, + /*chunkSize=*/0, /*enableBlockReuse=*/true, CacheType::kSELF, + /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, + /*enablePartialReuse=*/true, /*copyOnpartialReuse=*/true, + /*kvCacheConnectorManager=*/nullptr, + /*enableIndexerKCache=*/false, /*indexerKCacheQuantBlockSize=*/128, /*indexerKCacheIndexHeadDim=*/0, + /*indexerKCacheUseFp4=*/false, + /*linearAttentionMetadata=*/std::nullopt, poolConfigurations); + kvCacheManager.allocatePools(/*useUvm=*/false); + + // Both windows registered with their distinct head_dims. + EXPECT_EQ(kvCacheManager.getBlockManager().getSizePerHeadForWindow(smallWindow), smallSizePerHead); + EXPECT_EQ(kvCacheManager.getBlockManager().getSizePerHeadForWindow(largeWindow), largeSizePerHead); + + auto const freeBefore = kvCacheManager.getBlockManager().getNumFreeBlocksPerWindowSize(); + + auto constexpr beamWidth = maxBeamWidth; + tr::SamplingConfig const samplingConfig{beamWidth}; + bool constexpr isStreaming{false}; + + // Sequence A: 12-token prompt across two blocks (block-aligned at 8 tokens, partial third). + auto inputTokensA = std::make_shared(VecTokens{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); + auto const inputLengthA = static_cast(inputTokensA->size()); + auto llmRequestA + = std::make_shared(/*requestId=*/0, maxNewTokens, inputTokensA, samplingConfig, isStreaming); + + EXPECT_NO_THROW( + kvCacheManager.addSequenceBatch({{{/*requestId=*/0, inputLengthA, beamWidth}}}, {std::ref(*llmRequestA)})); + + // Both windows must have allocated some blocks for this sequence — neither pool + // can be empty (i.e. neither pool was skipped due to mixed head_dim routing). + auto const& seqA = kvCacheManager.getSequence(/*requestId=*/0); + EXPECT_FALSE(seqA.getCacheBlockIds(smallWindow).at(0).empty()) << "small-window pool must allocate"; + EXPECT_FALSE(seqA.getCacheBlockIds(largeWindow).at(0).empty()) << "large-window pool must allocate"; + + tensorrt_llm::testing::KvCacheManagerTestUtil::simulatePrefillCompletion(*llmRequestA); + EXPECT_NO_THROW(static_cast(kvCacheManager.removeSequence(/*requestId=*/0, llmRequestA))); + + // Sequence B: shares an 8-token prefix with sequence A, then diverges. + auto inputTokensB = std::make_shared(VecTokens{1, 2, 3, 4, 5, 6, 7, 8, 99, 100}); + auto const inputLengthB = static_cast(inputTokensB->size()); + auto llmRequestB + = std::make_shared(/*requestId=*/1, maxNewTokens, inputTokensB, samplingConfig, isStreaming); + + EXPECT_NO_THROW( + kvCacheManager.addSequenceBatch({{{/*requestId=*/1, inputLengthB, beamWidth}}}, {std::ref(*llmRequestB)})); + + auto const& seqB = kvCacheManager.getSequence(/*requestId=*/1); + EXPECT_FALSE(seqB.getCacheBlockIds(smallWindow).at(0).empty()); + EXPECT_FALSE(seqB.getCacheBlockIds(largeWindow).at(0).empty()); + + tensorrt_llm::testing::KvCacheManagerTestUtil::simulatePrefillCompletion(*llmRequestB); + EXPECT_NO_THROW(static_cast(kvCacheManager.removeSequence(/*requestId=*/1, llmRequestB))); + + // After both sequences have been removed and their reuse-eligible blocks have + // been returned, the per-window free-block counts must equal what we started + // with (no leaked refcounts in either pool). + auto const freeAfter = kvCacheManager.getBlockManager().getNumFreeBlocksPerWindowSize(); + ASSERT_EQ(freeAfter.size(), freeBefore.size()); + for (auto const& [windowSize, free] : freeBefore) + { + ASSERT_NE(freeAfter.find(windowSize), freeAfter.end()); + EXPECT_EQ(freeAfter.at(windowSize), free) + << "Window " << windowSize << " has leaked or extra blocks after removeSequence."; + } +} + +// A6: VSWA + disagg dtype mismatch must fire the A4 guard. +// +// The constructor of CacheTransBufferManager picks pool 0's dtype as canonical for +// the wire transport. When a KVCacheManager hosts pools with differing dtypes +// (mixed-precision per-window), that silent coercion would corrupt the wire format. +// The guard added in cacheTransBuffer.cpp must throw at construction time. +// +// This test only exercises the helper / construction path that runs the guard; it +// does not stand up a full disaggregated transfer (out of scope at unit-test +// granularity). +TEST_F(KVCacheManagerTest, VswaDisaggDtypeMismatchTriggersGuard) +{ + auto constexpr numLayers = 2; + auto constexpr numKvHeads = 2; + auto constexpr sizePerHead = 64; + auto constexpr tokensPerBlock = 4; + auto constexpr blocksInPrimary = 4; + auto constexpr blocksInSecondary = 0; + auto constexpr maxNumSequences = 4; + auto constexpr maxBeamWidth = 1; + auto constexpr smallWindow = 32; + auto constexpr largeWindow = 64; + auto constexpr sinkTokenLength = 0; + auto constexpr maxSequenceLength = 64; + auto const stream = std::make_shared(); + auto const maxAttentionWindowVec = std::vector{smallWindow, largeWindow}; + auto const blocksPerWindow = BlocksPerWindow{ + {smallWindow, {blocksInPrimary, blocksInSecondary}}, {largeWindow, {blocksInPrimary, blocksInSecondary}}}; + auto const poolConfigurations = std::vector{ + {smallWindow, sizePerHead, nvinfer1::DataType::kHALF}, {largeWindow, sizePerHead, nvinfer1::DataType::kBF16}}; + + auto kvCacheManager = std::make_unique(numLayers, numKvHeads, sizePerHead, tokensPerBlock, + blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, /*dtype=*/nvinfer1::DataType::kHALF, + sinkTokenLength, stream, maxSequenceLength, + /*chunkSize=*/0, /*enableBlockReuse=*/false, CacheType::kSELF, + /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, + /*enablePartialReuse=*/true, /*copyOnpartialReuse=*/true, + /*kvCacheConnectorManager=*/nullptr, + /*enableIndexerKCache=*/false, /*indexerKCacheQuantBlockSize=*/128, /*indexerKCacheIndexHeadDim=*/0, + /*indexerKCacheUseFp4=*/false, + /*linearAttentionMetadata=*/std::nullopt, poolConfigurations); + kvCacheManager->allocatePools(/*useUvm=*/false); + + // Sanity: the manager really does host KV pools with two different dtypes. + auto const numKvPools = kvCacheManager->getBlockManager().getNumPools( + /*includeBlockScalePools=*/false, /*includeIndexerKCachePools=*/false); + ASSERT_GE(numKvPools, 2); + auto const dtype0 = kvCacheManager->getPrimaryPool(0)->getDataType(); + bool foundMismatch = false; + for (SizeType32 i = 1; i < numKvPools; ++i) + { + if (kvCacheManager->getPrimaryPool(i)->getDataType() != dtype0) + { + foundMismatch = true; + break; + } + } + ASSERT_TRUE(foundMismatch) << "Test setup must produce at least two pools with different dtypes"; + + // The guard in CacheTransBufferManager's constructor must reject this. + EXPECT_THROW({ CacheTransBufferManager const ctbm(kvCacheManager.get(), /*maxNumTokens=*/std::nullopt); }, + tensorrt_llm::common::TllmException); +} diff --git a/tensorrt_llm/_torch/auto_deploy/compile/backends/torch_cudagraph.py b/tensorrt_llm/_torch/auto_deploy/compile/backends/torch_cudagraph.py index 71752a2734fe..80226aca6dff 100644 --- a/tensorrt_llm/_torch/auto_deploy/compile/backends/torch_cudagraph.py +++ b/tensorrt_llm/_torch/auto_deploy/compile/backends/torch_cudagraph.py @@ -1043,8 +1043,8 @@ def _setup_piecewise_mixed_batch(seq_info: Any, num_tokens: int) -> None: input_ids=input_ids_flat, cu_seqlen=cu_seqlen, input_pos=0, - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, + cache_loc_per_pool=[cache_loc], + cu_num_pages_per_pool=[cu_num_pages], slot_idx=slot_idx, ) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/flashinfer_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/flashinfer_attention.py index 67ec45d04156..529556729c27 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/flashinfer_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/flashinfer_attention.py @@ -147,9 +147,12 @@ def plan_generate_only( cu_num_pages: torch.Tensor, cache_loc: torch.Tensor, last_page_len: torch.Tensor, + pool_window_left: Optional[int] = None, ): for plan_params in self.cached_cuda_graph_decode_wrappers: if plan_params.num_seq == num_seq: + if pool_window_left is not None and plan_params.window_left != pool_window_left: + continue wrapper = self.cached_cuda_graph_decode_wrappers[plan_params] flashinfer.decode.fast_decode_plan( wrapper, @@ -325,6 +328,7 @@ def prepare_flashinfer_metadata_host( cu_num_pages_host: torch.Tensor, cache_loc_host: torch.Tensor, last_page_len_host: torch.Tensor, + pool_window_left: Optional[int] = None, ) -> None: batch_info = BatchInfo(batch_info_host) num_prefill, num_prefill_tokens, num_decode = batch_info.get_absorbed_info() @@ -335,6 +339,7 @@ def prepare_flashinfer_metadata_host( cu_num_pages_host[: num_decode + 1], cache_loc_host, last_page_len_host[:num_decode], + pool_window_left=pool_window_left, ) @@ -577,10 +582,15 @@ def get_prepare_extra_metadata_info( def get_cache_initializers( cls, source_attn_node: Node, cache_config: KvCacheConfig ) -> ResourceHandlerDict: + """Build the per-layer KV handler used by the kvcache transform.""" # source op is [bsnd] layout already k_fake: FakeTensor = source_attn_node.args[1].meta["val"] num_kv_heads = k_fake.shape[2] head_dim = k_fake.shape[3] + # ``sliding_window`` is propagated into the handler so layers + # with different windows land in separate pools. + (sw,) = extract_op_args(source_attn_node, "sliding_window") + sliding_window = sw if isinstance(sw, int) and sw > 0 else 0 return { "kv_cache": KVPagedResourceHandler( @@ -589,6 +599,7 @@ def get_cache_initializers( dtype=cls.resolve_cache_dtype(cache_config.dtype, k_fake.dtype), kv_factor=2, kv_layout=_GlobalFlashInferPlanner.kv_layout, + sliding_window=sliding_window, ) } diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py index 2667f4e18ac5..192a2e659d44 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py @@ -1528,9 +1528,14 @@ def get_prepare_extra_metadata_info( def get_cache_initializers( cls, source_attn_node: Node, cache_config: KvCacheConfig ) -> ResourceHandlerDict: + """Build the per-layer KV handler used by the kvcache transform.""" k_fake: FakeTensor = source_attn_node.args[1].meta["val"] num_kv_heads = k_fake.shape[2] head_dim = k_fake.shape[3] + # ``sliding_window`` is propagated into the handler so layers + # with different windows land in separate pools. + (sw,) = extract_op_args(source_attn_node, "sliding_window") + sliding_window = sw if isinstance(sw, int) and sw > 0 else 0 return { "kv_cache": KVPagedResourceHandler( @@ -1539,6 +1544,7 @@ def get_cache_initializers( dtype=cls.resolve_cache_dtype(cache_config.dtype, k_fake.dtype), kv_factor=2, kv_layout=KV_LAYOUT, + sliding_window=sliding_window, ) } diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py index 875ea2d2e5a5..dfd04e4caefa 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py @@ -852,7 +852,7 @@ def get_standard_metadata_args(cls) -> List[str]: def get_cache_initializers( cls, source_attn_node: Node, cache_config: KvCacheConfig ) -> ResourceHandlerDict: - """Return only KV cache handler (no workspace handler, managed like flashinfer).""" + """Return only the KV cache handler (no workspace handler; managed like flashinfer).""" # In fused QKV mode, K arg is the same as Q (flat fused tensor), so use hints. if source_attn_node.meta.get("_trtllm_fused_qkv"): num_kv_heads = source_attn_node.meta["_trtllm_num_kv_heads"] @@ -876,6 +876,10 @@ def get_cache_initializers( num_kv_heads = k_fake.shape[2] head_dim = k_fake.shape[3] kv_dtype = k_fake.dtype + # ``sliding_window`` is propagated into the handler so layers + # with different windows land in separate pools. + (sw,) = extract_op_args(source_attn_node, "sliding_window") + sliding_window = sw if isinstance(sw, int) and sw > 0 else 0 return { "kv_cache": KVPagedResourceHandler( @@ -884,6 +888,7 @@ def get_cache_initializers( dtype=cls.resolve_cache_dtype(cache_config.dtype, kv_dtype), kv_factor=2, kv_layout="HND", + sliding_window=sliding_window, ) } diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention_interface.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention_interface.py index 50d163209f80..838a1731791a 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention_interface.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention_interface.py @@ -354,6 +354,29 @@ def resize(self, name: str, new_capacity: int) -> None: self._device_views[name] = self._trunc_device_bufs[name].view(dtype) self._host_views[name] = self._trunc_host_bufs[name].view(dtype) + def add_truncatable_tensor(self, name: str, max_numel: int, dtype: torch.dtype) -> None: + """Add a new truncatable tensor after initialization. + + Args: + name: Name of the tensor. + max_numel: Maximum number of elements. + dtype: Data type. + """ + assert name not in self._tensor_specs, f"Tensor '{name}' already registered" + self._tensor_specs[name] = (max_numel, dtype) + self._tensor_order.append(name) + self._truncatable_names.add(name) + self._current_lengths[name] = 0 + + byte_size = max_numel * dtype.itemsize + device = self._device_buffer.device + self._trunc_device_bufs[name] = torch.empty(byte_size, dtype=torch.uint8, device=device) + self._trunc_host_bufs[name] = torch.empty( + byte_size, dtype=torch.uint8, device="cpu", pin_memory=prefer_pinned() + ) + self._device_views[name] = self._trunc_device_bufs[name].view(dtype) + self._host_views[name] = self._trunc_host_bufs[name].view(dtype) + def to(self, *args, **kwargs) -> None: """Move all device buffers to a new device/dtype.""" old_device = self._device_buffer.device @@ -721,6 +744,11 @@ def __init__( self._extra_args: Dict[str, Optional[torch.Tensor]] = {} ############################################################################################ + # VSWA WINDOW GROUPS ####################################################################### + self._window_groups: List[int] = [] + self._window_group_map: Dict[int, int] = {} + ############################################################################################ + # HOST PREPARE FOR ATTENTION FORWARD ####################################################### self._host_prepare_functions: List[Tuple[PrepareMetadataHostCallable, List[str]]] = [] @@ -864,13 +892,20 @@ def estimate_cache_tokens_per_forward(self) -> int: num_blocks_estimate = num_blocks_estimate_per_seq * self.max_batch_size return num_blocks_estimate * self.tokens_per_block - def update_cache_information(self, num_blocks: int, block_offset_multiplier: int = 0) -> None: + def update_cache_information( + self, + num_blocks: int, + block_offset_multiplier: int = 0, + ) -> None: """Update cache information after cache manager creation. Sets num_blocks and block_offset_multiplier, writes max_seq_info into BatchInfo (constant after this call), and resizes cache_loc if needed. + + Args: + num_blocks: Number of blocks in the primary pool. + block_offset_multiplier: Block offset multiplier derived from kv_cache strides. """ - # set num_blocks and block_offset_multiplier self._num_blocks = num_blocks # write max_seq_info once into BatchInfo (constant after this call) @@ -891,6 +926,97 @@ def update_cache_information(self, num_blocks: int, block_offset_multiplier: int if estimated_capacity > cache_loc_capacity: self._input_buffer.resize("cache_loc", estimated_capacity) + # Keep per-group cache_loc buffers in sync + for group_idx in range(1, self.num_window_groups): + self._input_buffer.resize(f"cache_loc_g{group_idx}", estimated_capacity) + + def register_window_groups(self, window_sizes: List[int]) -> None: + """Register KV window groups and create per-group cache tensors. + + Group ordering here matches the KV-cache group/pool ordering produced by + the kvcache transform — ``window_sizes[g]`` belongs to pool ``g``, so a + single ``group_idx`` routes graph metadata, storage pools, and runtime + inputs together. + + Group 0 reuses the existing ``cache_loc`` / ``cu_num_pages`` / + ``last_page_len`` / ``extra_page_per_seq`` tensors. + Groups 1..N-1 get new dedicated tensors (``cache_loc_g{i}``, + ``cu_num_pages_g{i}``, ...). For a single-pool (non-VSWA) model only + group 0 is registered and no new tensors are created. + + Args: + window_sizes: List of per-group window sizes (one entry per group, in + the same order as the KV groups). Full-attention groups use + ``max_seq_len``. + """ + assert len(window_sizes) >= 1, "register_window_groups requires at least 1 window" + self._window_groups = list(window_sizes) + self._window_group_map = {ws: idx for idx, ws in enumerate(window_sizes)} + + cache_loc_cap = self._input_buffer.get_capacity("cache_loc") + for group_idx in range(1, len(window_sizes)): + suffix = f"_g{group_idx}" + self._input_buffer.add_truncatable_tensor( + f"cache_loc{suffix}", cache_loc_cap, torch.int + ) + self._input_buffer.add_truncatable_tensor( + f"cu_num_pages{suffix}", self.max_batch_size + 1, torch.int + ) + self._input_buffer.add_truncatable_tensor( + f"last_page_len{suffix}", self.max_batch_size, torch.int + ) + self._input_buffer.add_truncatable_tensor( + f"extra_page_per_seq{suffix}", self.max_batch_size, torch.int + ) + # Per-group seq_len_with_cache: under SWA front-eviction the window's + # live cache length diverges from the global (input_pos + seq_len), + # so groups 1..N-1 carry their own window-capped values for both the + # kernel's mask math and the prepare-extra-metadata op (which uses + # it to compute write positions for new KV tokens). + self._input_buffer.add_truncatable_tensor( + f"seq_len_with_cache{suffix}", self.max_batch_size, torch.int + ) + # Register as available args (device + host variants) + group_names = [ + f"cache_loc{suffix}", + f"cu_num_pages{suffix}", + f"last_page_len{suffix}", + f"extra_page_per_seq{suffix}", + f"seq_len_with_cache{suffix}", + ] + for base_name in group_names: + self._available_args.add(base_name) + self._available_args.add(base_name + self._host_suffix) + + # Zero-fill all per-group device buffers so shape-checking forward + # passes (resize_kv_cache) that run before nest_sequences don't read + # uninitialized data. With zeros: cache_loc→block 0 (safe), + # cu_num_pages→0 pages. + for base_name in group_names: + self._input_buffer._trunc_device_bufs[base_name].zero_() + + @property + def window_groups(self) -> List[int]: + """Per-pool window sizes (one entry per registered pool, in pool order).""" + return self._window_groups + + @property + def window_group_map(self) -> Dict[int, int]: + """Map from window_size to group index.""" + return self._window_group_map + + @property + def num_window_groups(self) -> int: + """Number of window groups (0 or 1 for non-VSWA, 2+ for VSWA).""" + return len(self._window_groups) + + def get_cache_loc_for_group(self, group_idx: int) -> str: + """Return the cache_loc tensor name for a given window group.""" + return "cache_loc" if group_idx == 0 else f"cache_loc_g{group_idx}" + + def get_cu_num_pages_for_group(self, group_idx: int) -> str: + """Return the cu_num_pages tensor name for a given window group.""" + return "cu_num_pages" if group_idx == 0 else f"cu_num_pages_g{group_idx}" def activate_arg(self, arg_name: str) -> bool: """Activate a desired argument. @@ -946,12 +1072,18 @@ def set_example_sequence( slot_idx = torch.arange(bs) assert len(slot_idx) >= bs + # Replicate the vanilla single-pool warm-up data for every registered + # pool so VSWA models (num_window_groups >= 2) receive one entry per + # pool. Block geometry (tokens_per_block, max_blocks_per_seq) is + # uniform across pools, so the same cache_loc / cu_num_pages tensors + # are valid metadata for every pool. + n_pools = max(1, self.num_window_groups) self.nest_sequences( input_ids.flatten(), cu_seqlen=torch.arange(bs + 1, dtype=torch.int) * seq_len, input_pos=0, # no cache history - cache_loc=cache_loc, # vanilla page assignments - cu_num_pages=cu_num_pages, # vanilla page assignments + cache_loc_per_pool=[cache_loc for _ in range(n_pools)], + cu_num_pages_per_pool=[cu_num_pages for _ in range(n_pools)], slot_idx=slot_idx, # vanilla slot indices **extra_args, ) @@ -1098,11 +1230,18 @@ def nest_sequences( cu_seqlen: Union[Sequence[int], torch.Tensor], input_pos: Union[Sequence[int], int, torch.Tensor], batch_info: Union[Sequence[int], torch.Tensor, None] = None, - cache_loc: Union[Sequence[int], torch.Tensor, None] = None, - cu_num_pages: Union[Sequence[int], torch.Tensor, None] = None, - extra_page_per_seq: Optional[Sequence[int]] = None, slot_idx: Union[Sequence[int], torch.Tensor, None] = None, prompt_lens: Union[Sequence[int], torch.Tensor, None] = None, + ### PER-POOL CACHE DATA (pool 0 is the only pool for non-VSWA configurations) ############# + cache_loc_per_pool: Optional[Sequence[Sequence[int]]] = None, + cu_num_pages_per_pool: Optional[Sequence[Sequence[int]]] = None, + extra_page_per_seq_per_pool: Optional[Sequence[Sequence[int]]] = None, + # Phase 2: per-pool window-capped values used by VSWA front-eviction so + # the live cache length / last-page length can diverge from the global + # (input_pos + seq_len). When None, non-zero pools replicate the global + # value (resp. fall back to ``lpl_host``). + seq_len_with_cache_per_pool: Optional[Sequence[Sequence[int]]] = None, + last_page_len_per_pool: Optional[Sequence[Sequence[int]]] = None, ### RUNTIME ARGUMENTS ###################################################################### gather_context_logits: bool = False, _gather_idx: Union[Sequence[int], torch.Tensor, None] = None, @@ -1124,12 +1263,21 @@ def nest_sequences( heuristic is used to compute it from seq_len. NOTE: the heuristic makes potentially incorrect assumptions about the batch composition. batch_info should be provided explicitly to ensure correctness. - cache_loc: Flat list of page indices for all sequences. Must be provided together with - cu_num_pages. - cu_num_pages: Cumulative number of pages for all sequences. Must be provided together with - cache_loc. - extra_page_per_seq: Extra page per sequence for deferred page insertion. slot_idx: State slot index for each sequence in the batch. + cache_loc_per_pool: Flat list of page indices for all sequences, per pool. Must be + provided together with cu_num_pages_per_pool. Indexed by pool_idx (0-based); + pool 0 is the only pool for non-VSWA configurations. + cu_num_pages_per_pool: Cumulative number of pages for all sequences, per pool. Must be + provided together with cache_loc_per_pool. + extra_page_per_seq_per_pool: Extra page per sequence for deferred page insertion, + per pool. + seq_len_with_cache_per_pool: Per-pool window-capped seq_len_with_cache + used by VSWA front-eviction. Pool 0 is unused (it carries the + unclamped global value). When None, non-zero pools replicate the + global ``input_pos + seq_len``. + last_page_len_per_pool: Per-pool window-capped last_page_len used by + VSWA front-eviction. Pool 0 is unused. When None, non-zero pools + fall back to ``lpl_host`` (the global value). gather_context_logits: If True, keep all context logits (no selective gathering). If False (default), only the last token per context sequence is gathered while all extend/decode tokens are kept. @@ -1182,17 +1330,68 @@ def nest_sequences( ### UPDATE CACHE ASSIGNMENTS (needs to be provided if required!) ########################### # check for updated page assignments - assert (cache_loc is None) == (cu_num_pages is None), "Both must be provided together!" - self._stage_arg("cache_loc", cache_loc) - self._stage_arg("cu_num_pages", cu_num_pages) + assert (cache_loc_per_pool is None) == (cu_num_pages_per_pool is None), ( + "cache_loc_per_pool and cu_num_pages_per_pool must be provided together!" + ) + # Global (unclamped) last_page_len, used as the per-pool fallback below + # when the caller doesn't supply ``last_page_len_per_pool`` (warmup, + # set_example_sequence, CUDA-graph capture). lpl_host = (ip_host + sl_host - 1) % self.tokens_per_block + 1 - self._stage_arg("last_page_len", lpl_host) # check for updated slot_idx self._stage_arg("slot_idx", slot_idx) - # check for updated extra_page_per_seq - self._stage_arg("extra_page_per_seq", extra_page_per_seq) + # cu_num_pages from pool 0 is reused below by the pages_per_seq derivative + # update; capture it before the per-pool staging loop. + cu_num_pages = cu_num_pages_per_pool[0] if cu_num_pages_per_pool is not None else None + + # Guard against partial lists: every registered pool must be present + # in each of the per-pool lists, otherwise omitted pools would silently + # reuse the previous batch's staged tensors. + if cache_loc_per_pool is not None and self.num_window_groups >= 2: + for name, seq in ( + ("cache_loc_per_pool", cache_loc_per_pool), + ("cu_num_pages_per_pool", cu_num_pages_per_pool), + ("extra_page_per_seq_per_pool", extra_page_per_seq_per_pool), + ("seq_len_with_cache_per_pool", seq_len_with_cache_per_pool), + ("last_page_len_per_pool", last_page_len_per_pool), + ): + if seq is None: + continue + assert len(seq) == self.num_window_groups, ( + f"{name} has {len(seq)} entries, expected {self.num_window_groups}" + ) + + # Stage per-pool cache data uniformly: pool 0 uses the unsuffixed + # base names; pools 1..N-1 use the f"_g{pool_idx}" suffix. When the + # caller doesn't provide per-pool data (warmup, set_example_sequence, + # CUDA graph capture), pools 1..N-1 fall back to pool 0's data so every + # kernel receives valid metadata. + if cache_loc_per_pool is not None: + num_pools_provided = len(cache_loc_per_pool) + pool0_cache_loc = cache_loc_per_pool[0] + pool0_cu_num_pages = ( + cu_num_pages_per_pool[0] if cu_num_pages_per_pool is not None else None + ) + pool0_extra_page = ( + extra_page_per_seq_per_pool[0] if extra_page_per_seq_per_pool is not None else None + ) + num_pools_active = max(num_pools_provided, self.num_window_groups) + for pool_idx in range(num_pools_active): + suffix = "" if pool_idx == 0 else f"_g{pool_idx}" + provides_pool = pool_idx < num_pools_provided + pool_cache_loc = cache_loc_per_pool[pool_idx] if provides_pool else pool0_cache_loc + self._stage_arg(f"cache_loc{suffix}", pool_cache_loc) + if cu_num_pages_per_pool is not None: + pool_cu_num_pages = ( + cu_num_pages_per_pool[pool_idx] if provides_pool else pool0_cu_num_pages + ) + self._stage_arg(f"cu_num_pages{suffix}", pool_cu_num_pages) + if extra_page_per_seq_per_pool is not None: + pool_extra_page = ( + extra_page_per_seq_per_pool[pool_idx] if provides_pool else pool0_extra_page + ) + self._stage_arg(f"extra_page_per_seq{suffix}", pool_extra_page) ### UPDATE OPTIONAL DERIVATIVE METADATA #################################################### if self._is_required("position_ids"): @@ -1213,9 +1412,26 @@ def nest_sequences( pages_per_seq = cu_num_pages_host[1:] - cu_num_pages_host[:-1] self._stage_arg("pages_per_seq", pages_per_seq) - # update sequence length with cache + # Stage seq_len_with_cache and last_page_len per pool through a single + # loop. Pool 0 uses the unsuffixed name; pools 1..N-1 use f"_g{i}". + # The unsuffixed name is a wire-level alias for pool 0 -- non-VSWA + # graphs, prepare-extra ops, and host-prep functions still reference + # it -- not a "full-attention pool" privilege; pool 0 is whichever KV + # handler the kvcache transform encountered first, which can be the + # SWA pool (Gemma-3n where layer 0 is sliding, or any pure-SWA model + # such as Mistral / Phi-3). When the caller doesn't supply per-pool + # values (warmup, set_example_sequence, CUDA-graph capture), every + # pool falls back to the global value. Dropping the unsuffixed alias + # entirely is a follow-up refactor that touches the transform, + # backend descriptors, and op signatures. seq_len_with_cache = ip_host + sl_host - self._stage_arg("seq_len_with_cache", seq_len_with_cache) + n_pools = max(1, self.num_window_groups) + per_pool_swc = seq_len_with_cache_per_pool or [seq_len_with_cache] * n_pools + per_pool_lpl = last_page_len_per_pool or [lpl_host] * n_pools + for pool_idx in range(n_pools): + suffix = "" if pool_idx == 0 else f"_g{pool_idx}" + self._stage_arg(f"seq_len_with_cache{suffix}", per_pool_swc[pool_idx]) + self._stage_arg(f"last_page_len{suffix}", per_pool_lpl[pool_idx]) # prompt_lens: original context length per sequence, constant across iterations. # Defaults to seq_len when not provided (correct for prefill). @@ -1404,6 +1620,25 @@ def offset_pos_and_cache_(self, offset: torch.Tensor) -> None: last_page_len %= self.tokens_per_block last_page_len += 1 + # Adjust per-group cache assignments for VSWA groups 1..N-1 + for group_idx in range(1, self.num_window_groups): + suffix = f"_g{group_idx}" + if self._is_active(f"cache_loc{suffix}"): + lpl_g = self.get_arg(f"last_page_len{suffix}", truncate=True) + lpl_g += offset + delta_g = (lpl_g > self.tokens_per_block).int() - (lpl_g <= 0).int() + torch.ops.auto_deploy.adjust_ragged_triton( + cache_loc=self.get_arg(f"cache_loc{suffix}"), + cu_num_blocks=self.get_arg(f"cu_num_pages{suffix}"), + extra_idx=self.get_arg(f"extra_page_per_seq{suffix}"), + delta=delta_g, + num_sequences=num_sequences, + max_blocks_per_seq=self.max_blocks_per_seq, + ) + lpl_g -= 1 + lpl_g %= self.tokens_per_block + lpl_g += 1 + # --- position_ids (device) --- position_ids = self.get_arg("position_ids", truncate=True, unflatten=False) # position_ids is per-token while offset is per-sequence; expand if needed @@ -1426,6 +1661,15 @@ def offset_pos_and_cache_(self, offset: torch.Tensor) -> None: swc = self.get_arg("seq_len_with_cache", truncate=True) swc += offset + # Keep per-group seq_len_with_cache in sync with the global value for + # the overlap scheduler path. This is exact in the non-eviction regime; + # under eviction the next nest_sequences re-caps to the window. + for group_idx in range(1, self.num_window_groups): + suffix = f"_g{group_idx}" + if self._is_active(f"seq_len_with_cache{suffix}"): + swc_g = self.get_arg(f"seq_len_with_cache{suffix}", truncate=True) + swc_g += offset + # --- use_initial_states (device) --- use_initial_states = self.get_arg("use_initial_states", truncate=True) use_initial_states[:] = input_pos > 0 @@ -1597,15 +1841,24 @@ def allocate(self, sequence_info: SequenceInfo) -> torch.Tensor: class KVPagedResourceHandler(ResourceHandler): """Handler for paged KV cache resources. - This handler indicates the resource should be managed by the standard KVCacheManager. + This handler indicates the resource should be managed by the standard + KVCacheManager. Handler equality (``__eq__``) is the single source of + truth for KV-cache grouping: layers whose handlers compare equal share a + storage pool and a metadata set (``group_idx == pool_idx``), and differ + otherwise. Including ``sliding_window`` in the equality means same-head-dim + layers with different windows land in separate pools with their own + ``max_attention_window``. Args: num_kv_heads: Number of key-value heads. head_dim: Dimension of each head. dtype: The dtype of the KV cache. kv_factor: The factor of the KV cache. Default is 2 for combined k/v cache. - kv_layout: Memory layout for the KV cache. Either "HND" (head-num-dim) or "NHD" (num-head-dim). - Default is "HND" which is the standard layout for flashinfer. + kv_layout: Memory layout for the KV cache. Either "HND" (head-num-dim) or + "NHD" (num-head-dim). Default is "HND" which is the standard layout + for flashinfer. + sliding_window: Sliding window size for this layer. ``0`` means full + attention; a positive value puts this layer in its own VSWA group. """ @property @@ -1620,6 +1873,7 @@ def __init__( dtype: torch.dtype, kv_factor: int = 2, kv_layout: Literal["HND", "NHD"] = "HND", + sliding_window: int = 0, ) -> None: """Initialize the KVPagedResourceHandler. @@ -1629,6 +1883,7 @@ def __init__( dtype: The dtype of the KV cache. kv_factor: The factor of the KV cache. Default is 2. kv_layout: Memory layout - "HND" or "NHD". Default is "HND". + sliding_window: Sliding window size for this layer. 0 means full attention. """ self.num_kv_heads = num_kv_heads self.head_dim = head_dim @@ -1636,9 +1891,18 @@ def __init__( self.kv_factor = kv_factor assert kv_factor in [1, 2], f"Invalid kv_factor: {kv_factor}" self.kv_layout = kv_layout + self.sliding_window = ( + sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 + ) def __eq__(self, other: Optional[ResourceHandler]) -> bool: - """Check compatibility for KVCacheManager (head_dim and dtype must match).""" + """Check compatibility — layers in the same group share a KVCacheManager pool. + + Including sliding_window means same-head-dim layers with different windows + get separate pools. This trades slightly higher memory (one pool per unique + window instead of a shared pool with multiple windows) for a simpler design + where group = pool = metadata set, with no cross-referencing needed. + """ if type(other) is not type(self): return False return ( @@ -1646,6 +1910,7 @@ def __eq__(self, other: Optional[ResourceHandler]) -> bool: and self.dtype == other.dtype and self.kv_factor == other.kv_factor and self.kv_layout == other.kv_layout + and self.sliding_window == other.sliding_window ) def _get_bytes_per_token(self) -> int: diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py index 43e9ee396fea..934d951d374e 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py @@ -2909,16 +2909,21 @@ class Gemma4TextExportInfo(TextModelExportInfo): def post_process(self, sub_mod: nn.Module, sub_gm: GraphModule): super().post_process(sub_mod, sub_gm) + # Gemma4Model.forward calls language_model.get_per_layer_inputs unconditionally, + # so the GraphModule must expose it on every variant. The method itself short-circuits + # to None when embed_tokens_per_layer is None, which is the non-E2B case. embed_tokens_per_layer = getattr(sub_mod, "embed_tokens_per_layer", None) + sub_gm.get_per_layer_inputs = types.MethodType( + sub_mod.get_per_layer_inputs.__func__, sub_gm + ) + if embed_tokens_per_layer is None: + sub_gm.embed_tokens_per_layer = None return sub_gm.config = sub_mod.config sub_gm.hidden_size_per_layer_input = sub_mod.hidden_size_per_layer_input sub_gm.vocab_size_per_layer_input = sub_mod.vocab_size_per_layer_input - sub_gm.get_per_layer_inputs = types.MethodType( - sub_mod.get_per_layer_inputs.__func__, sub_gm - ) for embed_name, subsubmod in sub_mod.named_modules(): if subsubmod is embed_tokens_per_layer: diff --git a/tensorrt_llm/_torch/auto_deploy/shim/ad_executor.py b/tensorrt_llm/_torch/auto_deploy/shim/ad_executor.py index d5da4e26641a..b633d6a3b625 100644 --- a/tensorrt_llm/_torch/auto_deploy/shim/ad_executor.py +++ b/tensorrt_llm/_torch/auto_deploy/shim/ad_executor.py @@ -14,7 +14,7 @@ from collections import abc, defaultdict from dataclasses import dataclass from types import SimpleNamespace -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Sequence, Tuple import torch from strenum import StrEnum @@ -202,6 +202,105 @@ def _call_func(): return wrapper +def _compute_window_local_view( + all_indices: Sequence[int], + front_removed: int, + end_compute_i: int, + group_window: int, + tokens_per_block: int, +) -> Tuple[List[int], int, int, int]: + """Compute the window-coherent metadata view for one (request, window) pair. + + The C++ KVCacheManager allocates blocks linearly during prefill and only + front-evicts during generation (kvCacheManager.cpp::adjustBlocksIfNeeded + is invoked from addToken, not addSequenceBatch). Two regimes follow: + + * Pre-eviction (typically a single long prefill that has not yet been + decoded past the window): ``front_removed == 0`` and the live page + list covers the full sequence. The kernel needs every live page and + the unclamped local cache length; the sliding-window mask is applied + inside the kernel. + + * Post-eviction (decode/extend has advanced past the window): the C++ + side has bumped ``mNumFrontBlocksRemovedPerWindow`` without popping + ``mCacheBlockIds``, so the head of the page list is stale. After + slicing those entries off, the live cache length in window-local + coords is ``end_compute_i - front_removed * tokens_per_block`` — + bounded above by ``group_window + tokens_per_block - 1`` by the + ``while (live >= window + page_size) detachFrontBlock`` loop. + + Both regimes collapse to a single rule: trust the C++ accounting and + derive the local cache length from ``end_compute_i`` and ``front_removed`` + rather than artificially clamping to ``group_window``. Disagreement + between the Python and C++ sides (e.g. a caller passing a stale + ``end_compute_i`` against a fresher ``front_removed``) is asserted on + rather than silently masked. + + Args: + all_indices: Full historical page list from ``get_cache_indices``, + including stale front entries when eviction has fired. + front_removed: Count of stale front entries, from + ``get_num_front_blocks_removed``. + end_compute_i: Global token position after this step's tokens are + processed (``input_pos + seq_len``). + group_window: Sliding-window size for this pool. Currently only used + for the defensive sanity assertion below; the window mask itself + is applied inside the kernel via the ``sliding_window`` constant. + tokens_per_block: Cache page size. + + Returns: + active_indices: the live slice of all_indices to hand the kernel. + extra_page: the next page after the live slice (used by the overlap + scheduler's deferred-page insertion), or -1 when there is no + slot past the live region. + seq_len_with_cache: window-local cache length the kernel should see. + last_page_len: derived from seq_len_with_cache and tokens_per_block. + """ + live_len = len(all_indices) - front_removed + # Window-local cache length: how many tokens the live pages actually hold. + # Pre-eviction: end_compute_i (the full prefill so far). + # Post-eviction: end_compute_i - front_removed * page_size (capped above by + # window + page_size - 1 thanks to the C++ eviction loop). + local_cache_len = end_compute_i - front_removed * tokens_per_block + # Live cache length must fit inside the live pages. The C++ KVCacheManager + # keeps the two in sync (it bumps front_removed exactly as it frees blocks, + # and allocates new ones for incoming tokens), so a violation means the + # Python side and C++ side disagree on either end_compute_i or + # front_removed -- fail loudly rather than silently clamp. + live_capacity = live_len * tokens_per_block + assert local_cache_len <= live_capacity, ( + f"window-local cache length {local_cache_len} exceeds live page capacity " + f"{live_capacity} (live_len={live_len}, end_compute_i={end_compute_i}, " + f"front_removed={front_removed}, tokens_per_block={tokens_per_block}). " + f"C++ KVCacheManager accounting is expected to keep these in sync." + ) + active_token_count = local_cache_len + # Sanity: live cache should never exceed window + one spillover page once + # front-eviction has fired (the C++ ``while (live >= window + page_size) + # detachFrontBlock`` loop guarantees this). Pre-eviction (front_removed=0, + # e.g. a single long prefill) may legitimately exceed the window because + # blocks are allocated linearly for the full prompt and only detached + # during generation -- so we skip the check there. + assert active_token_count <= group_window + tokens_per_block or front_removed == 0, ( + f"window-local cache length {active_token_count} exceeds " + f"window {group_window} + page_size {tokens_per_block} " + f"(end_compute_i={end_compute_i}, front_removed={front_removed})" + ) + # Pages needed to address active_token_count tokens; equals live_len in + # the common case (post-eviction live = window + spillover, pre-eviction + # live = ceil(end_compute_i / page_size)). + pages_for_active = (active_token_count + tokens_per_block - 1) // tokens_per_block + num_active = min(live_len, pages_for_active) + active_indices = list(all_indices[front_removed : front_removed + num_active]) + extra_slot_idx = front_removed + num_active + extra_page = all_indices[extra_slot_idx] if len(all_indices) > extra_slot_idx else -1 + if active_token_count > 0: + last_page_len = (active_token_count - 1) % tokens_per_block + 1 + else: + last_page_len = 0 + return active_indices, extra_page, active_token_count, last_page_len + + class ADEngine(ModelEngine): """The AutoDeploy Engine (ADEngine) is the main engine interface to execute AutoDeploy models. @@ -658,17 +757,36 @@ def _prepare_inputs( if is_overlap: mask_scatter_indices.extend(list(range(cu_seqlen[-2], cu_seqlen[-1]))) - # store cache information for all requests now + # Store cache information for all requests. + # All KV pools are hosted by a single C++ KVCacheManager; pool index is + # the position of the window in self.cache_seq_interface.kv_group_windows + # (the same source the kvcache transform used to register window groups + # on SequenceInfo). Per-window queries on the manager route to the + # correct C++ pool via mLayerToWindowSize. + kv_group_windows = self.cache_seq_interface.kv_group_windows + # Cache hot lookups so the per-request loop avoids repeated C++ + # dispatch / hasattr calls. _tokens_per_block = kv_cache_manager.tokens_per_block _use_mamba = hasattr(kv_cache_manager, "mamba_cache_index") - cache_loc: List[int] = [] - cu_num_pages: List[int] = [0] - extra_page_per_seq: List[int] = [] state_slot_idx: List[int] = [] - batch_cache_indices = kv_cache_manager.get_batch_cache_indices( - [r.py_request_id for r in ordered_requests] - ) + # Uniform per-pool transport: a single-pool deployment is just the + # degenerate case of VSWA with one pool, so all pools (including 0) + # take the same code path below. + num_pools = len(kv_group_windows) + cache_loc_per_pool: List[List[int]] = [[] for _ in range(num_pools)] + cu_num_pages_per_pool: List[List[int]] = [[0] for _ in range(num_pools)] + extra_page_per_seq_per_pool: List[List[int]] = [[] for _ in range(num_pools)] + seq_len_with_cache_per_pool: List[List[int]] = [[] for _ in range(num_pools)] + last_page_len_per_pool: List[List[int]] = [[] for _ in range(num_pools)] + + # Batched per-pool lookup preserves the #13560 host-overhead + # optimization: one C++ call per pool instead of one per (request, pool). + request_ids = [r.py_request_id for r in ordered_requests] + batch_cache_indices_per_pool: List[List[List[int]]] = [ + kv_cache_manager.get_batch_cache_indices(request_ids, window_size=group_window) + for group_window in kv_group_windows + ] for i, request in enumerate(ordered_requests): # store seq slot idx (use mamba_cache_index if available) @@ -682,17 +800,45 @@ def _prepare_inputs( # get some info on the current request seq_len_i = cu_seqlen[i + 1] - cu_seqlen[i] end_compute_i = input_pos[i] + seq_len_i - # Inline get_num_kv_blocks (pure Python, saves function-call overhead per request) - num_active_blocks_i = (end_compute_i + _tokens_per_block - 1) // _tokens_per_block - - # construct cache information for the current request - cache_indices = batch_cache_indices[i] - cache_loc.extend(cache_indices[:num_active_blocks_i]) - cu_num_pages.append(cu_num_pages[i] + num_active_blocks_i) - if len(cache_indices) > num_active_blocks_i: - extra_page_per_seq.append(cache_indices[num_active_blocks_i]) - else: - extra_page_per_seq.append(-1) + + for pool_idx, group_window in enumerate(kv_group_windows): + all_indices = batch_cache_indices_per_pool[pool_idx][i] + # SWA front-eviction: get_batch_cache_indices returns the FULL + # historical page list including front-evicted entries (the + # C++ side bumps a counter rather than popping mCacheBlockIds). + # _compute_window_local_view slices it down to the live window + # in window-local coords. + front_removed = kv_cache_manager.get_num_front_blocks_removed( + request.py_request_id, window_size=group_window + ) + ( + active_indices, + extra_page, + active_token_count, + lpl_i, + ) = _compute_window_local_view( + all_indices, + front_removed=front_removed, + end_compute_i=end_compute_i, + group_window=group_window, + tokens_per_block=_tokens_per_block, + ) + num_active = len(active_indices) + + cache_loc_per_pool[pool_idx].extend(active_indices) + cu_num_pages_per_pool[pool_idx].append( + cu_num_pages_per_pool[pool_idx][i] + num_active + ) + extra_page_per_seq_per_pool[pool_idx].append(extra_page) + # Window-local seq_len_with_cache / last_page_len for every + # pool (including 0). For full-attention pools the helper + # returns the unclamped global value (group_window equals + # max_seq_len, no clamping kicks in), so this is identical to + # the legacy single-pool path for non-SWA models. For SWA + # pools (whether pool 0 or pool 1+), it carries the + # window-local coords the kernel needs under front-eviction. + seq_len_with_cache_per_pool[pool_idx].append(active_token_count) + last_page_len_per_pool[pool_idx].append(lpl_i) # Store batch information based on prefill, decode, and extend requests. num_decode = len(generation_requests) @@ -717,9 +863,11 @@ def _prepare_inputs( cu_seqlen=cu_seqlen, input_pos=input_pos, batch_info=batch_info, - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, - extra_page_per_seq=extra_page_per_seq, + cache_loc_per_pool=cache_loc_per_pool if num_pools > 0 else None, + cu_num_pages_per_pool=cu_num_pages_per_pool if num_pools > 0 else None, + extra_page_per_seq_per_pool=extra_page_per_seq_per_pool if num_pools > 0 else None, + seq_len_with_cache_per_pool=seq_len_with_cache_per_pool if num_pools > 0 else None, + last_page_len_per_pool=last_page_len_per_pool if num_pools > 0 else None, slot_idx=state_slot_idx, prompt_lens=prompt_lens, gather_context_logits=gather_context_logits, diff --git a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py index cd9ed04cb503..4c48239ae98b 100644 --- a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py +++ b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py @@ -173,8 +173,8 @@ def generate_tokens_batched( input_ids=input_ids_flat, cu_seqlen=cu_seqlen, input_pos=[0] * len(total_lens), - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, + cache_loc_per_pool=[cache_loc], + cu_num_pages_per_pool=[cu_num_pages], slot_idx=list(range(len(total_lens))), **extra_args, ) @@ -217,8 +217,8 @@ def _generate_single_step(idx: int): input_ids=input_ids_flat, cu_seqlen=cu_seqlen, input_pos=input_pos_next, - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, + cache_loc_per_pool=[cache_loc], + cu_num_pages_per_pool=[cu_num_pages], slot_idx=list(range(batch_size)), prompt_lens=total_lens, ) diff --git a/tensorrt_llm/_torch/auto_deploy/shim/interface.py b/tensorrt_llm/_torch/auto_deploy/shim/interface.py index 4bfc0cf00f68..e6714f513184 100644 --- a/tensorrt_llm/_torch/auto_deploy/shim/interface.py +++ b/tensorrt_llm/_torch/auto_deploy/shim/interface.py @@ -29,7 +29,7 @@ MambaHybridCacheManager, MixedMambaHybridCacheManager, ) - from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager + from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager, PoolConfiguration from tensorrt_llm._utils import torch_dtype_to_binding from tensorrt_llm.mapping import Mapping @@ -40,6 +40,7 @@ # CachedSequenceInterface can still be instantiated and used for transforms, # but initialize_resources() and other cache-manager methods will raise. KVCacheManager = None + PoolConfiguration = None MambaHybridCacheManager = None MixedMambaHybridCacheManager = None CacheTypeCpp = None @@ -135,6 +136,11 @@ def __init__( self._caches: Dict[str, torch.Tensor] = {} # KVCacheManager (or MambaHybridCacheManager) for managed resources self._kv_cache_manager: Optional[Union[KVCacheManager, MambaHybridCacheManager]] = None + # Per-pool sliding window sizes, published by the kvcache transform and + # consumed by the executor. Pool index == position in this list, in the + # same order as the C++ manager's internal pool ordering (i.e. the + # insertion order of the per-window shape map keys). + self._kv_group_windows: List[int] = [] # lookup of unmanaged resources self._unmanaged_resources: List[str] = [] self._spec_config = spec_config @@ -296,37 +302,92 @@ def _get_mamba_state_params( def _identify_managed_kv_resources( self, - ) -> Tuple[Optional[KVPagedResourceHandler], ResourceHandlerDict]: - """Identify KV resources compatible with the reference handler for KVCacheManager. - - The first KVPagedResourceHandler becomes the reference. All handlers matching - the reference (via __eq__) are collected for managed allocation. + ) -> Tuple[ + ResourceHandlerDict, + List[PoolConfiguration], + ]: + """Identify managed KV resources and the per-pool configurations. + + Each ``KVPagedResourceHandler`` contributes to a pool keyed by its + effective sliding window (``sliding_window`` if > 0 else + ``max_seq_len``). Today, layers sharing a window must share the + same ``head_dim`` and ``dtype``; the returned list carries one + ``PoolConfiguration`` per window. The list shape is deliberately + flat -- future multi-pool-per-window cases append additional + entries with the same ``window_size`` rather than nesting shape + info under a window key. Insertion order fixes the pool index + used by the C++ ``mLayerToWindowSize`` routing. Returns: - Tuple of (reference_handler, managed_resources_dict). - reference_handler is None if no KV paged resources exist. + Tuple of: + - kv_managed: ResourceHandlerDict -- every KVPagedResourceHandler + in registration order; passed to the C++ ctor as the layer + list. + - pool_configurations: List[PoolConfiguration] -- one entry + per pool, carrying (window_size, head_dim, dtype). + + Raises: + RuntimeError: if two layers share an effective window but disagree + on head_dim or dtype, or if + ``self._requires_uniform_kv_caches`` is True and more than one + distinct pool is present. """ - kv_ref: Optional[KVPagedResourceHandler] = None kv_managed: ResourceHandlerDict = {} + pool_by_window: Dict[int, PoolConfiguration] = {} + + max_seq_len = self.info.max_seq_len for name, handler in self._resource_lookup.items(): if not isinstance(handler, KVPagedResourceHandler): continue - if kv_ref is None: - kv_ref = handler - if handler == kv_ref: - kv_managed[name] = handler - elif self._requires_uniform_kv_caches: - raise RuntimeError( - f"KV resource {name} is not compatible with the managed KV reference " - f"(reference: head_dim={kv_ref.head_dim}, dtype={kv_ref.dtype}, " - f"kv_factor={kv_ref.kv_factor}, kv_layout={kv_ref.kv_layout}; " - f"candidate: head_dim={handler.head_dim}, dtype={handler.dtype}, " - f"kv_factor={handler.kv_factor}, kv_layout={handler.kv_layout}). " - "This configuration requires all KV caches to be managed." + # Effective window: full-attention layers (sliding_window == 0) use + # max_seq_len so the C++ side gets a single concrete window key. + effective_window = handler.sliding_window if handler.sliding_window > 0 else max_seq_len + + kv_managed[name] = handler + + handler_dtype = torch_dtype_to_binding(handler.dtype) + existing = pool_by_window.get(effective_window) + if existing is not None: + if existing.head_dim != handler.head_dim: + raise RuntimeError( + f"KV layer {name} has head_dim={handler.head_dim} but window " + f"{effective_window} already has head_dim={existing.head_dim}. " + "The C++ KVCacheManager keys pools by window, so within a window all " + "layers must share head_dim. Place mixed-shape layers in " + "distinct windows (e.g. via sliding_window)." + ) + if existing.dtype != handler_dtype: + raise RuntimeError( + f"KV layer {name} has dtype={handler.dtype} but window " + f"{effective_window} already has dtype={existing.dtype}. " + "The C++ KVCacheManager keys pools by window, so within a window all " + "layers must share dtype. Place mixed-dtype layers in distinct " + "windows (e.g. via sliding_window)." + ) + else: + pool_by_window[effective_window] = PoolConfiguration( + window_size=effective_window, + head_dim=handler.head_dim, + dtype=handler_dtype, ) - return kv_ref, kv_managed + pool_configurations: List[PoolConfiguration] = list(pool_by_window.values()) + + # If the runtime requires uniform KV caches (e.g. legacy single-pool + # path), more than one distinct pool is incompatible. + if len(pool_configurations) > 1 and self._requires_uniform_kv_caches: + pools_repr = ", ".join( + f"window={pc.window_size} head_dim={pc.head_dim} dtype={pc.dtype}" + for pc in pool_configurations + ) + raise RuntimeError( + "KV resources are not uniform: " + f"{pools_repr}. " + "This configuration requires all KV caches to share a single pool." + ) + + return kv_managed, pool_configurations def _identify_managed_state_resources( self, @@ -421,8 +482,15 @@ def _prepare_kv_cache_config( ) -> KvCacheConfig: """Prepare and configure KvCacheConfig for cache manager creation. - Handles deep copy, max_tokens synchronization across ranks, block reuse settings, - copy_on_partial_reuse validation, and free_gpu_memory_fraction normalization. + Handles deep copy, max_tokens synchronization across ranks, block reuse + settings, copy_on_partial_reuse validation, and free_gpu_memory_fraction + normalization. + + ``max_attention_window`` is the per-layer window list (one entry per + managed KV layer, in the order ``kv_managed`` enumerates them). The C++ + ctor groups these layers internally by window into pools using its + ``mLayerToWindowSize`` map. Full-attention layers report + ``max_seq_len`` (i.e. their effective window). Args: max_tokens: Maximum tokens to allocate, or None to use config defaults. @@ -434,6 +502,16 @@ def _prepare_kv_cache_config( # Make a deep copy of the kv_cache_config to avoid modifying the original object kv_cache_config = copy.deepcopy(self._kv_cache_config_original) + # Build per-layer max_attention_window in kv_managed insertion order. + # The C++ side groups layers by this value into pools. + if kv_managed: + kv_cache_config.max_attention_window = [ + handler.sliding_window if handler.sliding_window > 0 else self.info.max_seq_len + for handler in kv_managed.values() + ] + else: + kv_cache_config.max_attention_window = None + # Update kv_cache_config based on max_tokens if provided if max_tokens is not None: # sync max_tokens across ranks @@ -474,31 +552,64 @@ def _prepare_kv_cache_config( def _build_kv_cache_kwargs( self, - kv_ref: Optional[KVPagedResourceHandler], kv_managed: ResourceHandlerDict, kv_cache_config: KvCacheConfig, + pool_configurations: Optional[List[PoolConfiguration]] = None, ) -> Dict: """Build common kwargs for KVCacheManager or MambaHybridCacheManager. + With per-pool ``PoolConfiguration`` carried through to the C++ ctor, + a single manager can host pools with mixed shapes. We pass: + + - ``head_dim`` / ``dtype`` as scalar defaults (fall-back values for + managers that run in uniform-shape mode). We pick the maximum + head_dim and the dtype of the first pool so the default is a sane + upper bound on per-pool memory accounting. + - ``pool_configurations`` as the per-pool config list; the C++ ctor + uses it to build pools with the correct shapes. + Args: - kv_ref: Reference KV handler defining head_dim and dtype, or None. kv_managed: Dict of KV resources to be managed. kv_cache_config: Configured KvCacheConfig. + pool_configurations: One PoolConfiguration per pool, in pool-index + order. Empty / None means uniform shape. Returns: Dict of kwargs suitable for both KVCacheManager and MambaHybridCacheManager. """ + pool_configurations = list(pool_configurations) if pool_configurations else [] + # create arguments first that differ whether we have managed kv caches or not - kv_cache_kwargs = {} + kv_cache_kwargs: Dict = {} if kv_managed: - kv_cache_type = CacheTypeCpp.SELFKONLY if kv_ref.kv_factor == 1 else CacheTypeCpp.SELF + # kv_factor is uniform across the managed set: __init__ asserts + # kv_factor in {1, 2}, and AutoDeploy only mixes kv_factor=2 layers + # in the same model today. We take the first handler's value. + ref_handler = next(iter(kv_managed.values())) + kv_cache_type = ( + CacheTypeCpp.SELFKONLY if ref_handler.kv_factor == 1 else CacheTypeCpp.SELF + ) + # Default head_dim: the largest across all pools, so any layer + # that falls back to the scalar gets a safe upper bound. + default_head_dim = ( + max(pc.head_dim for pc in pool_configurations) + if pool_configurations + else ref_handler.head_dim + ) + # Default dtype: dtype of the first pool. + default_dtype = ( + pool_configurations[0].dtype + if pool_configurations + else torch_dtype_to_binding(ref_handler.dtype) + ) kv_cache_kwargs.update( { "kv_cache_type": kv_cache_type, "num_layers": len(kv_managed), "num_kv_heads": [h.num_kv_heads for h in kv_managed.values()], - "head_dim": kv_ref.head_dim, - "dtype": torch_dtype_to_binding(kv_ref.dtype), + "head_dim": default_head_dim, + "dtype": default_dtype, + "pool_configurations": pool_configurations, } ) else: @@ -509,6 +620,7 @@ def _build_kv_cache_kwargs( "num_kv_heads": [1], "head_dim": 1, "dtype": DataType.HALF, + "pool_configurations": [], } ) # remaining arguments are the same for both cases @@ -610,6 +722,12 @@ def _create_and_assign_state_views( def _assign_kv_cache_views(self, kv_managed: Dict[str, KVPagedResourceHandler]) -> int: """Retrieve and assign buffer views for managed KV paged resources. + ``get_buffers`` on ``self._kv_cache_manager`` is per-layer-aware: it + reads the layer's head_dim from the underlying C++ pool configuration + when mixed-shape pools are present (Gemma4-style VSWA), and falls back + to the manager-level scalar otherwise. We can therefore use a single + call regardless of single-pool vs. multi-pool deployments. + Args: kv_managed: Dict of KV resources managed by the cache manager. @@ -675,13 +793,73 @@ def _allocate_unmanaged_state_resource(self, handler: StateResourceHandler) -> t dtype=handler.dtype, ) + def _has_swa_window(self, pool_configurations: List[PoolConfiguration]) -> bool: + """Return True if any pool's window is smaller than max_seq_len (i.e. SWA pool exists).""" + max_seq_len = self.info.max_seq_len + return any(pc.window_size < max_seq_len for pc in pool_configurations) + + def _compute_total_token_budget( + self, + kv_managed: ResourceHandlerDict, + pool_configurations: List[PoolConfiguration], + total_max_tokens: Optional[int], + ) -> Optional[int]: + """Compute the total max_tokens budget for the unified KVCacheManager. + + With a single C++ manager hosting all pools, ``BaseKVCacheManager:: + calculateMaxNumBlocks`` does the per-pool split internally using each + pool's ``PoolConfiguration``. We just need to compute the total token + budget; the C++ side derives N (max concurrent sequences) from the + total byte budget divided by the combined per-sequence cost across + all pools, then gives each pool N × its per-window tokens. + + We retain a Python-side cap to keep the budget feasible during prefill + warmup before the C++ side has a chance to validate against + ``calculateMaxNumBlocks``. + + Args: + kv_managed: All managed KV layers. + pool_configurations: Per-pool configurations, used here purely to + detect whether at least one SWA pool exists. + total_max_tokens: Caller-provided budget, or None to defer to + ``free_gpu_memory_fraction``. + + Returns: + Total max_tokens for the manager, or None to defer to the + ``free_gpu_memory_fraction`` path. + """ + # If the caller didn't pass a budget and there's no SWA pool, defer to + # free_gpu_memory_fraction inside the manager — same as the legacy path. + if total_max_tokens is None and not self._has_swa_window(pool_configurations): + return None + + if total_max_tokens is None: + # If the user already pinned max_tokens via KvCacheConfig, keep it: + # _prepare_kv_cache_config will preserve that value untouched. + if self._kv_cache_config_original.max_tokens is not None: + return None + # SWA present but no total budget — conservative single-sequence estimate. + # The C++ side will still bound this against available memory. + return self.info.max_seq_len + + tpb = self.info.tokens_per_block + # Cap at max_batch_size × max_seq_len (can't need more than one + # full sequence per slot during prefill). + capped = min(total_max_tokens, self.info.max_batch_size * self.info.max_seq_len) + # Floor: at least one block per sequence for warmup feasibility. + min_tokens = self.info.max_batch_size * tpb + return max(capped, min_tokens) + def _create_kv_cache_manager(self, max_tokens: Optional[int] = None) -> Dict: - """Create KVCacheManager or MambaHybridCacheManager with standard layout. + """Create a single KVCacheManager that hosts every KV pool. For paged resources (KVPagedResourceHandler): - - Uses the first KVPagedResourceHandler's head_dim and dtype as reference - - Compatible resources (matching head_dim and dtype) go into KVCacheManager - - Incompatible resources are allocated locally via handler.allocate() + - All managed layers are passed to one ``KVCacheManager`` instance. + - One ``PoolConfiguration`` per pool flows through the + ``pool_configurations`` ctor kwarg; the C++ side routes each layer + to the correct pool via its ``mLayerToWindowSize`` map. + - Cross-pool admission, event flush, disagg transfer, and the shared + radix tree are all handled in C++. For state resources (SSMResourceHandler, CausalConvResourceHandler, StateResourceHandler): - SSMResourceHandler maps to MambaHybridCacheManager's ssm_states buffer @@ -699,20 +877,33 @@ def _create_kv_cache_manager(self, max_tokens: Optional[int] = None) -> Dict: 1. the final number of tokens is synced (min) across ranks 2. rounding for getting a multiple of tokens_per_block """ - # 1. Identify managed resources - kv_ref, kv_managed = self._identify_managed_kv_resources() + # 1. Identify managed resources and per-pool configurations + kv_managed, pool_configurations = self._identify_managed_kv_resources() + ssm_ref, ssm_managed, ssm_spec, conv_ref, conv_managed, conv_spec = ( self._identify_managed_state_resources() ) + has_state_resources = ssm_managed or conv_managed - # 2. Prepare configuration - kv_cache_config = self._prepare_kv_cache_config(max_tokens, kv_managed) - kv_cache_kwargs = self._build_kv_cache_kwargs(kv_ref, kv_managed, kv_cache_config) + # 2. Compute the total token budget; the C++ side splits it across pools. + total_max_tokens = self._compute_total_token_budget( + kv_managed, pool_configurations, max_tokens + ) - # 3. Create cache manager (delegate to state helper if state resources exist) - has_state_resources = ssm_managed or conv_managed - if has_state_resources: - # NOTE: +1 for cuda graph padding + kv_cache_config = self._prepare_kv_cache_config(total_max_tokens, kv_managed) + kv_cache_kwargs = self._build_kv_cache_kwargs( + kv_managed, + kv_cache_config, + pool_configurations=pool_configurations, + ) + + # 3. Build the unified manager (single instance covers every pool). + if not kv_managed and not has_state_resources: + # Pure dummy manager: no KV layers, no state — used for state-only or + # cache-less models. We still need a manager so PyExecutor has a + # handle to call. + self._kv_cache_manager = KVCacheManager(**kv_cache_kwargs) + elif has_state_resources: kv_cache_kwargs["max_batch_size"] = self.info.max_num_state_slots self._kv_cache_manager, _ = self._create_and_assign_state_views( kv_cache_kwargs, @@ -724,18 +915,45 @@ def _create_kv_cache_manager(self, max_tokens: Optional[int] = None) -> Dict: conv_spec, ) else: - # No typed state resources - use pure KVCacheManager self._kv_cache_manager = KVCacheManager(**kv_cache_kwargs) - # 4. Store tuned config + ad_logger.info( + f"KV manager: {len(kv_managed)} layers across " + f"{len(pool_configurations)} pool(s); " + f"pool_configurations={pool_configurations}, " + f"max_attention_window={kv_cache_config.max_attention_window}, " + f"max_tokens={total_max_tokens}" + ) + + # 4. Store tuned config (mirrors the kv_cache_config used at ctor time). self._kv_cache_config_tuned = kv_cache_config - # 5. Assign KV views (compute block_offset_multiplier from first view's strides) - block_offset_multiplier = self._assign_kv_cache_views(kv_managed) + # 5. Refresh per-pool window list from the finalized manager: KVCacheManager + # may clamp each window to min(window, max_seq_len) during construction, so + # the transform-time _kv_group_windows can become stale. Downstream callers + # (ad_executor.get_cache_indices(window_size=...)) need the post-clamp + # values to hit the correct pool key in C++. + if kv_managed and self._kv_group_windows: + manager_windows = list(dict.fromkeys(self._kv_cache_manager.max_attention_window_vec)) + if manager_windows and manager_windows != self._kv_group_windows: + ad_logger.info( + f"Refreshing kv_group_windows from manager: " + f"{self._kv_group_windows} -> {manager_windows}" + ) + self._kv_group_windows = manager_windows - # 6. Update cache information (resize cache_loc, set max_seq_info with all max sizes) + # 6. Assign KV views and update cache information. + block_offset_multiplier = 0 + if kv_managed: + block_offset_multiplier = self._assign_kv_cache_views(kv_managed) + + num_blocks = getattr( + self._kv_cache_manager, + "blocks_in_primary_pool", + self._kv_cache_manager.get_max_resource_count(), + ) self.info.update_cache_information( - num_blocks=self._kv_cache_manager.blocks_in_primary_pool, + num_blocks=num_blocks, block_offset_multiplier=block_offset_multiplier, ) @@ -748,11 +966,11 @@ def _create_kv_cache_manager(self, max_tokens: Optional[int] = None) -> Dict: self._clear_caches, ) - # 9. Compute final token count and cache statistics + # 8. Compute final token count and cache statistics max_resource_count = self._kv_cache_manager.get_max_resource_count() max_tokens_final = max_resource_count * self._kv_cache_manager.tokens_per_block - # 10. Collect statistics of different types of resources + # 9. Collect statistics of different types of resources num_state_total = sum( 1 for h in self._resource_lookup.values() if isinstance(h, StateResourceHandler) ) @@ -871,12 +1089,16 @@ def resize_kv_cache_manager(self, mem_exclude: int = 0) -> None: This implements the two-phase approach: after running a forward pass during estimation to allocate intermediate memory, call this method to recreate the cache manager. - The new manager will compute optimal capacity based on current free GPU memory. + + For multi-pool (dual head_dim): the C++ + ``BaseKVCacheManager::calculateMaxNumBlocks`` distributes the budget + across pools using the per-window head_dim/dtype overrides — the + Python side just provides the total token budget. """ if not self.needs_resize(): return - # Calculate bytes-per-token for paged (resizable) resources + # Calculate bytes-per-token for ALL paged resources (across all groups) paged_bytes_per_token = sum( h.bytes_per_token for h in self._resource_lookup.values() if h.is_paged ) @@ -895,14 +1117,14 @@ def resize_kv_cache_manager(self, mem_exclude: int = 0) -> None: _, free_mem, *_ = get_mem_info(empty_cache=True) # Compute available memory for paged caches - # Reserve space for non-paged caches and mem_exclude, then apply free_gpu_memory_fraction free_gpu_memory_fraction = self._kv_cache_config_original.free_gpu_memory_fraction mem_for_paged_optimal = ( free_mem - non_paged_bytes_total - mem_exclude ) * free_gpu_memory_fraction max_tokens_optimal = int(mem_for_paged_optimal // paged_bytes_per_token) - # Create new cache manager with optimal capacity + # Recreate the unified manager — the C++ side splits the total token + # budget across pools internally based on per-window head_dim/dtype. cache_stats = self._create_kv_cache_manager(max_tokens=max_tokens_optimal) max_tokens_final = cache_stats["max_tokens"] @@ -920,9 +1142,27 @@ def resize_kv_cache_manager(self, mem_exclude: int = 0) -> None: f"total={bytes_to(total_cache_bytes, unit='GB'):.2f}GB" ) + @property + def kv_group_windows(self) -> List[int]: + """Per-pool window sizes, in the order the C++ manager exposes them. + + Pool index == position in this list, matching the order of the + unified manager's ``pool_configurations`` list. + """ + return self._kv_group_windows + + def set_kv_groups(self, group_windows: List[int]) -> None: + """Store per-pool window sizes (called by the kvcache transform). + + ``group_windows[i]`` is the effective window of pool ``i``; the order + must match the unified ``KVCacheManager``'s pool ordering (i.e., the + insertion order of the per-window keys). + """ + self._kv_group_windows = list(group_windows) + @property def kv_cache_manager(self) -> Optional[KVCacheManager]: - """Return the KVCacheManager managing paged resources, or None if not initialized.""" + """Return the unified KVCacheManager, or None if not initialized.""" assert self._kv_cache_manager is not None, "KVCacheManager not initialized." return self._kv_cache_manager diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py index 2b32b1987d0e..79afb6f6ace4 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py @@ -14,7 +14,30 @@ # limitations under the License. -"""Graph transformation to automatically add kv cache into fused MHA op.""" +"""Graph transformation to automatically add kv cache into fused MHA op. + +The transform runs in two passes so that KV grouping is driven by a single +source of truth — ``KVPagedResourceHandler.__eq__``: + + Pass 1: Walk each source attention node, ask the backend descriptor for the + KV handler via ``get_cache_initializers``, and assign a ``group_idx`` by + comparing the handler against previously-seen handlers (`_find_or_add_group` + semantics). Same equivalence class → same group. Because the handler's + equality includes ``sliding_window``, same-head-dim layers with different + windows automatically land in different groups. + + Pass 2: For multi-group models, publish the per-group window sizes to the + interface (``set_kv_groups``) and to SequenceInfo + (``register_window_groups``), allocate per-group metadata placeholders for + groups ``1..N-1``, and insert cached attention ops with their layer's + group's metadata nodes. + +After the transform, ``group_idx == pool_idx`` holds everywhere: the executor +reads per-pool windows from ``cache_seq_interface.kv_group_windows`` and +issues per-window queries (``get_cache_indices(request, window_size=W)``) +against the unified ``KVCacheManager``; the C++ side routes each call to the +correct pool via its ``mLayerToWindowSize`` map. +""" import inspect import operator @@ -29,6 +52,7 @@ AttentionDescriptor, AttentionRegistry, Constant, + KVPagedResourceHandler, PrepareMetadataCallable, ) from ...custom_ops.semantic_mask_registry import SemanticMaskRegistry @@ -183,18 +207,96 @@ def _process_metadata_extra( gm, prep_meta_op, inputs_for_prep_meta, const_args, num_meta_out ) - def _process_metadata_host(self, cm: CachedSequenceInterface): - """Process the host-side prepare metadata function.""" + def _process_metadata_host( + self, + cm: CachedSequenceInterface, + handler_groups: Optional[List[KVPagedResourceHandler]] = None, + ): + """Process the host-side prepare metadata function. + + When ``handler_groups`` has 2+ entries and the backend's host-prep + declares ``pool_window_left``, registers the host-prep once per pool: + pool 0 uses unsuffixed arg names, pools 1..N-1 use ``_g{i}`` / + ``_g{i}_host`` suffixed names plus a closure that remaps those + suffixed kwargs back to the function's canonical parameter names and + binds the pool's ``pool_window_left``. Each invocation only refreshes + cached wrappers belonging to its pool. + """ prep_meta_host_op = self.attn_descriptor.get_host_prepare_metadata_function() if prep_meta_host_op is None: return - # Register the host-side prepare metadata function with SequenceInfo. - # Arg availability is validated by require_copy() inside register_host_prepare. sig = inspect.signature(prep_meta_host_op) - cm.info.register_host_prepare_for_attention_forward( - prep_meta_host_op, list(sig.parameters.keys()) - ) + sig_arg_names = list(sig.parameters.keys()) + + # Backend opts in to per-pool dispatch by declaring `pool_window_left`. + backend_supports_per_pool = "pool_window_left" in sig.parameters + num_groups = len(handler_groups) if handler_groups is not None else 0 + + if not backend_supports_per_pool or num_groups < 2: + # Register the host-side prepare metadata function with SequenceInfo. + # Arg availability is validated by require_copy() inside register_host_prepare. + cm.info.register_host_prepare_for_attention_forward( + prep_meta_host_op, + [name for name in sig_arg_names if name != "pool_window_left"], + ) + return + + # VSWA: register one host-prep per pool with per-pool args + bound + # pool_window_left. Mirrors the per-group prepare_extra_metadata wiring + # below (vswa_swappable_bases) to keep the two routing paths consistent. + # + # The framework's run_host_prepare_for_attention_forward calls each + # registered function as ``host_function(**{arg: get_arg(arg)})`` — i.e. + # the registered arg names become the *kwarg keys* at call time. For + # pool > 0 the source tensors live under suffixed names + # (``cache_loc_g1_host`` etc.) but the underlying host-prep function + # only declares the unsuffixed parameter names. Each per-pool + # registration therefore wraps the host-prep in a closure that remaps + # suffixed kwargs back to the function's canonical parameter names + # before delegating, and binds this pool's ``pool_window_left``. + swappable_bases = { + "cache_loc", + "cu_num_pages", + "last_page_len", + "seq_len_with_cache", + } + host_suffix = "_host" + + def _make_pool_host_prep(canonical_op, pool_window_left, kwarg_remap): + """Closure: rename suffixed kwargs → canonical params, bind pool_window_left.""" + + def _invoke(**kwargs): + canonical = {kwarg_remap.get(k, k): v for k, v in kwargs.items()} + return canonical_op(**canonical, pool_window_left=pool_window_left) + + return _invoke + + for gi, handler in enumerate(handler_groups): + per_group_args: List[str] = [] + kwarg_remap: Dict[str, str] = {} # suffixed → canonical + for arg_name in sig_arg_names: + if arg_name == "pool_window_left": + continue # bound by closure + if gi == 0: + per_group_args.append(arg_name) + continue + base = arg_name.removesuffix(host_suffix) + if base in swappable_bases: + is_host = arg_name.endswith(host_suffix) + suffixed = f"{base}_g{gi}{host_suffix if is_host else ''}" + per_group_args.append(suffixed) + kwarg_remap[suffixed] = arg_name + else: + per_group_args.append(arg_name) + + # FlashInfer's _to_flashinfer_window_left convention: + # window_left = sliding_window - 1 for SWA, -1 for full attention. + sw = handler.sliding_window + pool_window_left = sw - 1 if isinstance(sw, int) and sw > 0 else -1 + + bound = _make_pool_host_prep(prep_meta_host_op, pool_window_left, kwarg_remap) + cm.info.register_host_prepare_for_attention_forward(bound, per_group_args) def _process_cache_node(self, gm: GraphModule, cache_name: str) -> Node: """Process the cache nodes by inserting a cached attention replacement op.""" @@ -256,17 +358,46 @@ def _apply( # insert metadata computation and extract each argument as a node meta_nodes_extra = self._process_metadata_extra(gm, cm, source_attn_nodes[0]) - # Register host-side prepare_metadata function for attention descriptor. - self._process_metadata_host(cm) + # Seed KvCacheConfig.max_attention_window from per-layer sliding_window + # annotations so the rest of the KV-cache config plumbing (e.g. + # downstream C++ validators that inspect the vector) sees the intended + # per-layer windows. The per-group KVCacheManager pools are configured + # separately in _prepare_kv_cache_config, using each group's reference + # handler. + per_layer_sliding_windows = [] + for attn_node in source_attn_nodes: + (sw,) = extract_op_args(attn_node, "sliding_window") + per_layer_sliding_windows.append(sw) + + has_any_sliding_window = any( + isinstance(sw, int) and sw > 0 for sw in per_layer_sliding_windows + ) + if cm.kv_cache_config.max_attention_window is None and has_any_sliding_window: + max_attention_window = [ + sw if isinstance(sw, int) and sw > 0 else cm.info.max_seq_len + for sw in per_layer_sliding_windows + ] + cm.update_kv_cache_config(max_attention_window=max_attention_window) + + # --- Pass 1: register resources and assign per-layer group_idx --- + # Group identity comes from KVPagedResourceHandler.__eq__ (which + # includes sliding_window). A group IS a pool IS a metadata set. + from ...custom_ops.attention_interface import KVPagedResourceHandler + + handler_groups: list[KVPagedResourceHandler] = [] + per_layer_group_idx: list[int] = [] + group_idx_by_layer_idx: dict[int, int] = {} - # replace fused attention node with attention node that has kv cache num_cached_attn_replacements = 0 cache_nodes_by_layer_idx = {} semantic_mask_cache: Dict[Node, Node] = {} + # Collect per-layer info for the second pass (node insertion). + # Tuple layout: + # (attn_node, qkv, cache_in_nodes, constants, group_idx, prepared_mask) + layer_infos: list[tuple] = [] + for attn_node in source_attn_nodes: - # pick out GEMMs qkv = attn_node.args[: attn_descriptor.get_num_qkv_args()] - layer_idx = attn_descriptor.get_layer_idx(attn_node) shared_kv_source_layer_idx = attn_descriptor.get_shared_kv_source_layer_idx(attn_node) @@ -287,23 +418,36 @@ def _apply( f"Missing shared-KV source layer {shared_kv_source_layer_idx}." ) cache_in_nodes = cache_nodes_by_layer_idx[shared_kv_source_layer_idx] + # Shared-KV layers inherit their source layer's group + group_idx = group_idx_by_layer_idx.get(shared_kv_source_layer_idx, 0) else: - # setup + store cache initializers and caches as input nodes if layer_idx is not None and layer_idx in cache_nodes_by_layer_idx: raise RuntimeError( f"Duplicate KV cache owner detected for layer {layer_idx}. " "Each non-shared attention layer must own exactly one cache." ) cache_in_nodes = [] + group_idx = 0 for k, resource_handler in attn_descriptor.get_cache_initializers( attn_node, cm.kv_cache_config ).items(): resource_name = cm.add_resource(k, resource_handler) cache_in_nodes.append(self._process_cache_node(gm, resource_name)) + # Determine group from handler equality + if isinstance(resource_handler, KVPagedResourceHandler): + for gi, ref in enumerate(handler_groups): + if resource_handler == ref: + group_idx = gi + break + else: + group_idx = len(handler_groups) + handler_groups.append(resource_handler) if layer_idx is not None: cache_nodes_by_layer_idx[layer_idx] = cache_in_nodes + group_idx_by_layer_idx[layer_idx] = group_idx + + per_layer_group_idx.append(group_idx) - # allow backend-specific prep before constants are extracted attn_descriptor.prepare_node_for_cache_insertion(gm, attn_node) prepared_mask = self._process_semantic_mask( @@ -316,20 +460,151 @@ def _apply( ) constants = attn_descriptor.get_constants(attn_node) - # insert cached attention replacement op + layer_infos.append( + (attn_node, qkv, cache_in_nodes, constants, group_idx, prepared_mask) + ) + + # --- Group setup: register metadata groups and create graph placeholders --- + # Register every group (including the single-pool case) so downstream + # consumers see a consistent representation: num_pools == num_groups. + # Per-group graph placeholders are still only created for groups 1..N-1 + # because group 0 reuses the legacy unsuffixed tensors. + num_groups = len(handler_groups) + is_multi_group = num_groups >= 2 + vswa_group_nodes: dict[int, dict[str, "Node"]] = {} + # Per-group extra-metadata: group 0 reuses the previously-built nodes; + # groups 1..N-1 get their own prepare-extra call wired to per-group + # swappable inputs. This is required because some backends' prepare-extra + # ops (e.g. Triton's, which consumes seq_len_with_cache) would otherwise + # produce group-0 outputs that silently feed every layer. + meta_nodes_extra_by_group: Dict[int, List[Node]] = {0: meta_nodes_extra} + host_suffix = "_host" + + has_unmanaged_paged = num_groups == 0 and any( + h.is_paged for h in cm._resource_lookup.values() + ) + + if num_groups >= 1: + group_windows = [ + h.sliding_window if h.sliding_window > 0 else cm.info.max_seq_len + for h in handler_groups + ] + cm.info.register_window_groups(group_windows) + cm.set_kv_groups(group_windows) + self._process_metadata_host(cm, handler_groups=handler_groups) + elif has_unmanaged_paged: + # MLA-only (and any future paged-handler family that does not contribute + # to handler_groups): no KVPagedResourceHandler entries, but at least one + # paged cache still consumes cache_loc/cu_num_pages metadata. Register a + # single full-attention pool so ad_executor.py:740 stages those tensors + # instead of passing None to nest_sequences. + # TODO: unify paged-handler grouping (KV + MLA) under a shared abstraction + # so the predicate at line 361 covers all paged handlers naturally. + group_windows = [cm.info.max_seq_len] + cm.info.register_window_groups(group_windows) + cm.set_kv_groups(group_windows) + self._process_metadata_host(cm, handler_groups=[]) + else: + # No paged caches at all (cache-less or state-only models like Mamba + # without attention). nest_sequences will correctly skip cache_loc + # staging because no kernel in the graph consumes it. Intentional no-op. + ad_logger.debug( + "kvcache transform: no paged KV resources registered; " + "cache_loc staging will be skipped. Expected for state-only or " + "cache-less models." + ) + self._process_metadata_host(cm, handler_groups=[]) + + if is_multi_group: + # Names that are routed per-group end-to-end. seq_len_with_cache is + # in this set because under SWA front-eviction the live window's + # length diverges from the global (input_pos + seq_len); the kernel, + # the prepare-extra op, and the host prepare all need the + # window-capped value. + vswa_swappable_bases = { + "cache_loc", + "cu_num_pages", + "last_page_len", + "seq_len_with_cache", + } + + # Create per-group graph placeholders for groups 1..N-1 + std_arg_names = self.attn_descriptor.get_standard_metadata_args() + for gi in range(1, num_groups): + vswa_group_nodes[gi] = {} + for arg_name in std_arg_names: + base = arg_name.removesuffix(host_suffix) + if base in vswa_swappable_bases: + is_host = arg_name.endswith(host_suffix) + group_arg = f"{base}_g{gi}{host_suffix if is_host else ''}" + vswa_group_nodes[gi][arg_name] = self._add_or_retrieve_input( + gm, cm, group_arg + ) + + # Per-group prepare-extra-metadata: re-invoke the op with this + # group's swappable inputs so each non-zero group's downstream + # consumers (e.g. update_paged_kv_cache write positions) reflect + # the window-capped view. Skipped when the backend has no extra-op. + prep_meta_op, num_meta_out, const_args_extra = ( + self.attn_descriptor.get_prepare_extra_metadata_info(source_attn_nodes[0]) + ) + if prep_meta_op is not None and num_meta_out > 0: + op_arg_names = [ + arg.name + for arg in get_op_schema(prep_meta_op).arguments + if arg.name in cm.info.available_args + ] + for gi in range(1, num_groups): + group_inputs: List[Node] = [] + for arg_name in op_arg_names: + base = arg_name.removesuffix(host_suffix) + if base in vswa_swappable_bases: + is_host = arg_name.endswith(host_suffix) + group_arg = f"{base}_g{gi}{host_suffix if is_host else ''}" + group_inputs.append(self._add_or_retrieve_input(gm, cm, group_arg)) + else: + group_inputs.append(self._add_or_retrieve_input(gm, cm, arg_name)) + meta_nodes_extra_by_group[gi] = self._insert_extra_metadata_op( + gm, prep_meta_op, group_inputs, const_args_extra, num_meta_out + ) + else: + # Backend has no extra-op: every group gets an empty list (which + # is what the original meta_nodes_extra would have been anyway). + for gi in range(1, num_groups): + meta_nodes_extra_by_group[gi] = [] + + # --- Pass 2: insert cached attention nodes with correct group metadata --- + for ( + attn_node, + qkv, + cache_in_nodes, + constants, + group_idx, + prepared_mask, + ) in layer_infos: + layer_meta_nodes_std = meta_nodes_std + if is_multi_group and group_idx > 0: + std_arg_names = self.attn_descriptor.get_standard_metadata_args() + layer_meta_nodes_std = list(meta_nodes_std) + for arg_pos, arg_name in enumerate(std_arg_names): + if arg_name in vswa_group_nodes.get(group_idx, {}): + layer_meta_nodes_std[arg_pos] = vswa_group_nodes[group_idx][arg_name] + + layer_meta_nodes_extra = meta_nodes_extra_by_group.get(group_idx, meta_nodes_extra) + self._insert_cached_attn_node( gm, attn_node, attn_descriptor.get_cached_attention_op(), qkv, - meta_nodes_std, - meta_nodes_extra, + layer_meta_nodes_std, + layer_meta_nodes_extra, cache_in_nodes, constants, prepared_mask, ) - num_cached_attn_replacements += 1 + num_cached_attn_replacements = len(layer_infos) info = TransformInfo( skipped=False, diff --git a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py index 5aeeeadfe861..50edbee51db7 100644 --- a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @@ -30,7 +30,8 @@ from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.resource_manager import ( - BaseResourceManager, CacheTypeCpp, DataType, KVCacheManager, get_pp_layers) + BaseResourceManager, CacheTypeCpp, DataType, KVCacheManager, + PoolConfiguration, get_pp_layers) from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._utils import (nvtx_range, prefer_pinned, torch_dtype_to_binding) @@ -892,6 +893,10 @@ def __init__( model_type: str = "nemotron_hybrid", is_draft: bool = False, use_replay_state_update: bool = False, + # Per-pool configurations forwarded to the C++ KVCacheManager ctor. + # Lets a single manager host pools with mixed shapes (e.g. Gemma4 + # hybrid attention). See KVCacheManager.__init__. + pool_configurations: Optional[List[PoolConfiguration]] = None, ) -> None: # mamba hybrid cache requires block reuse to be disabled in KV cache config @@ -941,6 +946,7 @@ def __init__( is_estimating_kv_cache=is_estimating_kv_cache, execution_stream=execution_stream, is_draft=is_draft, + pool_configurations=pool_configurations, ) def prepare_resources(self, scheduled_batch: ScheduledRequests): @@ -1207,6 +1213,7 @@ def __init__( is_estimating_kv_cache=is_estimating_kv_cache, is_draft=is_draft, linear_attention_metadata=self.linear_attention_metadata, + **kwargs, ) assert self.local_num_mamba_layers > 0, "At least one mamba layer is required" diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index 54f23ddda8ac..8f46d88794cd 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -7,6 +7,7 @@ import os from abc import ABC, abstractmethod from collections import OrderedDict, defaultdict, deque +from dataclasses import dataclass from typing import (TYPE_CHECKING, Dict, Iterable, List, NamedTuple, Optional, Sequence, Set, Tuple, Union) @@ -59,6 +60,7 @@ BufferManagerCpp = tensorrt_llm.bindings.internal.runtime.BufferManager KVCacheManagerCpp = tensorrt_llm.bindings.internal.batch_manager.KVCacheManager +PoolConfigurationCpp = tensorrt_llm.bindings.internal.batch_manager.PoolConfiguration CacheTypeCpp = tensorrt_llm.bindings.internal.batch_manager.CacheType ModelConfigCpp = tensorrt_llm.bindings.ModelConfig DataType = tensorrt_llm.bindings.DataType @@ -76,6 +78,22 @@ int]] # window_size -> (blocks_in_primary_pool, blocks_in_secondary_pool) +@dataclass +class PoolConfiguration: + """Configuration of a single KV pool. + + A pool is uniquely described by its attention ``window_size``, the + ``head_dim`` of the layers it serves, and the cache element ``dtype``. + A KVCacheManager is constructed from a ``list[PoolConfiguration]`` -- + one entry per pool the manager hosts. Multiple entries with the same + ``window_size`` are legal and reserved for future multi-pool-per-window + cases (e.g. mixed head_dim within a single window). + """ + window_size: int + head_dim: int + dtype: "DataType" + + class ResourceManagerType(enum.Enum): KV_CACHE_MANAGER = "KV_CACHE_MANAGER" DRAFT_KV_CACHE_MANAGER = "DRAFT_KV_CACHE_MANAGER" @@ -545,6 +563,14 @@ def __init__( is_estimating_kv_cache: bool = False, execution_stream: Optional[torch.cuda.Stream] = None, linear_attention_metadata: Optional[LinearAttentionMetadata] = None, + # Per-pool configuration list forwarded to the C++ ctor. One entry + # per pool the manager will host; each entry pins (window_size, + # head_dim, dtype) for that pool. None / empty = uniform shape + # across all windows (default behavior); a single KVCacheManager can + # host pools with mixed shapes when a model has heterogeneous + # attention types (e.g. Gemma4 SWA head_dim=256 + full-attention + # head_dim=512). + pool_configurations: Optional[List[PoolConfiguration]] = None, **kwargs, ) -> None: self.mapping = mapping @@ -605,6 +631,21 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], self.num_kv_heads = num_kv_heads self.head_dim = head_dim + # Per-pool configuration list -- the source of truth for per-pool + # (window_size, head_dim, dtype). When non-empty, each pool may + # have its own shape that differs from the manager-level scalars + # (e.g. Gemma4 SWA head_dim=256 alongside full-attention head_dim=512). + # Empty list means uniform shape (every window uses self.head_dim / + # self.dtype). Each pool's window_size is remapped after window + # clamping in _validate_and_adjust_attention_windows; the pool + # *indices* stay stable across that rewrite. + self.pool_configurations: List[PoolConfiguration] = ( + list(pool_configurations) if pool_configurations else []) + # Layer -> pool_idx mapping, built once after max_attention_window_vec + # is initialized below. This is the layer-centric replacement for any + # window-keyed shape dict: multi-pool-per-window is safe because pools + # are identified by index, not by window. + self._layer_to_pool_idx: Dict[int, int] = {} self.tokens_per_block = tokens_per_block self.max_seq_len = max_seq_len self.max_batch_size = max_batch_size @@ -639,6 +680,12 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], layer_mask=layer_mask, ) + # Now that max_attention_window_vec is known, build layer -> pool_idx + # from the (pre-clamp) pool_configurations. Stays valid through the + # window clamping below because that only rewrites per-pool + # window_size fields; pool indices don't shift. + self._layer_to_pool_idx = self._build_layer_to_pool_idx() + # Determine if this is VSWA (Variable Sliding Window Attention). # The `w > 0` check excludes LinearCacheType.RECURRENT_STATES sentinel # values (negative) used by hybrid linear attention models. @@ -702,7 +749,6 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], ), "calculate_max_num_blocks_for_vswa only accepts KvCacheConfig" blocks_per_window = self.calculate_max_num_blocks_for_vswa( kv_cache_config=kv_cache_config, - model_config=model_config, extra_cost_memory=0, ) if mapping.world_size > 1: @@ -753,7 +799,7 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], } # Validate and adjust attention windows against their upper bounds if needed - blocks_per_window, self.max_seq_len, self.max_attention_window_vec = self._validate_and_adjust_attention_windows( + blocks_per_window, self.max_seq_len, self.max_attention_window_vec, window_adjustments = self._validate_and_adjust_attention_windows( max_attention_window_vec=self.max_attention_window_vec, blocks_per_window=blocks_per_window, tokens_per_block=tokens_per_block, @@ -761,6 +807,23 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], max_beam_width=max_beam_width, ) + # Rewrite each pool's window_size to match the post-clamp window. + # Without this, a pool pinned to a pre-clamp window (e.g. 32768) + # would be silently dropped when the validator clamps the window + # down (e.g. to 16384), leaving the C++ side to fall back on the + # manager-level scalar -- which is exactly the heterogeneous case + # these per-pool configs exist to handle. Pool *indices* are + # preserved so self._layer_to_pool_idx stays valid. + if window_adjustments and self.pool_configurations: + self.pool_configurations = [ + PoolConfiguration( + window_size=window_adjustments.get(pc.window_size, + pc.window_size), + head_dim=pc.head_dim, + dtype=pc.dtype, + ) for pc in self.pool_configurations + ] + if kv_cache_type != CacheTypeCpp.SELF: assert len( blocks_per_window @@ -780,6 +843,18 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], ) logger.info(f"[KVCacheManager] execution_stream: {self._stream}") logger.info(f"[KVCacheManager] blocks_per_window: {blocks_per_window}") + + # The Python @dataclass PoolConfiguration is a distinct type from the + # nanobind C++ PoolConfiguration (Python uses ``head_dim``; C++ uses + # ``size_per_head``). Translate at the C++ boundary so nanobind can + # dispatch the ctor. + pool_configurations_cpp = [ + PoolConfigurationCpp(window_size=pc.window_size, + size_per_head=pc.head_dim, + dtype=pc.dtype) + for pc in self.pool_configurations + ] + kwargs = { 'num_kv_heads_per_layer': self.num_kv_heads_per_layer, 'size_per_head': head_dim, @@ -804,6 +879,9 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], 'indexer_k_cache_index_head_dim': indexer_k_cache_index_head_dim, 'indexer_k_cache_use_fp4': indexer_k_cache_use_fp4, 'linear_attention_metadata': linear_attention_metadata, + # Forward the (possibly remapped) per-pool configurations. + # window_size values are aligned with the post-clamp sizes. + 'pool_configurations': pool_configurations_cpp, } if self.event_buffer_max_size > 0: @@ -1395,6 +1473,26 @@ def get_cache_indices(self, assert len(result) == 1 return result[0] + def get_num_front_blocks_removed(self, + request_id: int, + window_size: Optional[int] = None) -> int: + """Get the number of front blocks evicted by SWA for a sequence. + + Args: + request_id: The request id. + window_size: Optional window size. When supplied, returns the + per-window eviction count (zero for non-SWA windows). When + omitted, defaults to ``self.max_attention_window_vec[0]`` — + this matches the historical single-pool behavior for + callers that never thought about windows, while keeping the + C++ contract uniformly per-window. VSWA callers should + always pass ``window_size`` explicitly. + """ + if window_size is None: + window_size = self.max_attention_window_vec[0] + return self.impl.get_num_front_blocks_removed(request_id, + window_size=window_size) + def unpin_blocks_by_id(self, kv_cache_block_id: int): self.impl.unpin_blocks_by_id(kv_cache_block_id) @@ -1424,15 +1522,20 @@ def get_batch_cache_indices( self, request_ids: List[int], layer_idx: Optional[int] = None, + window_size: Optional[int] = None, ) -> List[List[int]]: - if layer_idx is None: - if len(self.max_attention_window_vec) > 1: - raise ValueError("layer_idx must be provided for VSWA") - window_size = self.max_attention_window_vec[0] - else: - layer_offset = self.layer_offsets[layer_idx] - window_size = self.max_attention_window_vec[layer_offset % len( - self.max_attention_window_vec)] + if window_size is None: + if layer_idx is None: + if len(self.max_attention_window_vec) > 1: + raise ValueError( + "layer_idx or window_size must be provided for VSWA") + window_size = self.max_attention_window_vec[0] + else: + layer_offset = self.layer_offsets[layer_idx] + # Explicit layer_offset -> window_size mapping (no modulo + # masking length mismatches between pattern and num_local_layers). + window_size = self._get_layer_offset_to_window_size( + )[layer_offset] result = self.impl.get_batch_cache_block_ids(request_ids, window_size) for i in range(len(result)): @@ -1490,10 +1593,19 @@ def get_buffers(self, Note that different attention backend/implementation can have different KV layouts, "kv_layout" should be set accordingly to avoid surprises. + + Per-layer head_dim: when the underlying C++ manager hosts multiple pools with + distinct head_dim (e.g., Gemma4 SWA head_dim=256 alongside full-attention + head_dim=512), this method reads the layer's effective head_dim from the + layer's assigned ``PoolConfiguration`` rather than the manager-level + scalar. Single-pool managers fall back to ``self.head_dim``. ''' layer_offset = self.layer_offsets[layer_idx] result = self.impl.get_primary_pool_data(layer_offset) + pool = self.get_pool_for_layer(layer_offset) + layer_head_dim = pool.head_dim if pool else self.head_dim + assert kv_layout in ["NHD", "HND"], f"Unsupported kv_layout: {kv_layout}" if kv_layout == "NHD": @@ -1502,7 +1614,7 @@ def get_buffers(self, self.kv_factor, self.tokens_per_block, self.num_kv_heads_per_layer[layer_offset], - self.head_dim, + layer_head_dim, ) else: return result.reshape( @@ -1510,7 +1622,7 @@ def get_buffers(self, self.kv_factor, self.num_kv_heads_per_layer[layer_offset], self.tokens_per_block, - self.head_dim, + layer_head_dim, ) def get_indexer_k_cache_pool_data(self, layer_idx: int) -> torch.Tensor: @@ -1595,6 +1707,138 @@ def get_iteration_stats(self): def rewind_kv_cache(self, request: LlmRequest, rewind_len: int): self.impl.rewind_kv_cache(request.py_request_id, rewind_len) + def calculate_cache_size_per_token(self, + layers: Set[int], + window_size: Optional[int] = None + ) -> int: + """Compute the (raw, dtype-agnostic) KV cache size per token for a set of layers. + + head_dim is resolved per-layer via ``get_pool_for_layer``: each + layer's assigned ``PoolConfiguration`` supplies its ``head_dim``. + When the manager runs in uniform-shape mode (no + ``pool_configurations``), every layer uses ``self.head_dim``. + + Args: + layers: Set of layer offsets. + window_size: Accepted for backward compatibility; ignored. + Per-layer pool lookup already covers the homogeneous case. + + Returns: + cache size per token (number of elements, not bytes). + """ + del window_size # kept for compat; resolution is now per-layer + if not self.pool_configurations: + total_kv_heads = sum(self.num_kv_heads_per_layer[i] for i in layers) + return total_kv_heads * self.kv_factor * self.head_dim + + total = 0 + for i in layers: + pool = self.get_pool_for_layer(i) + layer_head_dim = pool.head_dim if pool else self.head_dim + total += self.num_kv_heads_per_layer[i] * layer_head_dim + return total * self.kv_factor + + def _calculate_cache_bytes_per_token_for_layers( + self, + layers: Set[int], + dtype_default: Optional[DataType] = None) -> int: + """Compute KV cache bytes per token for a set of layers. + + Resolves head_dim and dtype per layer through + ``get_pool_for_layer``; layers whose manager has no + ``pool_configurations`` fall back to ``self.head_dim`` and + ``dtype_default`` (or ``self.dtype``). Handles NVFP4 + scaling-factor overhead, computed per dtype. + """ + if dtype_default is None: + dtype_default = self.dtype + + total_bytes = 0 + for i in layers: + pool = self.get_pool_for_layer(i) + layer_head_dim = pool.head_dim if pool else self.head_dim + layer_dtype = pool.dtype if pool else dtype_default + layer_elements = (self.num_kv_heads_per_layer[i] * self.kv_factor * + layer_head_dim) + layer_bytes = get_size_in_bytes(layer_elements, layer_dtype) + if layer_dtype == DataType.NVFP4: + layer_bytes += KVCacheManager.calculate_scaling_factor_size_bytes( + layer_elements, + quant_vector_size=16, + scaling_factor_dtype=DataType.FP8) + total_bytes += layer_bytes + return total_bytes + + def _build_layer_to_pool_idx(self) -> Dict[int, int]: + """Build the layer_offset -> pool_idx mapping for self.pool_configurations. + + Today, pool assignment is implicit via window_size: each pool has a + unique window_size and a layer joins the pool whose window matches + its effective window. Multiple pools sharing a window_size would + require an explicit per-layer pool index from the caller (a future + ``pool_idx_per_layer`` ctor parameter); this helper raises instead + of silently collapsing them. + """ + if not self.pool_configurations: + return {} + window_to_pool_idx: Dict[int, int] = {} + for idx, pc in enumerate(self.pool_configurations): + if pc.window_size in window_to_pool_idx: + raise RuntimeError( + f"Multiple PoolConfigurations share window_size={pc.window_size}. " + "Multi-pool-per-window requires an explicit layer->pool mapping, " + "which is not yet wired through KVCacheManager.__init__.") + window_to_pool_idx[pc.window_size] = idx + layer_offset_to_window_size = self._get_layer_offset_to_window_size() + return { + offset: window_to_pool_idx[w] + for offset, w in layer_offset_to_window_size.items() + } + + def get_pool_configuration(self, pool_idx: int) -> PoolConfiguration: + """Return the PoolConfiguration at ``pool_idx``.""" + return self.pool_configurations[pool_idx] + + def get_pool_for_layer(self, + layer_offset: int) -> Optional[PoolConfiguration]: + """Return the pool serving ``layer_offset``, or None for uniform managers. + + Layer-centric replacement for any window-keyed shape lookup: the + returned pool's ``head_dim`` and ``dtype`` are authoritative for the + given layer, regardless of how many pools share the layer's window. + """ + if not self.pool_configurations: + return None + pool_idx = self._layer_to_pool_idx.get(layer_offset) + if pool_idx is None: + return None + return self.pool_configurations[pool_idx] + + def _get_layer_offset_to_window_size(self) -> Dict[int, int]: + """Inverse of _get_window_size_to_layers: layer_offset -> window_size. + + Asserts every local layer is mapped exactly once. This is the + explicit, length-mismatch-safe replacement for + ``max_attention_window_vec[layer_offset % len(max_attention_window_vec)]`` + — that modulo silently masks length mismatches between the window + pattern and num_local_layers; this helper catches them via the + assert below. + """ + window_size_to_layers = self._get_window_size_to_layers() + layer_offset_to_window_size: Dict[int, int] = {} + for window_size, layer_offsets in window_size_to_layers.items(): + for layer_offset in layer_offsets: + assert layer_offset not in layer_offset_to_window_size, ( + f"layer_offset {layer_offset} mapped to multiple window " + f"sizes ({layer_offset_to_window_size[layer_offset]} and " + f"{window_size}) — window pattern is malformed.") + layer_offset_to_window_size[layer_offset] = window_size + assert len(layer_offset_to_window_size) == self.num_local_layers, ( + f"layer_offset_to_window_size covers " + f"{len(layer_offset_to_window_size)} layers but num_local_layers " + f"is {self.num_local_layers}.") + return layer_offset_to_window_size + def _get_window_size_to_layers(self) -> dict[int, list[int]]: """ Get the window size to layers mapping. @@ -1636,34 +1880,27 @@ def adjust_window_sizes_for_vswa( window_size_to_layers: Dict[int, List[int]], max_attention_window_vec: List[int], kv_cache_config: KvCacheConfig, - model_config: ModelConfigCpp, pool_memory_bytes: int, kv_factor: int, dtype: DataType, is_cross_attention: bool = False, + model_config: Optional[ModelConfigCpp] = None, ) -> Tuple[Dict[int, List[int]], List[int]]: assert is_cross_attention is False, 'Cross attention is not supported' max_tokens_from_config = kv_cache_config.max_tokens - def calculate_cache_size_per_token(layers: Set[int]) -> int: - # Same as BaseKVCacheManager::calculateCacheSizePerTokenForSingleWindowSize - total_kv_heads = sum(self.num_kv_heads_per_layer[i] for i in layers) - return total_kv_heads * kv_factor * model_config.head_size - - # Calculate the required memory bytes per sequence. + # Calculate the required memory bytes per sequence. Each window's + # bytes-per-token is computed with that window's effective head_dim + # / dtype (per-window override map), falling back to the manager + # scalars when no override is registered. required_mem_bytes_per_seq = 0 for window_size in sorted(window_size_to_layers): layers = window_size_to_layers[window_size] - cache_size_per_token = calculate_cache_size_per_token(layers) - cache_size_bytes_per_token = get_size_in_bytes( - cache_size_per_token, dtype) - if dtype == DataType.NVFP4: - cache_size_bytes_per_token += KVCacheManager.calculate_scaling_factor_size_bytes( - cache_size_per_token, - quant_vector_size=16, - scaling_factor_dtype=DataType.FP8) + cache_size_bytes_per_token = ( + self._calculate_cache_bytes_per_token_for_layers( + layers, dtype_default=dtype)) required_mem_bytes_per_seq += window_size * cache_size_bytes_per_token logger.info( f'Required memory per sequence: {required_mem_bytes_per_seq} bytes') @@ -1692,16 +1929,13 @@ def calculate_cache_size_per_token(layers: Set[int]) -> int: for window_size in sorted(window_size_to_layers): layers = window_size_to_layers[window_size] if remaining_mem_bytes > 0 and remaining_layers: - # Calculate cache size per token for remaining layers only - cache_size_per_token = calculate_cache_size_per_token( - remaining_layers) - cache_size_bytes_per_token = get_size_in_bytes( - cache_size_per_token, dtype) - if dtype == DataType.NVFP4: - cache_size_bytes_per_token += KVCacheManager.calculate_scaling_factor_size_bytes( - cache_size_per_token, - quant_vector_size=16, - scaling_factor_dtype=DataType.FP8) + # Calculate cache size per token for remaining layers only. + # ``remaining_layers`` may span multiple windows with + # different head_dim / dtype, so the helper resolves each + # layer's effective shape and dtype individually. + cache_size_bytes_per_token = ( + self._calculate_cache_bytes_per_token_for_layers( + remaining_layers, dtype_default=dtype)) logger.debug( f'Cache size per token for {len(remaining_layers)} layers: ' f'{cache_size_bytes_per_token} bytes') @@ -1843,7 +2077,7 @@ def _calculate_max_num_blocks_for_linear_attention( def calculate_max_num_blocks_for_vswa( self, kv_cache_config: KvCacheConfig, - model_config: Optional[ModelConfigCpp], + model_config: Optional[ModelConfigCpp] = None, extra_cost_memory: int = 0) -> dict[int, tuple[int, int]]: """ Currently, this function is added to support *ONLY* VSWA. @@ -1889,31 +2123,19 @@ def calculate_max_num_blocks_for_vswa( extra_cost_memory=extra_cost_memory, ) - # VSWA case: use C++ implementation for variable window sizes - if model_config is None: - raise ValueError( - "model_config is required for VSWA (Variable Sliding Window Attention)" - ) - assert model_config.layer_types is not None, "layer_types have to be set correctly for VSWA" - if self.is_vswa: - # Adjust the window sizes to fit the memory if even a single sequence - # cannot fit in the memory. - window_size_to_layers, max_attention_window_vec = self.adjust_window_sizes_for_vswa( - window_size_to_layers=window_size_to_layers, - max_attention_window_vec=self.max_attention_window_vec, - model_config=model_config, - kv_cache_config=kv_cache_config, - pool_memory_bytes=self._primary_pool_memory_bytes, - kv_factor=self.kv_factor, - dtype=self.dtype, - is_cross_attention=is_cross_attention, - ) - self.max_attention_window_vec = max_attention_window_vec - - def calculate_cache_size_per_token(layers: Set[int]) -> int: - # Same as BaseKVCacheManager::calculateCacheSizePerTokenForSingleWindowSize - total_kv_heads = sum(self.num_kv_heads_per_layer[i] for i in layers) - return total_kv_heads * self.kv_factor * model_config.head_size + # VSWA case: adjust window sizes via Python helper that derives + # head_dim from self. model_config is no longer required because + # head_size is read from self.head_dim, set during __init__. + window_size_to_layers, max_attention_window_vec = self.adjust_window_sizes_for_vswa( + window_size_to_layers=window_size_to_layers, + max_attention_window_vec=self.max_attention_window_vec, + kv_cache_config=kv_cache_config, + pool_memory_bytes=self._primary_pool_memory_bytes, + kv_factor=self.kv_factor, + dtype=self.dtype, + is_cross_attention=is_cross_attention, + ) + self.max_attention_window_vec = max_attention_window_vec logger.info( f"Primary pool memory bytes: {self._primary_pool_memory_bytes}") @@ -1944,9 +2166,11 @@ def calculate_cache_size_per_token(layers: Set[int]) -> int: blocks_per_window = {} for window_idx, (window_size, layers) in enumerate( sorted(window_size_to_layers.items())): - cache_size_per_token = calculate_cache_size_per_token(layers) - cache_size_bytes_per_token = get_size_in_bytes( - cache_size_per_token, self.dtype) + # Per-window head_dim and dtype (with scalar fallback) — needed + # for heterogeneous-attention models like Gemma4 where SWA and + # full-attention pools have different head_dim. + cache_size_bytes_per_token = ( + self._calculate_cache_bytes_per_token_for_layers(layers)) primary_tokens = self._primary_pool_memory_bytes * window_size_shares[ window_idx] / cache_size_bytes_per_token @@ -1983,7 +2207,7 @@ def _validate_and_adjust_attention_windows( tokens_per_block: int, max_seq_len: int, max_beam_width: int, - ) -> Tuple[BlocksPerWindow, int, List[int]]: + ) -> Tuple[BlocksPerWindow, int, List[int], Dict[int, int]]: """ Validate and adjust attention windows against their upper bounds if needed. If there is no adjustment, the returned max_attention_window_vec will be the same as the input. @@ -1995,7 +2219,12 @@ def _validate_and_adjust_attention_windows( max_seq_len: Maximum sequence length Returns: - Tuple of (adjusted_blocks_per_window, adjusted_max_seq_len, adjusted_max_attention_window_vec) + Tuple of (adjusted_blocks_per_window, adjusted_max_seq_len, + adjusted_max_attention_window_vec, window_adjustments). + window_adjustments maps pre_clamp -> post_clamp window size for + every window that was clamped (empty if nothing was adjusted) so + callers can rewrite their per-pool configurations to match the + post-clamp window keys. """ window_adjustments = {} # Validate each window size in blocks_per_window against its upper bound @@ -2037,9 +2266,9 @@ def _validate_and_adjust_attention_windows( adjusted_max_seq_len = max(adjusted_window_vec) logger.warning(f"Adjusted max_seq_len to {adjusted_max_seq_len}") - return adjusted_blocks_per_window, adjusted_max_seq_len, adjusted_window_vec + return adjusted_blocks_per_window, adjusted_max_seq_len, adjusted_window_vec, window_adjustments else: - return blocks_per_window, max_seq_len, max_attention_window_vec + return blocks_per_window, max_seq_len, max_attention_window_vec, {} def pin_blocks(self, request_id: int): self.impl.pin_blocks(request_id) diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py new file mode 100644 index 000000000000..2da593035874 --- /dev/null +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py @@ -0,0 +1,360 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SWA correctness tests for Triton paged attention. + +Two regimes that the existing kernel tests don't exercise: + +1. **Long prefill SWA** (no eviction yet): a single prefill longer than the + sliding window. The kernel must apply the SW mask within the chunk itself. + This is the lightweight regression that catches mask-math regressions + without spinning up a real KVCacheManager. + +2. **Front-eviction SWA** (eviction has happened): the shim hands the kernel + a window-coherent local-coord view — sliced cache_loc, window-capped + seq_len_with_cache. The kernel must produce the same output as SDPA on the + live window only. Pre-fix kernels that mixed coordinate spaces silently + dropped boundary pages and corrupted masks; under Option B (everything + local) the kernel needs no changes, so these tests should pass before AND + after the shim/transform fix. They are the regression guard against future + coordinate-space mistakes. +""" + +import math + +import pytest +import torch + +import tensorrt_llm._torch.auto_deploy # noqa: F401 + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + + +def _sdpa_reference_with_sw( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + sliding_window: int, + cache_len_before_q: int, +) -> torch.Tensor: + """Reference SDPA with a per-query causal + sliding-window mask. + + Inputs are in [tokens, n_heads, head_dim] (flat). For each query token at + local position `q_pos = cache_len_before_q + i` (i ∈ [0, q_len)), the + valid keys are positions in [max(0, q_pos - W + 1), q_pos]. + """ + q_t = q.transpose(0, 1).unsqueeze(0) # [1, n_heads, q_len, head_dim] + k_t = k.transpose(0, 1).unsqueeze(0) # [1, n_kv_heads, kv_len, head_dim] + v_t = v.transpose(0, 1).unsqueeze(0) + + n_heads = q_t.shape[1] + n_kv_heads = k_t.shape[1] + if n_heads != n_kv_heads: + rep = n_heads // n_kv_heads + k_t = k_t.repeat_interleave(rep, dim=1) + v_t = v_t.repeat_interleave(rep, dim=1) + + q_len = q_t.shape[2] + kv_len = k_t.shape[2] + + head_dim = q_t.shape[-1] + scale = 1.0 / math.sqrt(head_dim) + scores = torch.matmul(q_t, k_t.transpose(-2, -1)) * scale # [1, h, q_len, kv_len] + + # mask in [q_len, kv_len]; q_pos = cache_len_before_q + i + q_positions = torch.arange(q_len, device=q.device) + cache_len_before_q + kv_positions = torch.arange(kv_len, device=q.device) + diff = q_positions.unsqueeze(1) - kv_positions.unsqueeze(0) + causal_ok = diff >= 0 + window_ok = diff < sliding_window + valid = causal_ok & window_ok + scores = scores.masked_fill(~valid.unsqueeze(0).unsqueeze(0), float("-inf")) + + weights = torch.softmax(scores, dim=-1) + out = torch.matmul(weights, v_t) # [1, h, q_len, head_dim] + return out.squeeze(0).transpose(0, 1) # [q_len, n_heads, head_dim] + + +def _write_kv_to_cache( + k: torch.Tensor, + v: torch.Tensor, + kv_cache: torch.Tensor, + page_table: torch.Tensor, + page_size: int, +): + """Write all of (k, v) sequentially into kv_cache at the listed pages. + + k, v are flat [tokens, n_kv_heads, head_dim]. page_table is a 1D int32 + tensor listing physical page ids in token-order. Assumes page_table + covers exactly enough pages for k.shape[0] tokens. + """ + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + update_paged_kv_cache, + ) + + n_tokens = k.shape[0] + batch_indices = torch.zeros(n_tokens, dtype=torch.int32, device=k.device) + positions = torch.arange(n_tokens, dtype=torch.int32, device=k.device) + kv_indptr = torch.tensor([0, page_table.numel()], dtype=torch.int32, device=k.device) + update_paged_kv_cache(k, v, batch_indices, positions, kv_cache, page_table, kv_indptr) + + +class TestTritonPagedLongPrefillSWA: + """Prefill longer than the sliding window — no eviction yet. + + Verifies the kernel correctly applies the SW mask within a single long + prefill chunk. Covers BOTH the Phase-1 (full pages) and Phase-2 (boundary + pages) code paths since prefill_len > W * 2 forces both. + """ + + @pytest.mark.parametrize("page_size", [16, 64]) + @pytest.mark.parametrize("sliding_window", [128, 256]) + @pytest.mark.parametrize("prefill_len_mult", [1, 2, 4]) # 1*W, 2*W, 4*W + @pytest.mark.parametrize("n_heads,n_kv_heads", [(4, 4), (8, 2)]) + @pytest.mark.parametrize("head_dim", [64, 128]) + def test_long_prefill_sw_matches_sdpa( + self, + page_size: int, + sliding_window: int, + prefill_len_mult: int, + n_heads: int, + n_kv_heads: int, + head_dim: int, + ): + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + triton_paged_context, + ) + + torch.manual_seed(0) + prefill_len = sliding_window * prefill_len_mult + # Round prefill_len up so it spans whole + boundary pages naturally. + num_pages = (prefill_len + page_size - 1) // page_size + num_blocks = num_pages + 4 + + q = torch.randn(prefill_len, n_heads, head_dim, dtype=torch.float16, device="cuda") + k = torch.randn(prefill_len, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + v = torch.randn(prefill_len, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + + kv_cache = torch.zeros( + num_blocks, 2, n_kv_heads, page_size, head_dim, dtype=torch.float16, device="cuda" + ) + page_table = torch.arange(num_pages, dtype=torch.int32, device="cuda") + _write_kv_to_cache(k, v, kv_cache, page_table, page_size) + + qo_indptr = torch.tensor([0, prefill_len], dtype=torch.int32, device="cuda") + kv_indptr = torch.tensor([0, num_pages], dtype=torch.int32, device="cuda") + kv_indices = page_table.clone() + last_token_in_page = prefill_len % page_size + kv_last_page_len = torch.tensor( + [last_token_in_page if last_token_in_page > 0 else page_size], + dtype=torch.int32, + device="cuda", + ) + seq_len_with_cache = torch.tensor([prefill_len], dtype=torch.int32, device="cuda") + + sm_scale = 1.0 / math.sqrt(head_dim) + output = triton_paged_context( + q, + kv_cache, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + seq_len_with_cache, + sm_scale, + sliding_window=sliding_window, + ) + + ref = _sdpa_reference_with_sw( + q.float(), + k.float(), + v.float(), + sliding_window=sliding_window, + cache_len_before_q=0, + ).to(output.dtype) + + torch.testing.assert_close(output.float(), ref.float(), rtol=1e-2, atol=1e-2) + + +class TestTritonPagedSWAFrontEviction: + """Front-eviction SWA — the shim hands the kernel a window-local view. + + The setup mirrors what `ad_executor.py` does after Phase 2: it slices + `cache_loc` to the live (post-eviction) pages and caps + `seq_len_with_cache` at the window. With Option B (everything local), the + kernel needs no changes — these tests are the regression guard for that + contract. + """ + + @pytest.mark.parametrize("front_removed", [0, 1, 4]) + @pytest.mark.parametrize("chunk", [1, 64, 128]) + @pytest.mark.parametrize("page_size", [16, 64]) + @pytest.mark.parametrize("with_sliding_window", [True, False]) + def test_front_evicted_context_matches_sdpa_on_live_window( + self, + front_removed: int, + chunk: int, + page_size: int, + with_sliding_window: bool, + ): + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + triton_paged_context, + ) + + # Fixed simple geometry for clarity. Window W=256 → 16 pages at PAGE_SIZE=16, + # 4 pages at PAGE_SIZE=64. + torch.manual_seed(0) + sliding_window = 256 + n_heads = 4 + n_kv_heads = 4 + head_dim = 64 + + # Total cached tokens (post-fill, pre-chunk). Make this exactly equal to W + # so the "live" window contains exactly W tokens regardless of front_removed. + total_cache_tokens = sliding_window + # The historical page list must include the front-evicted pages too. + historical_pages = front_removed + (total_cache_tokens + page_size - 1) // page_size + num_blocks = historical_pages + 4 + (chunk + page_size - 1) // page_size + + # Build a historical KV history: front_removed*page_size junk tokens + # (will not be read) + total_cache_tokens real tokens, then `chunk` new + # tokens being processed. + live_cache_tokens = total_cache_tokens + new_chunk = chunk + + # Allocate cache and a "historical" page table (front_removed pages + # contain stale data; the rest contain live cached tokens). + kv_cache = torch.zeros( + num_blocks, 2, n_kv_heads, page_size, head_dim, dtype=torch.float16, device="cuda" + ) + + # Real (live + new) KV data we want the kernel to attend to. + live_k = torch.randn( + live_cache_tokens, n_kv_heads, head_dim, dtype=torch.float16, device="cuda" + ) + live_v = torch.randn( + live_cache_tokens, n_kv_heads, head_dim, dtype=torch.float16, device="cuda" + ) + new_k = torch.randn(new_chunk, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + new_v = torch.randn(new_chunk, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + # Stale data the kernel must NOT read (we'll keep these pages + # zero-filled — if the kernel reads them it'll skew the output). + + # Page table layout (historical, full list): + # [stale_0, stale_1, ..., stale_{front_removed-1}, live_0, ..., new_pages...] + live_pages_needed = (live_cache_tokens + page_size - 1) // page_size + new_pages_needed = (new_chunk + page_size - 1) // page_size + # Total physical pages used by this request, from page 0..end: + all_pages = torch.arange( + historical_pages + new_pages_needed, dtype=torch.int32, device="cuda" + ) + + # Write live KV to the live pages (positions [front_removed*page_size, + # front_removed*page_size + live_cache_tokens)) of the global cache. + live_page_table = all_pages[front_removed : front_removed + live_pages_needed] + _write_kv_to_cache(live_k, live_v, kv_cache, live_page_table, page_size) + # Write new chunk KV to the new pages (just after the live pages). + new_page_table = all_pages[ + front_removed + live_pages_needed : front_removed + live_pages_needed + new_pages_needed + ] + # Pad to page boundary for the write helper. + new_k_padded = torch.zeros( + new_pages_needed * page_size, n_kv_heads, head_dim, dtype=torch.float16, device="cuda" + ) + new_v_padded = torch.zeros_like(new_k_padded) + # Page-aligned local offset for the chunk's first token: it sits right + # after the live cache, in local coords that's live_cache_tokens. + # Since live_cache_tokens is a multiple of page_size (we set it = W and + # require page_size | W), the chunk starts at offset 0 within new_page_table. + assert live_cache_tokens % page_size == 0, ( + "test geometry assumes live cache is page-aligned" + ) + new_k_padded[:new_chunk] = new_k + new_v_padded[:new_chunk] = new_v + # We can't easily write a partial-page; only fill if new_chunk is multiple of page_size. + # For partial pages, fall back to per-token write. + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + update_paged_kv_cache, + ) + + b_idx = torch.zeros(new_chunk, dtype=torch.int32, device="cuda") + positions = torch.arange(new_chunk, dtype=torch.int32, device="cuda") + kv_indptr_for_write = torch.tensor([0, new_pages_needed], dtype=torch.int32, device="cuda") + update_paged_kv_cache( + new_k, new_v, b_idx, positions, kv_cache, new_page_table, kv_indptr_for_write + ) + + # Shim's view: slice the historical page table to live + new only. + # This is exactly `all_indices[front_removed : front_removed + num_active]` + # in ad_executor.py after Phase 2. + live_view_pages = all_pages[ + front_removed : front_removed + live_pages_needed + new_pages_needed + ].contiguous() + num_kv_pages = live_view_pages.numel() + + # Q is the chunk's queries. + q = torch.randn(new_chunk, n_heads, head_dim, dtype=torch.float16, device="cuda") + + qo_indptr = torch.tensor([0, new_chunk], dtype=torch.int32, device="cuda") + kv_indptr = torch.tensor([0, num_kv_pages], dtype=torch.int32, device="cuda") + # Window-capped seq_len_with_cache: live_cache_tokens + new_chunk + # (NOT the global value which would include the evicted prefix). + seq_len_with_cache = torch.tensor( + [live_cache_tokens + new_chunk], dtype=torch.int32, device="cuda" + ) + last_token_in_page = (live_cache_tokens + new_chunk) % page_size + kv_last_page_len = torch.tensor( + [last_token_in_page if last_token_in_page > 0 else page_size], + dtype=torch.int32, + device="cuda", + ) + + sm_scale = 1.0 / math.sqrt(head_dim) + sw_arg = sliding_window if with_sliding_window else 0 + output = triton_paged_context( + q, + kv_cache, + qo_indptr, + kv_indptr, + live_view_pages, + kv_last_page_len, + seq_len_with_cache, + sm_scale, + sliding_window=sw_arg, + ) + + # SDPA reference computed on the LIVE window only: keys/values are + # [live_k; new_k], queries are `q`. Cache length before q is + # live_cache_tokens. Sliding window applied if requested (passing + # sw=0 corresponds to plain causal on the live window). + full_k = torch.cat([live_k, new_k], dim=0) + full_v = torch.cat([live_v, new_v], dim=0) + ref = _sdpa_reference_with_sw( + q.float(), + full_k.float(), + full_v.float(), + # sw=0 in the kernel ⇒ no SW pruning; reproduce that by feeding a + # window larger than the entire kv length. + sliding_window=sliding_window + if with_sliding_window + else (live_cache_tokens + new_chunk + 1), + cache_len_before_q=live_cache_tokens, + ).to(output.dtype) + + torch.testing.assert_close(output.float(), ref.float(), rtol=1e-2, atol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_trtllm_attention_op.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_trtllm_attention_op.py index d06ed725aff5..353938830fa1 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_trtllm_attention_op.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_trtllm_attention_op.py @@ -845,8 +845,8 @@ def test_extend_iteration_preserves_original_prompt_length(self, context_length: cu_seqlen=cu_seqlen, input_pos=input_pos, batch_info=batch_info, - cache_loc=torch.arange(num_pages), - cu_num_pages=torch.tensor([0, num_pages], dtype=torch.int), + cache_loc_per_pool=[torch.arange(num_pages)], + cu_num_pages_per_pool=[torch.tensor([0, num_pages], dtype=torch.int)], slot_idx=torch.arange(1), prompt_lens=[context_length], ) @@ -868,8 +868,8 @@ def test_metadata_handles_two_sequences_with_different_lengths(self): input_ids.flatten(), cu_seqlen=cu_seqlen, input_pos=0, - cache_loc=torch.arange(300 // info.tokens_per_block + 2), - cu_num_pages=torch.tensor([0, 4, 10], dtype=torch.int), + cache_loc_per_pool=[torch.arange(300 // info.tokens_per_block + 2)], + cu_num_pages_per_pool=[torch.tensor([0, 4, 10], dtype=torch.int)], slot_idx=torch.arange(2), ) _prepare_from_sequence_info(info) diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/test_switch_to_generate_inplace.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/test_switch_to_generate_inplace.py index 6772e8cde779..16eba6cc79c5 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/test_switch_to_generate_inplace.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/test_switch_to_generate_inplace.py @@ -68,9 +68,9 @@ def _nest_prefill(si: SequenceInfo, input_ids, pages_per_seq, cache_loc, **kw): flat_ids, cu_seqlen, input_pos=0, - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, - extra_page_per_seq=extra_page, + cache_loc_per_pool=[cache_loc], + cu_num_pages_per_pool=[cu_num_pages], + extra_page_per_seq_per_pool=[extra_page], **kw, ) return si @@ -158,9 +158,9 @@ def test_single_prefill_with_nonzero_input_pos(self): [1, 2, 3, 4, 5], cu_seqlen=[0, 5], input_pos=[10], - cache_loc=[0, 1, 2, 3], - cu_num_pages=[0, 4], - extra_page_per_seq=[-1], + cache_loc_per_pool=[[0, 1, 2, 3]], + cu_num_pages_per_pool=[[0, 4]], + extra_page_per_seq_per_pool=[[-1]], ) increment = torch.tensor([1], dtype=torch.int32, device=si.device) @@ -195,9 +195,9 @@ def _setup_decode_batch(self, si, positions, swc_vals, pages_per_seq, cache_loc) cu_seqlen=list(range(batch_size + 1)), input_pos=positions, batch_info=[0, 0, 0, 0, batch_size, batch_size], - cache_loc=cache_loc, - cu_num_pages=cu_num_pages, - extra_page_per_seq=[-1] * batch_size, + cache_loc_per_pool=[cache_loc], + cu_num_pages_per_pool=[cu_num_pages], + extra_page_per_seq_per_pool=[[-1] * batch_size], ) def test_decode_increment_by_one(self): @@ -282,9 +282,9 @@ def test_page_crossing_with_extra_page_inserts_into_cache_loc(self): cu_seqlen=[0, 1], input_pos=[7], batch_info=[0, 0, 0, 0, 1, 1], - cache_loc=[10, 11], - cu_num_pages=[0, 2], - extra_page_per_seq=[99], + cache_loc_per_pool=[[10, 11]], + cu_num_pages_per_pool=[[0, 2]], + extra_page_per_seq_per_pool=[[99]], ) increment = torch.tensor([1], dtype=torch.int32, device=si.device) @@ -304,9 +304,9 @@ def _setup_decode_at_page_boundary(si): cu_seqlen=[0, 1], input_pos=[7], batch_info=[0, 0, 0, 0, 1, 1], - cache_loc=[10, 11], - cu_num_pages=[0, 2], - extra_page_per_seq=[-1], + cache_loc_per_pool=[[10, 11]], + cu_num_pages_per_pool=[[0, 2]], + extra_page_per_seq_per_pool=[[-1]], ) @@ -342,9 +342,9 @@ def test_inactive_host_mirrors_not_synced_by_offset_pos_and_cache(self): cu_seqlen=[0, 1, 2], input_pos=[5, 10], batch_info=[0, 0, 0, 0, 2, 2], - cache_loc=[10, 11, 20, 21, 22], - cu_num_pages=[0, 2, 5], - extra_page_per_seq=[-1, -1], + cache_loc_per_pool=[[10, 11, 20, 21, 22]], + cu_num_pages_per_pool=[[0, 2, 5]], + extra_page_per_seq_per_pool=[[-1, -1]], ) assert not _has_active_host_arg(si) diff --git a/tests/unittest/auto_deploy/singlegpu/shim/test_ad_executor_swa_eviction.py b/tests/unittest/auto_deploy/singlegpu/shim/test_ad_executor_swa_eviction.py new file mode 100644 index 000000000000..95c873f8a160 --- /dev/null +++ b/tests/unittest/auto_deploy/singlegpu/shim/test_ad_executor_swa_eviction.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration test for the shim's SWA front-eviction metadata path. + +Drives a real two-window CachedSequenceInterface + KVCacheManager (so the +historical `get_cache_indices` payload reflects real C++ page-table state), +then monkeypatches `get_num_front_blocks_removed` to simulate eviction at +controlled counts. The shim helper `_compute_window_local_view` must +produce a window-coherent local-coord view: live page slice, window-capped +seq_len_with_cache, derived last_page_len, and a coherent spillover slot. + +`get_num_front_blocks_removed` is monkeypatched (not driven by real decode) +because firing real SWA eviction from Python without a model requires +either spinning up a full ADEngine + model or adding a test-only C++ +binding to bump the counter directly — both are out of scope. The patch +covers exactly what the C++ side would otherwise do: bump a counter +without mutating mCacheBlockIds, so the historical page list still +contains the evicted-but-stale-entries-at-the-front pattern that the +shim is supposed to handle. +""" + +import pytest +import torch +from _model_test_utils import default_max_num_tokens + +from tensorrt_llm._torch.auto_deploy._compat import KvCacheConfig +from tensorrt_llm._torch.auto_deploy.custom_ops.attention_interface import KVPagedResourceHandler +from tensorrt_llm._torch.auto_deploy.shim.ad_executor import _compute_window_local_view +from tensorrt_llm._torch.auto_deploy.shim.interface import CachedSequenceInterface + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + + +# Geometry shared across cases: SWA window of 64 tokens with tokens_per_block=32 +# gives 2 pages per window — small enough to exercise eviction boundaries +# without ballooning runtime. FULL_WINDOW doubles as the fixture's +# ``max_seq_len`` and therefore sets the SWA pool's per-sequence cap +# (``maxTokenNum = max(windowSize, maxSequenceLength)`` in kvCacheManager.cpp). +# It must exceed the longest scenario the tests construct (192-token prefill +# in ``test_helper_pre_eviction_long_prefill_passes_full_pages``) or the C++ +# side clamps allocation and the helper sees fewer live pages than expected. +SWA_WINDOW = 64 +FULL_WINDOW = 256 +TOKENS_PER_BLOCK = 32 +PAGES_PER_SWA_WINDOW = SWA_WINDOW // TOKENS_PER_BLOCK # 2 + + +@pytest.fixture +def two_window_interface(): + """A real CachedSequenceInterface hosting a dual-pool KVCacheManager. + + Layer 0 lives in an SWA window (W=64); layer 1 lives in a full window. + This is the Gemma-4 geometry — exactly the path Phase 2 fixes. + """ + interface = CachedSequenceInterface( + max_seq_len=FULL_WINDOW, + max_batch_size=2, + max_num_tokens=default_max_num_tokens(FULL_WINDOW, 2), + device="cuda", + kv_cache_config=KvCacheConfig( + tokens_per_block=TOKENS_PER_BLOCK, + max_tokens=1024, + free_gpu_memory_fraction=0.0, + ), + ) + interface.add_resource( + "kv_swa", KVPagedResourceHandler(4, 32, dtype=torch.float16, sliding_window=SWA_WINDOW) + ) + interface.add_resource("kv_full", KVPagedResourceHandler(4, 32, dtype=torch.float16)) + interface.initialize_resources() + return interface + + +def _add_request(manager, request_id: int, token_num: int): + """Add a dummy request that allocates pages for `token_num` tokens.""" + out = manager.add_dummy_requests([request_id], token_nums=[token_num]) + assert out is not None and len(out) == 1, "manager did not allocate the request" + return out[0] + + +def test_helper_no_eviction_passes_through(two_window_interface): + """Sanity guard: front_removed=0 reproduces the pre-Phase-2 view. + + Asserts `all_indices[:num_active]`, full window-capped + seq_len_with_cache, and the next slot as the spillover page. + """ + manager = two_window_interface.kv_cache_manager + req = _add_request(manager, request_id=42, token_num=SWA_WINDOW) + + all_indices = manager.get_cache_indices(req, window_size=SWA_WINDOW) + assert manager.get_num_front_blocks_removed(req.py_request_id, window_size=SWA_WINDOW) == 0 + + active_indices, extra_page, swc, lpl = _compute_window_local_view( + all_indices, + front_removed=0, + end_compute_i=SWA_WINDOW, + group_window=SWA_WINDOW, + tokens_per_block=TOKENS_PER_BLOCK, + ) + + assert active_indices == list(all_indices[:PAGES_PER_SWA_WINDOW]) + assert swc == SWA_WINDOW + assert lpl == TOKENS_PER_BLOCK + # No spillover slot when num_active fully consumes the historical list. + if len(all_indices) > PAGES_PER_SWA_WINDOW: + assert extra_page == all_indices[PAGES_PER_SWA_WINDOW] + else: + assert extra_page == -1 + + +@pytest.mark.parametrize("front_removed", [1, PAGES_PER_SWA_WINDOW]) +def test_helper_with_simulated_eviction_slices_correctly( + two_window_interface, monkeypatch, front_removed +): + """Eviction-aware slicing in window-local coordinates. + + With ``front_removed > 0`` the slice must start at ``front_removed`` and + ``seq_len_with_cache`` must equal the live (window-local) cache length: + ``end_compute_i - front_removed * tokens_per_block``. The C++ eviction + loop guarantees this stays within ``window + tokens_per_block``, so it + may exceed ``SWA_WINDOW`` by up to one spillover page — the helper does + not artificially clamp at ``SWA_WINDOW`` because doing so would discard + a live page's worth of data that the kernel still has to read. + + Simulates a request that has cumulatively seen + ``front_removed * tokens_per_block + SWA_WINDOW + 1`` tokens, i.e. + ``front_removed`` pages have been front-evicted and one spillover token + sits in the page after the window. + """ + manager = two_window_interface.kv_cache_manager + # Allocate enough pages to span (evicted + live + a spillover slot) so + # get_cache_indices returns a realistic-length list. + total_tokens = front_removed * TOKENS_PER_BLOCK + SWA_WINDOW + 1 + req = _add_request(manager, request_id=43, token_num=total_tokens) + + # Bump the eviction counter the way detachFrontBlock would (the C++ side + # bumps `mNumFrontBlocksRemovedPerWindow[ws]` without mutating + # mCacheBlockIds, so get_cache_indices STILL returns the full list and + # the shim must slice [front_removed:] off it). + real_query = manager.get_num_front_blocks_removed + + def fake_front_removed(request_id, window_size=None): + if request_id == req.py_request_id and window_size == SWA_WINDOW: + return front_removed + return real_query(request_id, window_size=window_size) + + monkeypatch.setattr(manager, "get_num_front_blocks_removed", fake_front_removed) + + all_indices = manager.get_cache_indices(req, window_size=SWA_WINDOW) + front = manager.get_num_front_blocks_removed(req.py_request_id, window_size=SWA_WINDOW) + assert front == front_removed + + end_compute_i = total_tokens + active_indices, extra_page, swc, lpl = _compute_window_local_view( + all_indices, + front_removed=front, + end_compute_i=end_compute_i, + group_window=SWA_WINDOW, + tokens_per_block=TOKENS_PER_BLOCK, + ) + + # Window-local cache length matches the C++ accounting. + expected_swc = end_compute_i - front_removed * TOKENS_PER_BLOCK + assert swc == expected_swc, ( + f"swc={swc} != expected {expected_swc} " + f"(end_compute_i={end_compute_i}, front_removed={front_removed})" + ) + # C++ eviction invariant: live cache ≤ window + one spillover page. + assert swc <= SWA_WINDOW + TOKENS_PER_BLOCK + + # Live slice starts at front_removed and covers every page needed to + # address swc tokens — including the spillover page when present. + expected_num_active = (expected_swc + TOKENS_PER_BLOCK - 1) // TOKENS_PER_BLOCK + expected_slice = list(all_indices[front_removed : front_removed + expected_num_active]) + assert active_indices == expected_slice, ( + f"slice mismatch: front_removed={front_removed}, " + f"all_indices={list(all_indices)}, active={active_indices}" + ) + + # last_page_len matches the modular arithmetic on the local cache length. + expected_lpl = (swc - 1) % TOKENS_PER_BLOCK + 1 if swc > 0 else 0 + assert lpl == expected_lpl + + # Page table capacity covers the live cache length. + assert len(active_indices) * TOKENS_PER_BLOCK >= swc + + +def test_helper_caps_seq_len_with_cache_below_window(two_window_interface, monkeypatch): + """Below-window end_compute_i reports actual progress, not the window size. + + When end_compute_i is BELOW the window (early prefill / short request), + seq_len_with_cache reports the actual progress, not the window size — the + helper must not over-pad. + """ + manager = two_window_interface.kv_cache_manager + req = _add_request(manager, request_id=44, token_num=SWA_WINDOW) + + monkeypatch.setattr(manager, "get_num_front_blocks_removed", lambda req_id, window_size=None: 0) + all_indices = manager.get_cache_indices(req, window_size=SWA_WINDOW) + + short_end = TOKENS_PER_BLOCK + 5 # < SWA_WINDOW + _, _, swc, lpl = _compute_window_local_view( + all_indices, + front_removed=0, + end_compute_i=short_end, + group_window=SWA_WINDOW, + tokens_per_block=TOKENS_PER_BLOCK, + ) + assert swc == short_end + assert lpl == ((short_end - 1) % TOKENS_PER_BLOCK + 1) + + +def test_helper_pre_eviction_long_prefill_passes_full_pages(two_window_interface, monkeypatch): + """Long prefill that has not yet been front-evicted — hand FULL live pages. + + Captures the MMLU-on-Gemma3n regime: addSequenceBatch admits a prefill + longer than the SWA window and allocates blocks linearly for the full + prompt (kvCacheManager.cpp::addSequenceBatch comment "For SWA, blocks + are allocated linearly for the full prompt; out-of-window blocks are + only detached during generation in adjustBlocksIfNeeded"). At the + first forward pass ``front_removed == 0`` and the kernel needs every + allocated page plus ``seq_len_with_cache == end_compute_i`` so the + sliding-window mask is applied inside the kernel over the full prefill, + matching the contract validated by + ``test_long_prefill_sw_matches_sdpa``. + + Pre-fix the helper unconditionally clamped ``active_token_count`` to + ``group_window`` even when no eviction had fired, which truncated the + page list to the first ``window/page_size`` pages and silently + corrupted long prefills. + """ + manager = two_window_interface.kv_cache_manager + # End at 3x the window so the prefill clearly exceeds W on the SWA pool + # but front-eviction has not run yet. + prefill_len = SWA_WINDOW * 3 # 192 tokens + req = _add_request(manager, request_id=46, token_num=prefill_len) + monkeypatch.setattr(manager, "get_num_front_blocks_removed", lambda req_id, window_size=None: 0) + + all_indices = manager.get_cache_indices(req, window_size=SWA_WINDOW) + expected_pages = (prefill_len + TOKENS_PER_BLOCK - 1) // TOKENS_PER_BLOCK + assert len(all_indices) >= expected_pages, ( + f"manager must over-allocate pages for SWA prefill: " + f"got {len(all_indices)}, expected >= {expected_pages}" + ) + + active_indices, extra_page, swc, lpl = _compute_window_local_view( + all_indices, + front_removed=0, + end_compute_i=prefill_len, + group_window=SWA_WINDOW, + tokens_per_block=TOKENS_PER_BLOCK, + ) + + # Hand the kernel every live page (not just the window's worth). + assert active_indices == list(all_indices[:expected_pages]), ( + f"expected full prefill page list, got {active_indices} vs all_indices={list(all_indices)}" + ) + # Window-local cache length equals the unclamped prefill length — the + # kernel will mask down to the window itself. + assert swc == prefill_len, f"swc={swc} should equal prefill_len={prefill_len}" + assert lpl == (prefill_len - 1) % TOKENS_PER_BLOCK + 1 + # Page table capacity covers the full prefill (the kernel writes 1 + # K/V token per query token). + assert len(active_indices) * TOKENS_PER_BLOCK >= swc + # extra_page only populated if the manager pre-allocated past the prefill. + if len(all_indices) > expected_pages: + assert extra_page == all_indices[expected_pages] + else: + assert extra_page == -1 + + +def test_helper_does_not_consume_evicted_extra_slot(two_window_interface, monkeypatch): + """extra_page must live AFTER the live region, never inside the evicted range. + + Pre-Phase-2 code that referenced ``all_indices[num_active]`` would land + inside the evicted region whenever front_removed > 0 — this test guards + against that. Under the live-page-derived view the live region already + covers every allocated page up to and including the spillover, so + ``extra_page == -1`` in the typical post-eviction case; we additionally + check that whenever extra_page IS populated, the index sits past the + evicted prefix. + """ + manager = two_window_interface.kv_cache_manager + front_removed = 2 + total_tokens = front_removed * TOKENS_PER_BLOCK + SWA_WINDOW + 1 + req = _add_request(manager, request_id=45, token_num=total_tokens) + monkeypatch.setattr( + manager, + "get_num_front_blocks_removed", + lambda req_id, window_size=None: front_removed, + ) + + all_indices = manager.get_cache_indices(req, window_size=SWA_WINDOW) + active_indices, extra_page, swc, _ = _compute_window_local_view( + all_indices, + front_removed=front_removed, + end_compute_i=total_tokens, + group_window=SWA_WINDOW, + tokens_per_block=TOKENS_PER_BLOCK, + ) + + # Live region exhausts every page the C++ side handed us, so the + # spillover slot beyond it is -1 (no allocated next page). + num_active = len(active_indices) + expected_extra_idx = front_removed + num_active + if len(all_indices) > expected_extra_idx: + assert extra_page == all_indices[expected_extra_idx] + # Crucially, the extra slot is NOT in the front-evicted range. + assert expected_extra_idx >= front_removed + else: + assert extra_page == -1 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/auto_deploy/singlegpu/shim/test_cached_sequence_interface.py b/tests/unittest/auto_deploy/singlegpu/shim/test_cached_sequence_interface.py index 58bcc3985b06..91aca4a4f8a7 100644 --- a/tests/unittest/auto_deploy/singlegpu/shim/test_cached_sequence_interface.py +++ b/tests/unittest/auto_deploy/singlegpu/shim/test_cached_sequence_interface.py @@ -269,36 +269,10 @@ def test_initialize_resources_paged_only_creates_kv_cache_manager(paged_kv_cache assert not isinstance(interface.kv_cache_manager, MambaHybridCacheManager) -def test_initialize_resources_incompatible_kv_layers_can_be_unmanaged(paged_kv_cache_config): - """Compatible layers are managed; incompatible ones may fall back to unmanaged buffers.""" - interface = CachedSequenceInterface( - max_seq_len=128, - max_batch_size=4, - max_num_tokens=default_max_num_tokens(128, 4), - device="cuda", - kv_cache_config=paged_kv_cache_config, - requires_uniform_kv_caches=False, - ) - - managed_name = interface.add_resource( - "kv_cache_0", - KVPagedResourceHandler(8, 64, dtype=torch.float16, kv_layout="HND"), - ) - unmanaged_name = interface.add_resource( - "kv_cache_1", - KVPagedResourceHandler(8, 80, dtype=torch.float16, kv_layout="HND"), - ) - - interface.initialize_resources() - - assert managed_name not in interface._unmanaged_resources - assert unmanaged_name in interface._unmanaged_resources - - -def test_initialize_resources_incompatible_kv_layers_raise_when_uniform_managed_caches_required( +def test_initialize_resources_mixed_shape_pools_raise_when_uniform_managed_caches_required( paged_kv_cache_config, ): - """Strict interface configurations reject mixed managed and unmanaged KV layouts.""" + """Strict configurations reject more than one distinct window-pool.""" interface = CachedSequenceInterface( max_seq_len=128, max_batch_size=4, @@ -308,8 +282,10 @@ def test_initialize_resources_incompatible_kv_layers_raise_when_uniform_managed_ requires_uniform_kv_caches=True, ) - interface.add_resource("kv_cache_0", KVPagedResourceHandler(8, 64, dtype=torch.float16)) - interface.add_resource("kv_cache_1", KVPagedResourceHandler(8, 80, dtype=torch.float16)) + interface.add_resource("kv_cache_full", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + interface.add_resource( + "kv_cache_swa", KVPagedResourceHandler(8, 80, dtype=torch.float16, sliding_window=32) + ) with pytest.raises(RuntimeError): interface.initialize_resources() @@ -875,6 +851,20 @@ def test_sequence_info_update_cache_information_resizes(): assert seq_info._input_buffer.get_capacity("cache_loc") >= expected_capacity +def test_sequence_info_update_cache_information_preserves_max_blocks(): + """Verify max_blocks_per_seq is always based on max_seq_len (not window).""" + seq_info = SequenceInfo( + max_seq_len=128, + max_batch_size=4, + max_num_tokens=default_max_num_tokens(128, 4), + tokens_per_block=32, + ) + + original_max_blocks = seq_info.max_blocks_per_seq # ceil(128 / 32) = 4 + seq_info.update_cache_information(num_blocks=100) + assert seq_info.max_blocks_per_seq == original_max_blocks + + def test_sequence_info_last_page_len_uses_tokens_per_block(): """Verify nest_sequences calculates last_page_len using tokens_per_block.""" seq_info = SequenceInfo( @@ -891,8 +881,8 @@ def test_sequence_info_last_page_len_uses_tokens_per_block(): input_ids, cu_seqlen=[0, 25], input_pos=[0], - cache_loc=[0, 1], # 2 pages - cu_num_pages=[0, 2], + cache_loc_per_pool=[[0, 1]], # 2 pages + cu_num_pages_per_pool=[[0, 2]], ) expected_last_page_len = (25 - 1) % 16 + 1 @@ -914,8 +904,8 @@ def test_sequence_info_page_assignments(): input_ids, cu_seqlen=[0, 10, 30], input_pos=[0, 0], - cache_loc=[0, 1, 2], # seq 0 has page 0, seq 1 has pages 1 and 2 - cu_num_pages=[0, 1, 3], + cache_loc_per_pool=[[0, 1, 2]], # seq 0 has page 0, seq 1 has pages 1 and 2 + cu_num_pages_per_pool=[[0, 1, 3]], ) cache_loc = seq_info.get_arg("cache_loc_host", truncate=True).tolist() @@ -1168,8 +1158,8 @@ def test_args_stored_to_input_buffer(): input_ids=[1, 2, 3], cu_seqlen=[0, 3], input_pos=[0], - cache_loc=[0], - cu_num_pages=[0, 1], + cache_loc_per_pool=[[0]], + cu_num_pages_per_pool=[[0, 1]], ) token_gather_indices = seq_info.get_arg("token_gather_indices", truncate=True) @@ -1183,8 +1173,8 @@ def test_args_stored_to_input_buffer(): input_ids=[1, 2, 3], cu_seqlen=[0, 3], input_pos=[0], - cache_loc=[0], - cu_num_pages=[0, 1], + cache_loc_per_pool=[[0]], + cu_num_pages_per_pool=[[0, 1]], gather_context_logits=True, ) @@ -1214,3 +1204,99 @@ def dummy_host_prepare(batch_info_host: torch.Tensor, cu_num_pages_host: torch.T assert "batch_info_host" in seq_info._active_host_prep_args assert "cu_num_pages_host" in seq_info._active_host_prep_args + + +# ============================================================================= +# Multi-Window (VSWA) KV Cache Tests — single unified KVCacheManager hosting +# multiple C++ pools via the per-window head_dim/dtype overrides. +# ============================================================================= + + +def test_identify_managed_kv_resources_single_window(): + """Single head_dim and no sliding window collapses into one window entry.""" + interface = CachedSequenceInterface( + max_seq_len=128, + max_batch_size=4, + max_num_tokens=default_max_num_tokens(128, 4), + device="cuda", + kv_cache_config=KvCacheConfig( + tokens_per_block=32, max_tokens=1024, free_gpu_memory_fraction=0.0 + ), + ) + interface.add_resource("kv_0", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + interface.add_resource("kv_1", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + + kv_managed, pool_configurations = interface._identify_managed_kv_resources() + + assert len(kv_managed) == 2 + # No sliding_window → effective window = max_seq_len. + assert {pc.window_size: pc.head_dim for pc in pool_configurations} == {128: 64} + + +def test_identify_managed_kv_resources_dual_window_gemma4_pattern(): + """Gemma4-style mix: SWA layers with smaller head_dim + full-attn layer with larger head_dim.""" + interface = CachedSequenceInterface( + max_seq_len=128, + max_batch_size=4, + max_num_tokens=default_max_num_tokens(128, 4), + device="cuda", + kv_cache_config=KvCacheConfig( + tokens_per_block=32, max_tokens=1024, free_gpu_memory_fraction=0.0 + ), + ) + # SWA window: head_dim=64 + interface.add_resource( + "kv_0", KVPagedResourceHandler(8, 64, dtype=torch.float16, sliding_window=64) + ) + interface.add_resource( + "kv_1", KVPagedResourceHandler(8, 64, dtype=torch.float16, sliding_window=64) + ) + # Full-attention window: head_dim=128 + interface.add_resource("kv_2", KVPagedResourceHandler(4, 128, dtype=torch.float16)) + + kv_managed, pool_configurations = interface._identify_managed_kv_resources() + + assert len(kv_managed) == 3 + # Two windows, keyed by effective window size; full-attention falls back to max_seq_len. + assert {pc.window_size: pc.head_dim for pc in pool_configurations} == {64: 64, 128: 128} + + +def test_identify_managed_kv_resources_rejects_mixed_head_dim_in_same_window(): + """Layers sharing an effective window must agree on head_dim.""" + interface = CachedSequenceInterface( + max_seq_len=128, + max_batch_size=4, + max_num_tokens=default_max_num_tokens(128, 4), + device="cuda", + kv_cache_config=KvCacheConfig( + tokens_per_block=32, max_tokens=1024, free_gpu_memory_fraction=0.0 + ), + ) + # Both default to sliding_window=0 → effective window = max_seq_len. Different head_dims + # in the same window are not representable by one C++ pool. + interface.add_resource("kv_0", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + interface.add_resource("kv_1", KVPagedResourceHandler(4, 128, dtype=torch.float16)) + + with pytest.raises(RuntimeError, match="head_dim"): + interface._identify_managed_kv_resources() + + +def test_single_window_creates_plain_kv_cache_manager(): + """One window creates a plain KVCacheManager with a single C++ pool.""" + interface = CachedSequenceInterface( + max_seq_len=128, + max_batch_size=4, + max_num_tokens=default_max_num_tokens(128, 4), + device="cuda", + kv_cache_config=KvCacheConfig( + tokens_per_block=32, max_tokens=1024, free_gpu_memory_fraction=0.0 + ), + ) + interface.add_resource("kv_0", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + interface.add_resource("kv_1", KVPagedResourceHandler(8, 64, dtype=torch.float16)) + + interface.initialize_resources() + + assert isinstance(interface.kv_cache_manager, KVCacheManager) + # Exactly one pool (max_seq_len, since sliding_window=0). + assert len(interface.kv_cache_manager.impl.pool_configurations) == 1 diff --git a/tests/unittest/auto_deploy/singlegpu/shim/test_engine.py b/tests/unittest/auto_deploy/singlegpu/shim/test_engine.py index 2741e623dc60..a4ccba2610e2 100644 --- a/tests/unittest/auto_deploy/singlegpu/shim/test_engine.py +++ b/tests/unittest/auto_deploy/singlegpu/shim/test_engine.py @@ -175,7 +175,7 @@ def get_cache_indices(self, request): # Return many dummy page IDs; ADEngine will truncate as needed return list(range(1024)) - def get_batch_cache_indices(self, request_ids): + def get_batch_cache_indices(self, request_ids, layer_idx=None): return [list(range(1024)) for _ in request_ids] def get_num_kv_blocks(self, num_tokens: int) -> int: @@ -561,7 +561,7 @@ def get_cache_indices(self, request): return list(range(num_blocks)) return list(range(1024)) - def get_batch_cache_indices(self, request_ids): + def get_batch_cache_indices(self, request_ids, layer_idx=None): return [list(range(1024)) for _ in request_ids] def get_num_kv_blocks(self, num_tokens: int) -> int: diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py index 594e0f5a0d69..f2b3a0ac82ae 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py @@ -314,8 +314,8 @@ def _call_and_unnest(x, input_pos): x.flatten().tolist(), cu_seqlen=cu_seqlen, input_pos=input_pos, - cache_loc=list(range(bs)), - cu_num_pages=list(range(bs + 1)), + cache_loc_per_pool=[list(range(bs))], + cu_num_pages_per_pool=[list(range(bs + 1))], slot_idx=list(range(bs)), gather_context_logits=True, ) @@ -833,3 +833,379 @@ def test_insert_cached_mla_attention_preserves_non_flashinfer_backend(): backend = InsertCachedMLAAttention.resolve_backend_for_node("torch_mla", attn_node) assert backend == "torch_mla" + + +# ============================================================================= +# Sliding Window KV Cache Integration Tests +# ============================================================================= + + +class SlidingWindowGQA(GQAWithSdpaAndEmbedding): + """GQA model with a configurable per-layer sliding_window for testing.""" + + def __init__( + self, + num_attention_heads: int, + hidden_size: int, + num_key_value_heads: int, + vocab_size: int = 1000, + sliding_window: Optional[int] = None, + ): + super().__init__(num_attention_heads, hidden_size, num_key_value_heads, vocab_size) + self._sliding_window = sliding_window + + @torch.no_grad() + def forward( + self, input_ids: torch.Tensor, position_ids: Optional[torch.Tensor] = None + ) -> torch.Tensor: + x = self.embed_tokens(input_ids) + b, s, _ = x.shape + q = self.q_proj(x).view(b, s, self.num_heads, self.head_dim) + k = self.k_proj(x).view(b, s, self.num_kv_heads, self.head_dim) + v = self.v_proj(x).view(b, s, self.num_kv_heads, self.head_dim) + attn_output = torch.ops.auto_deploy.torch_attention( + q, + k, + v, + attn_mask=None, + dropout_p=0.0, + is_causal=True, + scale=None, + sinks=None, + sliding_window=self._sliding_window, + logit_cap=None, + layout="bsnd", + ) + return self.o_proj(attn_output.reshape(b, s, -1)) + + +class VSWAModel(nn.Module): + """Model with two attention layers: one sliding-window, one full-attention (VSWA).""" + + def __init__( + self, + num_attention_heads: int, + hidden_size: int, + num_key_value_heads: int, + vocab_size: int = 1000, + sliding_window: int = 32, + ): + super().__init__() + self.num_heads = num_attention_heads + self.num_kv_heads = num_key_value_heads + self.head_dim = hidden_size // num_attention_heads + self.embed_tokens = nn.Embedding(vocab_size, hidden_size) + self.q_proj_0 = nn.Linear(hidden_size, hidden_size, bias=False) + self.k_proj_0 = nn.Linear(hidden_size, num_key_value_heads * self.head_dim, bias=False) + self.v_proj_0 = nn.Linear(hidden_size, num_key_value_heads * self.head_dim, bias=False) + self.o_proj_0 = nn.Linear(hidden_size, hidden_size, bias=False) + self.q_proj_1 = nn.Linear(hidden_size, hidden_size, bias=False) + self.k_proj_1 = nn.Linear(hidden_size, num_key_value_heads * self.head_dim, bias=False) + self.v_proj_1 = nn.Linear(hidden_size, num_key_value_heads * self.head_dim, bias=False) + self.o_proj_1 = nn.Linear(hidden_size, hidden_size, bias=False) + self._sliding_window = sliding_window + + @torch.no_grad() + def forward( + self, input_ids: torch.Tensor, position_ids: Optional[torch.Tensor] = None + ) -> torch.Tensor: + x = self.embed_tokens(input_ids) + b, s, _ = x.shape + + # Layer 0: sliding window attention + q0 = self.q_proj_0(x).view(b, s, self.num_heads, self.head_dim) + k0 = self.k_proj_0(x).view(b, s, self.num_kv_heads, self.head_dim) + v0 = self.v_proj_0(x).view(b, s, self.num_kv_heads, self.head_dim) + a0 = torch.ops.auto_deploy.torch_attention( + q0, + k0, + v0, + attn_mask=None, + dropout_p=0.0, + is_causal=True, + scale=None, + sinks=None, + sliding_window=self._sliding_window, + logit_cap=None, + layout="bsnd", + ) + x = x + self.o_proj_0(a0.reshape(b, s, -1)) + + # Layer 1: full attention (no sliding window) + q1 = self.q_proj_1(x).view(b, s, self.num_heads, self.head_dim) + k1 = self.k_proj_1(x).view(b, s, self.num_kv_heads, self.head_dim) + v1 = self.v_proj_1(x).view(b, s, self.num_kv_heads, self.head_dim) + a1 = torch.ops.auto_deploy.torch_attention( + q1, + k1, + v1, + attn_mask=None, + dropout_p=0.0, + is_causal=True, + scale=None, + sinks=None, + sliding_window=None, + logit_cap=None, + layout="bsnd", + ) + return x + self.o_proj_1(a1.reshape(b, s, -1)) + + +def _build_optimizer_with_backend(model, backend="triton_paged"): + """Helper to create an InferenceOptimizer for testing.""" + return InferenceOptimizer( + DummyFactory(model, cache_config_updates={}), + { + "build_model": { + "stage": "factory", + "run_per_gm": False, + "device": "cuda", + "run_graph_cleanup": False, + "requires_clean_graph": False, + }, + "export_to_gm": { + "stage": "export", + "strict": False, + "run_per_gm": False, + "clone_state_dict": True, + "run_graph_cleanup": False, + "requires_clean_graph": False, + }, + "cleanup_input_constraints": { + "stage": "post_export", + }, + "insert_cached_attention": { + "stage": "cache_init", + "backend": backend, + }, + }, + ) + + +@torch.inference_mode() +def test_insert_cached_attention_no_sliding_window_leaves_config_unchanged(): + """Verify insert_cached_attention does not set max_attention_window for non-SWA models.""" + max_seq_len = 128 + batch_size = 4 + + kv_cache_config = KvCacheConfig( + tokens_per_block=max_seq_len, + max_tokens=batch_size * max_seq_len, + free_gpu_memory_fraction=0.0, + ) + cm = CachedSequenceInterface( + max_seq_len=max_seq_len, + max_batch_size=batch_size, + max_num_tokens=default_max_num_tokens(max_seq_len, batch_size), + device="cuda", + kv_cache_config=kv_cache_config, + ) + + model = GQAWithSdpaAndEmbedding( + num_attention_heads=8, + hidden_size=512, + num_key_value_heads=8, + ).to(dtype=torch.float16, device="cuda") + + optimizer = _build_optimizer_with_backend(model) + optimizer(cm) + + assert cm.kv_cache_config.max_attention_window is None + + +@torch.inference_mode() +def test_insert_cached_attention_respects_user_override(): + """Verify insert_cached_attention does not overwrite user-set max_attention_window.""" + max_seq_len = 128 + batch_size = 4 + user_window = [64] + + kv_cache_config = KvCacheConfig( + tokens_per_block=max_seq_len, + max_tokens=batch_size * max_seq_len, + free_gpu_memory_fraction=0.0, + max_attention_window=user_window, + ) + cm = CachedSequenceInterface( + max_seq_len=max_seq_len, + max_batch_size=batch_size, + max_num_tokens=default_max_num_tokens(max_seq_len, batch_size), + device="cuda", + kv_cache_config=kv_cache_config, + ) + + model = SlidingWindowGQA( + num_attention_heads=8, + hidden_size=512, + num_key_value_heads=8, + sliding_window=32, + ).to(dtype=torch.float16, device="cuda") + + optimizer = _build_optimizer_with_backend(model) + optimizer(cm) + + # User-provided value must be preserved + assert cm.kv_cache_config.max_attention_window == user_window + + +@torch.inference_mode() +def test_insert_cached_attention_vswa_preserves_per_layer_windows(): + """Verify VSWA model preserves per-layer window sizes for proportional allocation.""" + sliding_window = 32 + max_seq_len = 128 + batch_size = 4 + + kv_cache_config = KvCacheConfig( + tokens_per_block=32, + max_tokens=batch_size * max_seq_len, + free_gpu_memory_fraction=0.0, + ) + cm = CachedSequenceInterface( + max_seq_len=max_seq_len, + max_batch_size=batch_size, + max_num_tokens=default_max_num_tokens(max_seq_len, batch_size), + device="cuda", + kv_cache_config=kv_cache_config, + ) + + model = VSWAModel( + num_attention_heads=8, + hidden_size=512, + num_key_value_heads=8, + sliding_window=sliding_window, + ).to(dtype=torch.float16, device="cuda") + + optimizer = _build_optimizer_with_backend(model) + optimizer(cm) + + # VSWA preserves per-layer windows: [32, 128] (not collapsed) + assert cm.kv_cache_config.max_attention_window is not None + assert len(cm.kv_cache_config.max_attention_window) == 2 + assert cm.kv_cache_config.max_attention_window == [sliding_window, max_seq_len] + + # Window groups should be registered on SequenceInfo + assert cm.info.num_window_groups == 2 + assert cm.info.window_groups == [sliding_window, max_seq_len] + assert cm.info.window_group_map == {sliding_window: 0, max_seq_len: 1} + + +@torch.inference_mode() +def test_kv_cache_manager_initialized_with_sliding_window(): + """Verify KVCacheManager receives max_attention_window_vec from SWA model. + + Runs the full insert_cached_attention + initialize_cache pipeline. + """ + import math + + sliding_window = 32 + max_seq_len = 128 + batch_size = 4 + tokens_per_block = 16 + + kv_cache_config = KvCacheConfig( + tokens_per_block=tokens_per_block, + max_tokens=batch_size * max_seq_len, + free_gpu_memory_fraction=0.0, + ) + cm = CachedSequenceInterface( + max_seq_len=max_seq_len, + max_batch_size=batch_size, + max_num_tokens=default_max_num_tokens(max_seq_len, batch_size), + device="cuda", + kv_cache_config=kv_cache_config, + ) + + model = SlidingWindowGQA( + num_attention_heads=8, + hidden_size=512, + num_key_value_heads=8, + sliding_window=sliding_window, + ).to(dtype=torch.float16, device="cuda") + + # Run insert_cached_attention + initialize_cache + optimizer = InferenceOptimizer( + DummyFactory(model, cache_config_updates={}), + { + "build_model": { + "stage": "factory", + "run_per_gm": False, + "device": "cuda", + "run_graph_cleanup": False, + "requires_clean_graph": False, + }, + "export_to_gm": { + "stage": "export", + "strict": False, + "run_per_gm": False, + "clone_state_dict": True, + "run_graph_cleanup": False, + "requires_clean_graph": False, + }, + "cleanup_input_constraints": { + "stage": "post_export", + }, + "insert_cached_attention": { + "stage": "cache_init", + "backend": "triton_paged", + }, + "initialize_cache": { + "stage": "cache_init", + "run_per_gm": False, + }, + }, + ) + optimizer(cm) + + # KVCacheManager should exist and carry the window vector + mgr = cm.kv_cache_manager + assert mgr.max_attention_window_vec == [sliding_window] + + # max_blocks_per_seq is based on max_seq_len (not window) because sequences + # temporarily need full blocks during prefill; SWA eviction frees them during decode. + expected_max_blocks = math.ceil(max_seq_len / tokens_per_block) + assert cm.info.max_blocks_per_seq == expected_max_blocks + + +# ============================================================================= +# VSWA SequenceInfo and Graph Wiring Tests +# ============================================================================= + + +@torch.inference_mode() +def test_sequence_info_register_window_groups(): + """Verify register_window_groups creates per-group tensors in InputBuffer.""" + from tensorrt_llm._torch.auto_deploy.custom_ops.attention_interface import SequenceInfo + + max_seq_len = 128 + batch_size = 4 + tokens_per_block = 16 + + seq_info = SequenceInfo( + max_seq_len=max_seq_len, + max_batch_size=batch_size, + max_num_tokens=default_max_num_tokens(max_seq_len, batch_size), + tokens_per_block=tokens_per_block, + ) + + # Before registration: no window groups + assert seq_info.num_window_groups == 0 + assert seq_info.window_groups == [] + + # Register two groups: [32, 128] + seq_info.register_window_groups([32, 128]) + + assert seq_info.num_window_groups == 2 + assert seq_info.window_groups == [32, 128] + assert seq_info.window_group_map == {32: 0, 128: 1} + + # Group 0 reuses existing tensors (cache_loc, cu_num_pages, etc.) + # Group 1 gets new tensors + assert "cache_loc_g1" in seq_info.available_args + assert "cu_num_pages_g1" in seq_info.available_args + assert "cu_num_pages_g1_host" in seq_info.available_args + assert "last_page_len_g1" in seq_info.available_args + assert "last_page_len_g1_host" in seq_info.available_args + assert "extra_page_per_seq_g1" in seq_info.available_args + + # Group 0 names should NOT appear with _g0 suffix + assert "cache_loc_g0" not in seq_info.available_args diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py new file mode 100644 index 000000000000..6e9043bc96b7 --- /dev/null +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Graph-shape tests for the VSWA per-group metadata wiring in the kvcache transform. + +The kvcache transform builds a per-window page table layout. Phase-1 of the +Gemma-4 enablement (PR #13745) only routed *standard* metadata (cache_loc, +cu_num_pages, last_page_len) per group; the *extra*-metadata op (the host-side +prepare that produces e.g. update_paged_kv_cache write positions) was built +once from group-0 inputs and reused for every layer. With seq_len_with_cache +added to the swappable set (Phase 2), the extra-metadata op MUST also run per +group, otherwise non-zero-group layers silently consume group-0's +seq_len_with_cache (window-uncapped) → wrong write positions under eviction. + +These tests are graph-shape only — no GPU, no kernel execution. +""" + +import pytest +import torch + +import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401 +from tensorrt_llm._torch.auto_deploy.export import torch_export_to_gm +from tensorrt_llm._torch.auto_deploy.shim.interface import CachedSequenceInterface +from tensorrt_llm._torch.auto_deploy.transform.interface import SharedConfig, Stages +from tensorrt_llm._torch.auto_deploy.transform.library.kvcache import ( + InsertCachedAttention, + InsertCachedAttentionConfig, +) + + +class _TwoWindowModule(torch.nn.Module): + """A tiny two-layer model with one SWA layer and one full-attention layer. + + Layer 0 is SWA (sliding_window=256), layer 1 is full attention. This + forces the transform to create two window groups. + """ + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + qkv = hidden_states.view(hidden_states.shape[0], hidden_states.shape[1], 2, 4) + swa_layer = torch.ops.auto_deploy.torch_attention( + qkv, + qkv, + qkv, + None, # attn_mask + 0.0, # dropout_p + True, # is_causal + 1.0, # scale + None, # sinks + 256, # sliding_window <-- SWA group + None, # logit_cap + "bsnd", + 0, # layer_idx + ) + full_layer = torch.ops.auto_deploy.torch_attention( + qkv, + qkv, + qkv, + None, + 0.0, + True, + 1.0, + None, + None, # sliding_window=None <-- full-attention group + None, + "bsnd", + 1, + ) + return swa_layer + full_layer + + +def _run_transform(backend: str): + module = _TwoWindowModule().eval() + gm = torch_export_to_gm(module, (torch.randn(1, 4, 8),)) + cm = CachedSequenceInterface( + max_seq_len=512, + max_batch_size=2, + max_num_tokens=512, + device="cpu", + ) + transform = InsertCachedAttention( + InsertCachedAttentionConfig(stage=Stages.CACHE_INIT, backend=backend) + ) + gm, info = transform._apply(gm, cm, factory=None, shared_config=SharedConfig()) + return gm, info, cm + + +def _placeholder_names(gm) -> list: + return [node.target for node in gm.graph.nodes if node.op == "placeholder"] + + +def test_vswa_registers_per_group_seq_len_with_cache_placeholders(): + """Group 1 must have its own seq_len_with_cache_g1{_host} placeholders. + + This is required so the SWA group's window-capped value reaches the + kernel and the prepare-extra-metadata op. + """ + gm, info, cm = _run_transform(backend="triton_paged") + assert info.num_matches == 2, "expected 2 cached-attention insertions" + + names = _placeholder_names(gm) + # Group 0 keeps the unsuffixed name; group 1 must have its own _g1 variant. + assert "seq_len_with_cache_host" in names + assert "seq_len_with_cache_g1_host" in names, ( + "seq_len_with_cache must be in vswa_swappable_bases — without per-group " + "wiring, SWA layers silently consume group-0's window-uncapped value." + ) + + +def test_vswa_extra_metadata_is_per_group(): + """Prepare-extra-metadata must be invoked once per window group. + + The Triton backend's prepare-extra op consumes seq_len_with_cache. With + seq_len_with_cache routed per-group, the transform MUST invoke the + extra-op once per group; otherwise non-zero-group layers route through + group-0's prepare-extra output and the swappable seq_len_with_cache_g1 + placeholder is dead. + + This is the regression guard against the pre-fix transform behavior + described in the autodeploy-kvcache-vswa-meta-nodes-extra backlog. + """ + gm, info, cm = _run_transform(backend="triton_paged") + + prep_meta_op = torch.ops.auto_deploy.triton_paged_prepare_metadata.default + prep_calls = [ + node + for node in gm.graph.nodes + if node.op == "call_function" and node.target == prep_meta_op + ] + assert len(prep_calls) == 2, ( + f"expected one prepare-extra call per group (2 groups), got {len(prep_calls)}" + ) + + # The two calls must consume DIFFERENT seq_len_with_cache placeholders: + # one from group 0 (unsuffixed), one from group 1 (suffixed). + swc_inputs_by_call = [] + for call in prep_calls: + swc_input = None + for arg in call.args: + if ( + isinstance(arg, torch.fx.Node) + and arg.op == "placeholder" + and "seq_len_with_cache" in str(arg.target) + ): + swc_input = str(arg.target) + break + assert swc_input is not None, ( + f"prepare-extra call {call.format_node()} did not consume any " + f"seq_len_with_cache placeholder" + ) + swc_inputs_by_call.append(swc_input) + + assert len(set(swc_inputs_by_call)) == 2, ( + f"both prepare-extra calls consume the same seq_len_with_cache " + f"input ({swc_inputs_by_call}); per-group routing failed" + ) + + +def test_vswa_each_layer_routes_to_its_groups_extra_metadata(): + """Each layer's cached-attn must wire to its own group's prepare-extra outputs. + + The cached-attn call for group 1 (the SWA layer) must consume the + group-1 prepare-extra outputs, not group 0's. + """ + gm, info, cm = _run_transform(backend="triton_paged") + + cached_op = torch.ops.auto_deploy.triton_paged_mha_with_cache.default + cached_calls = [ + node for node in gm.graph.nodes if node.op == "call_function" and node.target == cached_op + ] + assert len(cached_calls) == 2 + + # For each cached-attn call, find which seq_len_with_cache placeholder its + # extra-metadata args ultimately depend on. We do a simple recursive walk + # through args (bounded by graph size) since the prepare-extra outputs are + # getitem nodes of a call_function whose args include the placeholder. + def find_swc_dep(node, visited=None): + if visited is None: + visited = set() + if id(node) in visited: + return None + visited.add(id(node)) + if isinstance(node, torch.fx.Node): + if node.op == "placeholder" and "seq_len_with_cache" in str(node.target): + return str(node.target) + if node.op in ("call_function", "call_method"): + for arg in list(node.args) + list(node.kwargs.values()): + result = find_swc_dep(arg, visited) + if result is not None: + return result + return None + + deps_per_call = [] + for call in cached_calls: + # The first three args are q/k/v; standard metadata follows. Walk all + # args until we find a seq_len_with_cache placeholder dependency. + dep = None + for arg in call.args: + dep = find_swc_dep(arg) + if dep is not None: + break + assert dep is not None, "no seq_len_with_cache dependency found" + deps_per_call.append(dep) + + # The two cached-attn calls (one per layer = one per group) must each route + # to a distinct seq_len_with_cache placeholder. + assert len(set(deps_per_call)) == 2, ( + f"both cached-attn calls depend on the same seq_len_with_cache " + f"placeholder ({deps_per_call}); per-group wiring failed" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 0715e15aa8f3d10eb17cd0c265742694fbcd6db4 Mon Sep 17 00:00:00 2001 From: Grzegorz Kwasniewski <213329731+greg-kwasniewski1@users.noreply.github.com> Date: Thu, 28 May 2026 17:38:59 +0200 Subject: [PATCH 090/308] [TRTLLM-13960][test] Offline equivalence test for sharding IR (#13963) Signed-off-by: greg-kwasniewski1 <213329731+greg-kwasniewski1@users.noreply.github.com> --- .claude/skills/ad-sharding-ir-port/SKILL.md | 52 +- .../_utils_test/_sharding_ir_helpers.py | 518 ++++++++++++++++++ .../transformations/library/conftest.py | 58 ++ .../library/test_sharding_ir_equivalence.py | 421 ++++++++++++++ 4 files changed, 1046 insertions(+), 3 deletions(-) create mode 100644 tests/unittest/auto_deploy/_utils_test/_sharding_ir_helpers.py create mode 100644 tests/unittest/auto_deploy/multigpu/transformations/library/conftest.py create mode 100644 tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py diff --git a/.claude/skills/ad-sharding-ir-port/SKILL.md b/.claude/skills/ad-sharding-ir-port/SKILL.md index b619d580452a..1b2aa804d3e0 100644 --- a/.claude/skills/ad-sharding-ir-port/SKILL.md +++ b/.claude/skills/ad-sharding-ir-port/SKILL.md @@ -136,9 +136,11 @@ transforms: enabled: true ``` -Use `world_size: 8` when validating TP head-divisibility. Optional `shard_layers` limits which `layer_type` hints are processed; unset means shard all shardable nodes. +Set `world_size` once, to the **maximum number of GPUs available on the machine**, auto-detected with `python -c 'import torch; print(torch.cuda.device_count())'` (or `nvidia-smi --list-gpus | wc -l`). Do **not** hardcode `world_size: 8` (or any other literal) — porting agents run on heterogeneous hardware and an 8-GPU literal will simply fail to launch on a 2- or 4-GPU machine. If the model's `num_attention_heads` (and, for GQA, `num_key_value_heads`) does not divide the detected GPU count, fall back to the largest power-of-two divisor that does (e.g. 4 on an 8-GPU machine if `num_attention_heads = 12`). Run the end-to-end command exactly once at that size — there is no value in repeating it at multiple smaller sizes, because the offline sharding equivalence test (Step 11b) already exercises 2- and 4-GPU dist configs cheaply. -### Step 11: Validate +Optional `shard_layers` limits which `layer_type` hints are processed; unset means shard all shardable nodes. + +### Step 11a — End-to-end run Do not report success until a run completes successfully. @@ -149,6 +151,49 @@ Do not report success until a run completes successfully. **Layer type strings** (for `layer_type` / `shard_layers`): use `"mha"`, `"mla"`, `"mlp"`, `"moe"`, `"ssm"`, `"delta"`, or `"unknown"` (default; skipped when `shard_layers` is set). Match the conventions used in `apply_sharding_hints` and project enums. +### Step 11b — Sharding equivalence test (MANDATORY) + +Run the offline sharding-IR equivalence test ([`tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py`](tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py)) against the modeling file you just edited, under **every** parallelism configuration the test exposes. The port is **not** complete until every configuration passes. Skipping this step or treating a partial pass (e.g. only `tep`) as success is not allowed. + +The test compares a sharded prefill against the unsharded eager reference on a tiny (4-layer, hidden_size=64) instance of the model and asserts `rel_rmse < tol`, where `tol` is the test-defined relative-RMSE tolerance (`REL_RMSE_TOL` constant in [`test_sharding_ir_equivalence.py`](tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py); overridable per invocation via the `SHARDING_IR_REL_RMSE_TOL` env var). It uses no PyExecutor / no compile / no checkpoint download, so each cell runs in ~30s on 4xGPU. + +**Run the matrix:** + +```bash +MODEL=tensorrt_llm/_torch/auto_deploy/models/custom/modeling_.py +TEST=tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py + +for CFG in tp-only ep-only tep attn-dp; do + pytest "$TEST" --sharding-ir-modeling-file "$MODEL" --sharding-ir-dist-config "$CFG" -s -v \ + 2>&1 | tee /tmp/sharding_ir_${CFG}.log +done +``` + +**Parse the output for each cell. A cell PASSES iff ALL of these are true:** + +1. pytest exit code is `0`. +2. The log contains the line `1 passed` in the pytest summary block. +3. The log contains the rank-0 metrics line `[sharding-ir-eq] |y_s - y_u|: max=... mean=... rel_rmse= (tol=)` and the parsed `rel_rmse` is **strictly less than the parsed `tol`** from the same line. Do not hardcode a tolerance value in the parser — read both `rel_rmse=` and `tol=` from the test's own log and compare them. This stays correct if the test's `REL_RMSE_TOL` is later changed or a per-invocation `SHARDING_IR_REL_RMSE_TOL` is supplied. + +Quick one-liner that prints PASS/FAIL plus the parsed `rel_rmse` and `tol` per cell: + +```bash +for CFG in tp-only ep-only tep attn-dp; do + log=/tmp/sharding_ir_${CFG}.log + if grep -q "1 passed" "$log"; then status=PASS; else status=FAIL; fi + line=$(grep "sharding-ir-eq" "$log" | grep "rel_rmse=" | head -1) + rmse=$(echo "$line" | sed -E 's/.*rel_rmse=([0-9.]+).*/\1/') + tol=$(echo "$line" | sed -E 's/.*\(tol=([0-9.]+)\).*/\1/') + echo "${CFG}: ${status} rel_rmse=${rmse:-NA} tol=${tol:-NA}" +done +``` + +**Failure handling:** + +- A cell failing with `KeyError`, `AttributeError`, `ValueError: You must specify exactly one of input_ids or inputs_embeds`, or any exception *before* `[sharding-ir-eq]` prints means the **modeling code itself** does not yet build / export on a tiny config — fix the modeling code (within the Step 0 allowlist) before proceeding. Do not silently skip the cell. +- A cell where `[sharding-ir-eq]` prints `rel_rmse >= tol` (from the same log line) means a **sharding-hint bug**: a missing `all_reduce`, a wrong `tp_mode`, a `view` without `tp_scaled_dim`, a `split_with_sizes` whose sizes do not scale, etc. Re-read Step 6 (all_reduce), Step 3 (tp_mode), Step 5 (view), Step 4 (split_with_sizes) and the layer-specific patterns. Iterate on the hints until clean. If the failure is small (rel_rmse just slightly above tol) and you have reason to believe it is real numerical noise from the specific layer mix of this model rather than a sharding-hint bug, raise it with the parent agent rather than silently bumping `SHARDING_IR_REL_RMSE_TOL`. +- A cell that the modeling file legitimately does not support (e.g. `ep-only` on a dense model with no MoE) is acceptable only if the failure is a documented `pytest.skip(...)` from the test infrastructure. A silent `FAIL` is **not** acceptable. + ### Step 12 — Pre-finalization self-audit (MANDATORY) Before reporting the file as done, you MUST diff your changes against the git baseline: @@ -216,7 +261,8 @@ You are NOT done until every row in the table is a yes-allowed category. ## Validation checklist (human review) +- All four configurations of the **sharding equivalence test** (Step 11b) pass with the parsed `rel_rmse` strictly below the parsed `tol` from the same rank-0 log line. Report the per-cell `rel_rmse` and `tol` pair. - `world_size=1`: unsharded path; hints should not break correctness. -- `world_size=2` and `8`: shape checks and coherent output. +- `world_size=`: end-to-end run (Step 11a) at the maximum GPU count auto-detected on the machine (head-divisibility permitting; see Step 11). - `apply_sharding_hints` node count vs expectation. - Optional: `shard_layers: ['moe']` to verify selective sharding. diff --git a/tests/unittest/auto_deploy/_utils_test/_sharding_ir_helpers.py b/tests/unittest/auto_deploy/_utils_test/_sharding_ir_helpers.py new file mode 100644 index 000000000000..8336efc73058 --- /dev/null +++ b/tests/unittest/auto_deploy/_utils_test/_sharding_ir_helpers.py @@ -0,0 +1,518 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Helpers for the offline sharding-IR equivalence test. + +The test takes a path to a sharding-IR-aware modeling file (e.g. +``tensorrt_llm/_torch/auto_deploy/models/custom/modeling_deepseek.py``, +``modeling_qwen3.py``, or any ``modeling_.py`` whose canonical +implementation uses the sharding-IR path) and builds a tiny variant of that +model -- few layers, small hidden size, random weights -- without touching +the filesystem or LLM_MODELS_ROOT. + +The path is the *only* user input: everything else (Python module name, +``*ForCausalLM`` class, HF config class) is derived from the file by walking +``AutoModelForCausalLMFactory._custom_model_mapping`` (populated when the +modeling module is imported). No assumption is made about the filename -- +post-#13478 the IR-aware implementation is the canonical version for +deepseek/nemotron_h/qwen3/qwen3_5_moe (no ``_ir`` suffix); the helpers also +work for any other modeling file that opts into the sharding-IR path. + +The tiny config is a single universal ``tiny_kwargs`` dict applied with +``setattr`` (so fields not used by a given model are silent no-ops), plus a +hasattr-driven feature-detection pass for the residue that depends on +``num_hidden_layers``. +""" + +import importlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Tuple + +import torch +import torch.nn as nn + + +@dataclass(frozen=True) +class IRModelSpec: + """Wires a modeling-file path to the classes the test will instantiate. + + All four fields are derived from the path by :func:`spec_from_modeling_file`; + this class is just a typed bag of the derived values. + """ + + config_module: str + """Python path of the HF ``configuration_*.py`` module owning the config.""" + + config_cls: str + """Class name of the HF config (e.g. ``Qwen3Config``).""" + + modeling_module: str + """Python path of the sharding-IR modeling module.""" + + modeling_cls: str + """Class name of the ``*ForCausalLM`` to instantiate.""" + + +# ----------------------------------------------------------------------------- +# Tiny-config building blocks +# ----------------------------------------------------------------------------- + +# Single shared kitchen-sink dict. Covers fields that have a single sensible +# scalar default across all currently-onboarded sharding-IR model families +# (dense, GQA, MoE, MLA, SSM/Mamba). Applied with ``setattr`` (not constructor +# kwargs), so fields not read by a given config are harmless no-ops -- no +# per-family gating needed. +_TINY_KWARGS_UNIVERSAL: Dict[str, Any] = { + # 4 layers so the rotation (mamba, attention, moe, mamba) covers every + # block family AND a uniform-scale bug at layer N gets re-normalized into + # a non-uniform-shape error at layer N+1 -- otherwise the final + # ``norm_f`` RMSNorm before ``lm_head`` is scale-invariant and hides + # uniform-scaling bugs (e.g. a missing all_reduce after a rowwise linear). + "num_hidden_layers": 4, + "hidden_size": 64, + "intermediate_size": 64, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "head_dim": 16, + "vocab_size": 64, + "max_position_embeddings": 256, + "rope_theta": 1000000.0, + # ``rope_scaling`` is intentionally NOT set here: in transformers 5.x, + # ``PretrainedConfig.__setattr__`` aliases ``rope_scaling`` <-> + # ``rope_parameters`` and on composite configs (e.g. + # ``Qwen3_5MoeConfig``) propagates the value into ``text_config``, + # so a universal ``rope_scaling = None`` would also nuke + # ``text_config.rope_parameters`` and break models that read it + # (e.g. ``Qwen3_5MoeTextRotaryEmbedding``). DeepSeek-V3 needs a + # ``factor`` key in its ``rope_scaling`` dict to clear + # ``DeepSeekV3Attention.__init__``; that family-specific patch is + # handled in ``_apply_layer_count_dependent_quirks`` below, gated + # on the DeepSeek MLA marker so it does not touch other families. + # MoE -- deepseek requires num_experts % n_group == 0 (n_group defaults to 8) + "num_experts": 8, + "num_experts_per_tok": 2, + "num_local_experts": 8, + "moe_intermediate_size": 16, + "first_k_dense_replace": 0, + "n_routed_experts": 8, + "n_shared_experts": 1, + # DeepSeek-V3 alternates dense and MoE blocks every ``moe_layer_freq`` + # layers (``layer_idx % moe_layer_freq == 0`` -> MoE). Real configs ship + # ``moe_layer_freq = 1`` (every layer is MoE) but ``DeepseekV3Config()`` + # leaves the attribute unset, which trips ``DeepSeekV3DecoderLayer`` + # before sharding ever runs. + "moe_layer_freq": 1, + # MLA (DeepSeek-V3) + "q_lora_rank": 8, + "kv_lora_rank": 8, + "qk_rope_head_dim": 8, + "qk_nope_head_dim": 8, + "v_head_dim": 8, + # SSM (NemotronH / Mamba) + "ssm_state_size": 8, + "mamba_d_conv": 4, + "mamba_expand": 2, + # GDN / linear attention (Qwen3.5-MoE delta block) -- defaults are huge + # (value_dim = 32 * 128 = 4096), so shrink to per-tp_size friendly sizes. + "linear_num_value_heads": 4, + "linear_num_key_heads": 4, + "linear_key_head_dim": 16, + "linear_value_head_dim": 16, + "linear_conv_kernel_dim": 4, +} + + +def fix_moe_routers_deterministic(model) -> int: + """Force MoE routers to deterministically pick experts ``[0..top_k-1]``. + + Walks ``model.named_modules`` looking for modules whose class name contains + ``Router`` or ``Gate`` (e.g. ``Qwen3_5MoeTopKRouter``, ``DeepSeekV3MoEGate``, + ``NemotronHTopkRouter``) with a 2-d ``weight`` of shape + ``(num_experts, hidden_size)`` and ``num_experts <= 64``. For each match: + + * ``weight[i, :] = (num_experts - i) / sqrt(H)`` -- small but monotonic + decreasing coefficient over experts, keeps the linear projection in the + export graph. + * ``e_score_correction_bias[i] = (num_experts - i) * 100`` (if present) + -- the grouped-top-k path in DeepSeek-V3 / Nemotron-H reads this bias, + so making it large and monotonic dominates the routing decision. + * For Qwen3.5-MoE routers (no built-in bias), the router's ``forward`` is + replaced by a version that adds a strong monotonic logit bias before + softmax / top-k. The bias dominates the linear contribution from + ``hidden_states``, so top-k always picks experts ``[0..top_k-1]`` + regardless of input -- killing the cross-precision routing flips that + would otherwise mask sharding bugs vs reduction-order noise. + + Router weights themselves are not TP-sharded, so both the unsharded and + sharded forwards see the same router output for a given token. Returns the + count of fixed router modules. + """ + import types + + import torch + import torch.nn.functional as F + + def _patched_qwen_router_forward(self, hidden_states): + hidden_states = hidden_states.reshape(-1, self.hidden_dim) + router_logits = F.linear(hidden_states, self.weight) + self._test_router_bias + routing_weights = F.softmax(router_logits, dtype=torch.float, dim=-1) + routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) + routing_weights = routing_weights / routing_weights.sum(dim=-1, keepdim=True) + return routing_weights.to(hidden_states.dtype), selected_experts + + fixed = 0 + with torch.no_grad(): + for name, mod in model.named_modules(): + cls = type(mod).__name__ + if "Router" not in cls and "Gate" not in cls: + continue + if not hasattr(mod, "weight"): + continue + w = mod.weight + if w.ndim != 2: + continue + num_experts, hidden = w.shape + if num_experts > 64: + continue # not a router (probably a FFN projection) + coeffs_fp32 = torch.arange(num_experts, 0, -1, dtype=torch.float32, device=w.device) + new_w = ( + (coeffs_fp32 / (hidden**0.5)).unsqueeze(1).expand(num_experts, hidden).to(w.dtype) + ) + w.data.copy_(new_w) + if hasattr(mod, "e_score_correction_bias"): + # Used by DeepSeek-V3 / Nemotron-H grouped top-k path (noaux_tc_op). + b = mod.e_score_correction_bias + b.data.copy_((coeffs_fp32 * 100.0).to(b.dtype)) + elif cls == "Qwen3_5MoeTopKRouter": + # Qwen3.5-MoE has no built-in bias; install one and patch forward. + bias = (coeffs_fp32 * 100.0).to(w.dtype) + if not hasattr(mod, "_test_router_bias"): + mod.register_buffer("_test_router_bias", bias) + else: + mod._test_router_bias.data.copy_(bias) + mod.forward = types.MethodType(_patched_qwen_router_forward, mod) + fixed += 1 + return fixed + + +def _balanced_layers_block_type(num_layers: int) -> list: + """Construct a ``num_layers``-long list rotating over the supported block types. + + Used to patch hybrid SSM configs that store per-layer block types as a + list (e.g. ``NemotronHConfig.layers_block_type``). The rotation covers + ``"mamba"`` (SSM), ``"attention"`` (self-attn), and ``"moe"`` (mixture of + experts) so all three families' branches get exercised by the sharding-IR + equivalence test, including the MoE path inside hybrid models. With + ``num_layers >= 3`` the resulting pattern contains at least one of each. + """ + rotation = ("mamba", "attention", "moe") + return [rotation[i % len(rotation)] for i in range(num_layers)] + + +def _is_writable_attr(obj: Any, name: str) -> bool: + """``True`` iff ``name`` is a real settable attribute on ``obj``. + + Filters out getter-only ``property`` descriptors (e.g. + ``NemotronHConfig.hybrid_override_pattern`` is a backward-compat property + derived from the settable ``layers_block_type`` list, not a settable + field itself). Plain instance attributes and properties with a setter + return ``True``. + """ + cls_attr = getattr(type(obj), name, None) + if isinstance(cls_attr, property): + return cls_attr.fset is not None + return hasattr(obj, name) + + +def _apply_layer_count_dependent_quirks(config: Any, num_layers: int) -> None: + """Patch config fields whose valid value depends on ``num_layers``. + + The kitchen-sink :data:`_TINY_KWARGS_UNIVERSAL` covers fields that have a + single sensible scalar default across all model families. This function + handles the residue: fields whose value has to *match* ``num_layers`` or + encode a per-layer layout (e.g. hybrid Mamba/Attention interleaving + expressed as a per-layer character string or list). + + *This is a heuristic, not an exhaustive check.* Each branch was added in + response to a real model surfacing a layer-count-dependent field during + bring-up. When a future model surfaces a new such field, add a new + ``hasattr`` branch here -- dispatch is by *feature presence on the config + object*, not by the modeling-file name. That way the patch fires for any + model that exposes the same field, independent of file naming or family. + + Currently handled: + + * ``layers_block_type`` -- per-layer list of block-type strings + (``"mamba"`` / ``"attention"`` / ``"moe"``). Hybrid SSM models validate + ``len(layers_block_type) == num_hidden_layers``, so the default list + shipped with a full-size config rejects a 4-layer override. We replace + it with a list that rotates over all three block types so that each + family branch (SSM, attention, MoE) is exercised by the test. (Some + configs expose a derived ``hybrid_override_pattern`` getter on top of + this list; that getter is *not* settable, so we patch the underlying + list.) + + *To extend*: add a new branch of the form:: + + if _is_writable_attr(config, ""): + config. = + + and document the new case in the list above. Use + :func:`_is_writable_attr` rather than bare ``hasattr`` so getter-only + ``property`` attributes are skipped (they raise ``AttributeError`` on + assignment). + """ + if _is_writable_attr(config, "layers_block_type"): + config.layers_block_type = _balanced_layers_block_type(num_layers) + + +def _apply_per_family_quirks(config: Any) -> None: + """Patch config fields whose default is broken for a specific family. + + These are NOT layer-count dependent and NOT safe to put in + :data:`_TINY_KWARGS_UNIVERSAL`, because the universal kwargs are + applied via ``setattr`` to every model's config, and some keys + (notably ``rope_scaling``) trigger aliasing / sub-config + propagation inside ``PretrainedConfig.__setattr__`` that would + break other families. Each branch dispatches on a feature marker + on the config object (never on the filename) so it fires for any + config that exposes the same field, independent of family naming. + + **Ordering invariant**: must be called on a *pristine* config -- + i.e. before :data:`_TINY_KWARGS_UNIVERSAL` is applied -- because + the universal kwargs ``setattr`` many family-specific keys + (``kv_lora_rank``, ``q_lora_rank``, ``ssm_state_size``, ...) onto + every config to keep one tiny-kwargs dict universal. After the + universal pass every config looks like every family, so + feature-presence detection here would over-match. + + Currently handled: + + * ``kv_lora_rank`` -- present iff this is a DeepSeek-V3 / MLA + family config. ``modeling_deepseek.DeepSeekV3Attention.__init__`` + reads ``config.rope_scaling["factor"]`` unconditionally when + ``rope_scaling is not None``. In production deployments the real + DeepSeek-V3 checkpoint ships a full yarn dict that includes + ``factor``, but a default-constructed ``DeepseekV3Config`` in + transformers 5.x sets ``rope_scaling = {"rope_type": "default"}`` + without the yarn keys, so the lookup raises ``KeyError: 'factor'`` + before any sharding work runs. We override with a minimal dict + that (a) provides ``factor`` to clear the unconditional lookup + and (b) keeps ``rope_type = "default"`` so ``_init_rope`` routes + through the vanilla rotary branch (correct stimulus for a sharding + equivalence test; yarn extrapolation behaviour is out of scope). + """ + if _is_writable_attr(config, "kv_lora_rank"): + config.rope_scaling = {"rope_type": "default", "factor": 1.0} + + +# ----------------------------------------------------------------------------- +# Spec derivation from a modeling-file path +# ----------------------------------------------------------------------------- + + +def _path_to_dotted_module(path: str) -> str: + """Convert a modeling-file path to its Python dotted module name. + + Accepts: + + * An absolute path: ``/.../tensorrt_llm/_torch/auto_deploy/models/custom/modeling_x.py`` + * A path relative to cwd or repo root: ``tensorrt_llm/_torch/.../modeling_x.py`` + * A bare module short name: ``modeling_x`` (resolved under + ``tensorrt_llm._torch.auto_deploy.models.custom``) + + The conversion is purely syntactic -- no filename pattern is required. + """ + if "/" not in path and not path.endswith(".py"): + return f"tensorrt_llm._torch.auto_deploy.models.custom.{path}" + + p = Path(path).resolve() if not Path(path).is_absolute() else Path(path) + p = p.with_suffix("") + parts = p.parts + if "tensorrt_llm" not in parts: + raise ValueError(f"Path {path!r} does not contain a 'tensorrt_llm' package root anchor.") + idx = parts.index("tensorrt_llm") + return ".".join(parts[idx:]) + + +def _resolve_config_cls_from_transformers_registry(config_cls_name: str) -> Any: + """Look up an HF config class by its ``__name__`` via the transformers registry. + + Iterates ``transformers.models.auto.configuration_auto.CONFIG_MAPPING_NAMES`` + (a ``model_type -> config_class_name`` dict) to find the model_type whose + config name matches, then resolves the (lazy) entry via ``CONFIG_MAPPING``. + Returns ``None`` if the class isn't in the upstream transformers registry. + """ + from transformers.models.auto.configuration_auto import CONFIG_MAPPING, CONFIG_MAPPING_NAMES + + for model_type, name in CONFIG_MAPPING_NAMES.items(): + if name == config_cls_name: + try: + return CONFIG_MAPPING[model_type] + except KeyError: + return None + return None + + +def spec_from_modeling_file(path: str) -> IRModelSpec: + """Derive a full :class:`IRModelSpec` from any modeling-file path. + + Makes no assumption about filename -- works for canonical + ``modeling_qwen3.py`` (post-#13478), legacy ``modeling__ir.py`` if + any still exist, or any future ``modeling_.py``. Importing the + module triggers its self-registration via + ``AutoModelForCausalLMFactory.register_custom_model_cls("", )``; + we then look up the registered ``*ForCausalLM`` class and resolve its + HF config class. + + The HF config class is resolved in this order: + + 1. ``model_cls.config_class`` -- the HF convention; preferred. + 2. The modeling module's own globals, looked up by the registered config + class *name* (the registration key) -- works for IR files that import + their config class at the top. + 3. The upstream transformers registry + (``transformers.models.auto.configuration_auto.CONFIG_MAPPING``) -- + works for any config class transformers knows about, including IR + files that only reference the config class by string in their + registration call. + """ + module_name = _path_to_dotted_module(path) + mod = importlib.import_module(module_name) + + # Deferred import: pulls in tensorrt_llm and so must run *after* the + # caller has done any necessary sys.path / env-var setup. + from tensorrt_llm._torch.auto_deploy.models.hf import AutoModelForCausalLMFactory + + candidates = [ + (cfg_name, cls) + for cfg_name, cls in AutoModelForCausalLMFactory._custom_model_mapping.items() + if cls.__module__ == module_name and cls.__name__.endswith("ForCausalLM") + ] + if not candidates: + raise RuntimeError( + f"No '*ForCausalLM' class registered from {module_name!r}. " + "Ensure the modeling file ends with a " + "'AutoModelForCausalLMFactory.register_custom_model_cls(...)' " + "call for a 'ForCausalLM' class." + ) + config_cls_name, model_cls = candidates[0] + + config_cls = getattr(model_cls, "config_class", None) + if config_cls is None: + config_cls = getattr(mod, config_cls_name, None) + if config_cls is None: + config_cls = _resolve_config_cls_from_transformers_registry(config_cls_name) + if config_cls is None: + raise RuntimeError( + f"Could not resolve config class {config_cls_name!r} for " + f"{model_cls.__module__}.{model_cls.__name__}. Tried " + f"`model_cls.config_class`, the modeling module's globals, and " + f"the transformers CONFIG_MAPPING. Either set " + f"`{model_cls.__name__}.config_class = {config_cls_name}` in the " + f"modeling file, or import {config_cls_name} at the module top." + ) + + return IRModelSpec( + config_module=config_cls.__module__, + config_cls=config_cls.__name__, + modeling_module=module_name, + modeling_cls=model_cls.__name__, + ) + + +# ----------------------------------------------------------------------------- +# Tiny model build + forward helpers +# ----------------------------------------------------------------------------- + + +def build_ir_model(spec: IRModelSpec, device: torch.device, dtype: torch.dtype) -> nn.Module: + """Programmatically build the IR-onboarded model with a tiny config. + + Does not touch the filesystem and does not require LLM_MODELS_ROOT. The + universal :data:`_TINY_KWARGS_UNIVERSAL` is applied with ``setattr`` on a + default-constructed config; this works for fields not accepted as + constructor kwargs in the installed transformers version. Family-specific + field defaults that the universal kwargs cannot safely cover are patched + via :func:`_apply_per_family_quirks`, which runs **before** the universal + kwargs are applied so that its feature-presence dispatch reads the + pristine config (the universal kwargs ``setattr`` many family-specific + keys onto every config, which would otherwise cause spurious matches). + Layer-count dependent fields are then patched via + :func:`_apply_layer_count_dependent_quirks`. + """ + cfg_module = importlib.import_module(spec.config_module) + cfg_cls = getattr(cfg_module, spec.config_cls) + config = cfg_cls() + _apply_per_family_quirks(config) + for k, v in _TINY_KWARGS_UNIVERSAL.items(): + setattr(config, k, v) + _apply_layer_count_dependent_quirks(config, _TINY_KWARGS_UNIVERSAL["num_hidden_layers"]) + + modeling_module = importlib.import_module(spec.modeling_module) + model_cls = getattr(modeling_module, spec.modeling_cls) + model = model_cls(config).to(device=device, dtype=dtype).eval() + return model + + +def extract_logits(out: Any) -> torch.Tensor: + """Pull the logits tensor out of a model forward result. + + Accepts a raw tensor (post torch.export typically yields a tuple), a tuple + or list with logits at position 0, or an HF ``ModelOutput`` with a + ``.logits`` attribute. + """ + if isinstance(out, torch.Tensor): + return out + if isinstance(out, (tuple, list)): + return out[0] + if hasattr(out, "logits"): + return out.logits + raise TypeError(f"Cannot extract logits from forward output of type {type(out)}") + + +def build_random_prefill_inputs( + batch_size: int, + seq_len: int, + vocab_size: int, + device: torch.device, + seed: int = 42, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Deterministic ``(input_ids, position_ids)`` for the equivalence prefill.""" + gen = torch.Generator(device=device).manual_seed(seed) + input_ids = torch.randint( + 0, vocab_size, (batch_size, seq_len), device=device, dtype=torch.long, generator=gen + ) + position_ids = ( + torch.arange(seq_len, device=device, dtype=torch.long) + .unsqueeze(0) + .expand(batch_size, seq_len) + .contiguous() + ) + return input_ids, position_ids + + +def random_init_with_seed(model: nn.Module, seed: int, std: float = 0.02) -> None: + """Re-initialize model parameters in-place with deterministic random values. + + Uses a CPU ``Generator`` to avoid relying on rank-dependent CUDA RNG state; + every rank that calls this with the same ``seed`` ends up with bit-identical + weights, which is what the equivalence test relies on. Random draws are + always done at fp32 and cast to the parameter dtype (fp8/bf16 don't have + a ``normal_kernel_cpu`` implementation). + """ + gen = torch.Generator(device="cpu").manual_seed(seed) + with torch.no_grad(): + for p in model.parameters(): + flat = torch.empty(p.numel(), dtype=torch.float32).normal_(0.0, std, generator=gen) + p.data.copy_(flat.view_as(p).to(dtype=p.dtype, device=p.device)) + for b in model.buffers(): + if b.dtype.is_floating_point: + flat = torch.empty(b.numel(), dtype=torch.float32).normal_(0.0, std, generator=gen) + b.data.copy_(flat.view_as(b).to(dtype=b.dtype, device=b.device)) diff --git a/tests/unittest/auto_deploy/multigpu/transformations/library/conftest.py b/tests/unittest/auto_deploy/multigpu/transformations/library/conftest.py new file mode 100644 index 000000000000..ee9d38022b25 --- /dev/null +++ b/tests/unittest/auto_deploy/multigpu/transformations/library/conftest.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Local conftest for multigpu/transformations/library tests.""" + +import pytest + +_DIST_CONFIG_CHOICES = ("tp-only", "ep-only", "tep", "attn-dp") +_DIST_CONFIG_DEFAULT = "tep" + + +def pytest_addoption(parser): + parser.addoption( + "--sharding-ir-modeling-file", + action="store", + default=None, + help=( + "Path to a sharding-IR-aware modeling file to verify with " + "test_sharding_ir_equivalence. Accepts an absolute path, a path " + "relative to cwd or repo root, or a bare module short name " + "(resolved under tensorrt_llm._torch.auto_deploy.models.custom). " + "No filename pattern is required. The test is skipped (not " + "failed) when this option is absent." + ), + ) + parser.addoption( + "--sharding-ir-dist-config", + action="store", + choices=_DIST_CONFIG_CHOICES, + default=_DIST_CONFIG_DEFAULT, + help=( + "Parallelism config to exercise in test_sharding_ir_equivalence. " + "See test_sharding_ir_equivalence._DIST_CONFIGS for grids: " + "'tp-only' (2 ranks), 'ep-only' (2 ranks), 'tep' (4 ranks, default), " + "'attn-dp' (4 ranks, attention-DP + MoEAllToAll)." + ), + ) + + +@pytest.fixture +def sharding_ir_modeling_file(request) -> str: + path = request.config.getoption("--sharding-ir-modeling-file") + if path is None: + pytest.skip( + "--sharding-ir-modeling-file not supplied; sharding IR equivalence " + "test is only run on-demand per modeling file." + ) + return path + + +@pytest.fixture +def sharding_ir_dist_config(request) -> str: + return request.config.getoption("--sharding-ir-dist-config") diff --git a/tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py b/tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py new file mode 100644 index 000000000000..7dfb3fb65f5d --- /dev/null +++ b/tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +r"""Offline sharding-IR equivalence test. + +Verifies that, for the sharding-IR modeling code at a given file path, +applying the IR sharding transforms (``apply_sharding_hints`` + +``strip_sharding_hints`` + per-rank weight slicing via load hooks) preserves +the prefill numerical output of the same graph under the *unsharded* +configuration. Runs without the full inference runtime: no PyExecutor, no +cache init, no compile, no checkpoint download, no LLM_MODELS_ROOT setup. + +Both sides of the comparison are post-``torch_export_to_gm`` graphs of the +same model instance with the same random weights -- only ``apply_sharding_hints`` ++ ``strip_sharding_hints`` are applied to the sharded side. Any +``torch_export_to_gm`` semantic gap against eager (which exists for some +custom ops, notably MoE with Python loops over experts) is therefore +*invisible* to this test: both sides see identical export behavior, so any +constant bias introduced by export cancels out and only the delta from +sharding remains. + +Usage: + + pytest tests/unittest/auto_deploy/multigpu/transformations/library/test_sharding_ir_equivalence.py \ + --sharding-ir-modeling-file tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3.py + +The test is skipped (not failed) when ``--sharding-ir-modeling-file`` is +absent. No filename pattern is assumed -- works for the canonical +``modeling_.py`` files that became the IR-aware default in #13478 +(deepseek, nemotron_h, qwen3, qwen3_5_moe), and for any future modeling +file that opts into the sharding-IR path. See +``_sharding_ir_helpers.spec_from_modeling_file`` for how class / config +inference works, and ``_apply_layer_count_dependent_quirks`` for the +hasattr-driven layer-count-dependent config patches. + +The ``SHARDING_IR_SABOTAGE=1`` env var is a negative-control switch: it +removes every ``all_reduce`` / ``all_gather`` / ``all_to_all`` node from the +sharded graph after sharding, then runs the same comparison. Used to confirm +the test actually rejects broken sharding (rel_rmse spikes far above the +tolerance) rather than rubber-stamping it. +""" + +import os +import sys +from functools import partial +from pathlib import Path +from typing import Optional + +import pytest +import torch + +# Make sure the helpers directory is importable both when running under +# pytest (which adds it via the ``pythonpath`` directive in +# ``tests/unittest/pytest.ini``) and when running inside a spawn() worker +# that does not inherit pytest's sys.path manipulation. +_HELPERS_DIR = str(Path(__file__).resolve().parents[3] / "_utils_test") +if _HELPERS_DIR not in sys.path: + sys.path.insert(0, _HELPERS_DIR) + +from _sharding_ir_helpers import ( # noqa: E402 + build_ir_model, + build_random_prefill_inputs, + extract_logits, + fix_moe_routers_deterministic, + random_init_with_seed, + spec_from_modeling_file, +) + +# Parallelism configurations exercised by the test. The key is the value of +# ``--sharding-ir-dist-config``; the dict supplies the world_size, the MoE +# TP/EP grid, and the attention-DP flag. ``tp_size`` equals ``world_size`` +# always (DistConfig validates ``moe_tp_size * moe_ep_size * moe_cluster_size +# == tp_size``); ``enable_attention_dp`` flips attention/MLP from TP to DP +# independently of the grid extent. +# +# tp-only: pure TP across 2 ranks, no MoE EP. +# ep-only: pure MoE EP across 2 ranks, no TP. +# tep: 2x2 grid (TP + MoE EP) across 4 ranks (default). +# attn-dp: attention-DP across 4 ranks with MoEAllToAll inside the MoE +# block (pure EP recipe: moe_tp=1, moe_ep=4). +_DIST_CONFIGS = { + "tp-only": dict(world_size=2, moe_tp_size=2, moe_ep_size=1, enable_attention_dp=False), + "ep-only": dict(world_size=2, moe_tp_size=1, moe_ep_size=2, enable_attention_dp=False), + "tep": dict(world_size=4, moe_tp_size=2, moe_ep_size=2, enable_attention_dp=False), + "attn-dp": dict(world_size=4, moe_tp_size=1, moe_ep_size=4, enable_attention_dp=True), +} + +# Tiny prefill: SEQ_LEN = 256 keeps the test under ~10s end-to-end on the +# small (4-layer) configs while still exposing reduction-order issues that a +# single-token forward would miss. BATCH_SIZE = 4 is the max ``world_size`` +# we exercise, so it scatters cleanly under attention-DP. +BATCH_SIZE = 4 +SEQ_LEN = 256 +WEIGHT_SEED = 0 +INPUT_SEED = 42 + +# bf16 is the default forward dtype for unquantized AutoDeploy deployments. +# fp32 would give tighter numerics but production runs in bf16, so that's +# what the equivalence test should validate. With random init std=0.05 and a +# deterministic-router fix applied to MoE blocks, clean sharding produces +# rel_rmse < 0.012 on every IR family; sabotaged sharding produces > 0.05. +FORWARD_DTYPE = torch.bfloat16 + +# Random init std. Small enough that 4 stacked layers don't blow up in bf16, +# large enough that the per-rank contribution missing under sabotage is +# detectable. Anything below ~0.03 makes sabotage indistinguishable from +# noise on dense models; anything above ~0.1 starts triggering bf16 routing +# noise in MoE blocks even with the deterministic-router fix. +INIT_STD = 0.05 + +# Relative-RMSE tolerance: ``||y_s - y_u||_F / ||y_u||_F``. Scale-invariant +# across models with very different output magnitudes (dense models have +# ``|y|`` ~0.08, MoE models ~3.6). Picked to be above the worst clean +# rel_rmse observed on any IR family (~0.012 on qwen3_5_moe due to +# softmax-amplified bf16 noise in router weights) and well below the +# smallest sabotage rel_rmse (~0.05 on dense models). Override via +# ``SHARDING_IR_REL_RMSE_TOL`` env var when triaging. +REL_RMSE_TOL = 0.02 + +pytestmark = pytest.mark.threadleak(enabled=False) + + +def _all_gather_concat(local: torch.Tensor, world_size: int) -> torch.Tensor: + """Gather rank-local tensors across the default process group and concat along dim 0. + + Used to reassemble a full-batch output from the per-rank slabs produced + under attention-DP. + """ + import torch.distributed as dist + + gathered = [torch.empty_like(local) for _ in range(world_size)] + dist.all_gather(gathered, local) + return torch.cat(gathered, dim=0) + + +def _sabotage_remove_collectives(gm) -> int: + """Negative-control hook: replace every collective op in ``gm`` with its first arg. + + Activated by ``SHARDING_IR_SABOTAGE=1``. Erases ``all_reduce`` / + ``all_gather`` / ``all_to_all`` nodes so each rank keeps only its partial + result; the test should then fail with rel_rmse far above + :data:`REL_RMSE_TOL`. Used to verify the test actually detects broken + sharding rather than rubber-stamping it. + """ + n_removed = 0 + for node in list(gm.graph.nodes): + if node.op != "call_function": + continue + tgt = str(node.target) + if "all_reduce" in tgt or "all_gather" in tgt or "all_to_all" in tgt: + if node.args: + node.replace_all_uses_with(node.args[0]) + gm.graph.erase_node(node) + n_removed += 1 + gm.graph.lint() + gm.recompile() + return n_removed + + +def _run_equivalence_job_impl( + modeling_file: str, + rank: int, + world_size: int, + dist_config_name: str, +) -> None: + """Per-rank job body invoked by ``spawn_multiprocess_job``. + + Each rank independently: + 1. Builds the tiny IR model with deterministic random weights and a + monotonic-coefficient MoE router so top-k decisions don't flip under + bf16 reduction-order noise. + 2. Exports the model twice via ``torch_export_to_gm`` -- once as the + unsharded reference, once as the basis for sharding. + 3. Runs ``apply_sharding_hints`` + ``strip_sharding_hints`` on the sharded + copy. Optionally sabotages the sharded graph by removing collectives + (negative-control mode). + 4. Loads the *same* unsharded snapshot into both graphs. The hooks + registered on the sharded graph slice each parameter to the per-rank + shard; the unsharded graph absorbs the snapshot identity-wise. + 5. Forwards both graphs on identical inputs and asserts numerical + equivalence via relative RMSE. + + All ranks use the same CPU-seeded random init so the unsharded reference + is bit-identical across ranks; the sharded forward converges to it via + the all-reduce inserted by sharding. + """ + # Imports are deferred until inside the worker so spawn() picks up any + # parent-side sys.path / env-var setup before tensorrt_llm is loaded. + import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401 + import tensorrt_llm._torch.auto_deploy.models.custom # noqa: F401 -- registers IR classes + import tensorrt_llm._torch.auto_deploy.transform.library # noqa: F401 + from tensorrt_llm._torch.auto_deploy.export import torch_export_to_gm + from tensorrt_llm._torch.auto_deploy.transform.optimizer import InferenceOptimizer + from tensorrt_llm._torch.auto_deploy.utils.dist_config import DistConfig + + torch.cuda.set_device(rank) + device = torch.device(f"cuda:{rank}") + + # ------------------------------------------------------------------ + # 1. Build tiny IR model with deterministic random weights. + # ------------------------------------------------------------------ + spec = spec_from_modeling_file(modeling_file) + model = build_ir_model(spec, device=device, dtype=FORWARD_DTYPE) + random_init_with_seed(model, seed=WEIGHT_SEED, std=INIT_STD) + # MoE top-k routing is a non-smooth ``argmax`` whose decisions can flip + # under bf16 reduction-order noise -- producing per-token O(absmax) errors + # that look like sharding bugs but are really finite-precision artifacts. + # Override the router weights / biases (and Qwen3.5-MoE's bias-less + # router's forward) so top-k always picks experts ``[0..top_k-1]`` + # regardless of input. Routing decisions then become rock-stable across + # ranks and precisions; only true sharding bugs cause output drift. + n_routers_fixed = fix_moe_routers_deterministic(model) + if rank == 0 and n_routers_fixed > 0: + print( + f"[sharding-ir-eq] fixed {n_routers_fixed} MoE router(s) to deterministic top-k", + flush=True, + ) + + # ------------------------------------------------------------------ + # 2. Build random prefill inputs and snapshot the unsharded weights. + # ------------------------------------------------------------------ + vocab_size = int(model.config.vocab_size) + input_ids, position_ids = build_random_prefill_inputs( + BATCH_SIZE, SEQ_LEN, vocab_size, device, seed=INPUT_SEED + ) + sd_snapshot = {k: v.detach().clone() for k, v in model.state_dict().items()} + + # ------------------------------------------------------------------ + # 3. Export the model on each side with the example inputs that side + # will actually receive. ``torch_export_to_gm`` bakes the batch + # dimension as static, so under attention-DP the sharded graph must + # be exported with the per-rank batch slab, not the full batch. + # Both sides still go through the same export step, so any + # torch_export_to_gm semantic gap is identical on both sides and + # cancels out of the comparison. + # + # We pass the example inputs as ``kwargs`` (not positional ``args``) + # to mirror the AutoDeploy runtime path -- the production export + # transform (``transform/library/export_to_gm.py:ExportToGM``) also + # calls ``torch_export_to_gm(..., args=(), kwargs=captured_kwargs)``. + # Binding by name makes the test invariant to ``forward()`` parameter + # ordering across IR modeling files, which currently differs between + # qwen3 (``(input_ids, position_ids, inputs_embeds, ...)``) and + # nemotron_h / qwen3_5_moe (``(input_ids, inputs_embeds, + # position_ids, ...)``). Positional export would silently bind + # ``position_ids`` to whatever lives in slot 2, which trips the + # "specify exactly one of input_ids or inputs_embeds" guard on the + # latter family. The runtime path dodges this by being kwarg-based + # and by stripping ``**kwargs`` via ``set_exact_signature``; this + # test dodges it by just being kwarg-based. + # ------------------------------------------------------------------ + dist_cfg_spec = _DIST_CONFIGS[dist_config_name] + enable_attention_dp = dist_cfg_spec["enable_attention_dp"] + + gm_unsharded = torch_export_to_gm( + model, + args=(), + kwargs={"input_ids": input_ids, "position_ids": position_ids}, + clone=True, + ) + if enable_attention_dp: + assert BATCH_SIZE % world_size == 0, ( + f"BATCH_SIZE={BATCH_SIZE} must be divisible by world_size={world_size} " + f"under attention-DP." + ) + chunk = BATCH_SIZE // world_size + local_in_for_export = input_ids[rank * chunk : (rank + 1) * chunk] + local_pos_for_export = position_ids[rank * chunk : (rank + 1) * chunk] + else: + local_in_for_export, local_pos_for_export = input_ids, position_ids + gm_sharded = torch_export_to_gm( + model, + args=(), + kwargs={"input_ids": local_in_for_export, "position_ids": local_pos_for_export}, + clone=True, + ) + + # ------------------------------------------------------------------ + # 4. Apply the sharding transforms only to gm_sharded. + # ------------------------------------------------------------------ + dist_config = DistConfig( + world_size=world_size, + rank=rank, + tp_size=world_size, + moe_tp_size=dist_cfg_spec["moe_tp_size"], + moe_ep_size=dist_cfg_spec["moe_ep_size"], + enable_attention_dp=enable_attention_dp, + ) + sharded_transforms = { + "apply_sharding_hints": {"stage": "sharding", "enabled": True}, + "strip_sharding_hints": {"stage": "weight_load"}, + } + optimizer = InferenceOptimizer(factory=None, config=sharded_transforms, dist_config=dist_config) + gm_sharded = optimizer(None, gm_sharded) + + if os.environ.get("SHARDING_IR_SABOTAGE") == "1": + n_removed = _sabotage_remove_collectives(gm_sharded) + if rank == 0: + print( + f"[sharding-ir-eq] SHARDING_IR_SABOTAGE=1: removed {n_removed} " + f"collective op(s) from gm_sharded", + flush=True, + ) + + # ------------------------------------------------------------------ + # 5. Load the same unsharded snapshot into both graphs. + # For gm_unsharded the load is identity-shaped. For gm_sharded the + # hooks registered by apply_sharding_hints fire here, slicing each + # parameter to the per-rank shard before assignment. + # ------------------------------------------------------------------ + gm_unsharded.load_state_dict(sd_snapshot, strict=False) + missing, _ = gm_sharded.load_state_dict(sd_snapshot, strict=False) + # ``unexpected`` keys are legitimate under EP sharding -- MoE expert + # partitioning removes per-rank expert params, so the full unsharded + # snapshot's keys for experts not held by this rank are expected to be + # rejected. ``missing`` is the bug signal: any param in the sharded + # graph that the snapshot can't supply. + assert not missing, f"Missing keys when loading sharded state_dict: {missing[:5]}" + + # ------------------------------------------------------------------ + # 6. Forward both graphs and compare via relative RMSE. + # + # Two flavors of the comparison, selected by ``enable_attention_dp``: + # + # - **TP / EP (enable_attention_dp=False):** every rank sees the full + # batch on both sides. The all-reduce inserted by sharding gives every + # rank the same full output, so the comparison is local per rank. + # + # - **Attention-DP (enable_attention_dp=True):** the unsharded reference + # still runs the full batch on every rank, but the sharded forward + # expects each rank to process only its slice of the batch -- attention + # and MLP are per-rank, and MoEAllToAll handles the dispatch+combine + # inside the MoE block. We scatter the batch into contiguous slabs, + # run the sharded forward locally, then all-gather the per-rank slabs + # and concatenate into the full-batch order to compare against the + # replicated unsharded reference. + # ------------------------------------------------------------------ + with torch.inference_mode(): + # Call the exported GMs with the same kwarg convention used at + # export time (see step 3 above). The GM's traced forward signature + # mirrors the export call, and calling by name keeps it that way -- + # we never reintroduce a positional dependency that would tie this + # test back to ``forward()`` parameter ordering. + y_unsharded = extract_logits(gm_unsharded(input_ids=input_ids, position_ids=position_ids)) + y_local = extract_logits( + gm_sharded(input_ids=local_in_for_export, position_ids=local_pos_for_export) + ) + if enable_attention_dp: + y_sharded = _all_gather_concat(y_local.contiguous(), world_size) + else: + y_sharded = y_local + + rel_rmse_tol = float(os.environ.get("SHARDING_IR_REL_RMSE_TOL", str(REL_RMSE_TOL))) + rel_rmse = ( + torch.sqrt(((y_sharded - y_unsharded).float() ** 2).mean()) + / torch.sqrt((y_unsharded.float() ** 2).mean()) + ).item() + if rank == 0: + u = y_unsharded.float() + diff = (y_sharded.float() - u).abs() + print( + f"[sharding-ir-eq] |y_s - y_u|: max={diff.max().item():.6f} " + f"mean={diff.mean().item():.6f} rel_rmse={rel_rmse:.6f} " + f"(tol={rel_rmse_tol})", + flush=True, + ) + assert rel_rmse < rel_rmse_tol, f"rel_rmse={rel_rmse:.4f} >= {rel_rmse_tol} on rank {rank}" + + +def _run_equivalence_job( + modeling_file: str, + dist_config_name: str, + rank: int, + world_size: int, +) -> None: + """Worker entry; mirrors the per-rank traceback to a log file on failure. + + ``spawn_multiprocess_job`` reports a coarse ``"process exited with code N"`` + when a worker raises; the side-channel log makes the underlying exception + recoverable from the parent process. + """ + import traceback + + try: + _run_equivalence_job_impl(modeling_file, rank, world_size, dist_config_name) + except BaseException: + with open(f"/tmp/sharding_ir_equiv_rank{rank}.log", "w") as f: + f.write(traceback.format_exc()) + raise + + +def _gpu_check(dist_config_name: str) -> Optional[str]: + """Return a skip-reason string if there aren't enough GPUs, else ``None``.""" + need = _DIST_CONFIGS[dist_config_name]["world_size"] + have = torch.cuda.device_count() + if have < need: + return f"requires {need} GPUs for dist_config={dist_config_name!r} (got {have})" + return None + + +def test_sharding_ir_equivalence( + sharding_ir_modeling_file: str, + sharding_ir_dist_config: str, +) -> None: + """Verify sharded == unsharded prefill for the supplied modeling file.""" + skip = _gpu_check(sharding_ir_dist_config) + if skip: + pytest.skip(skip) + + import tensorrt_llm._torch.auto_deploy.distributed.common as dist_common + + world_size = _DIST_CONFIGS[sharding_ir_dist_config]["world_size"] + dist_common.spawn_multiprocess_job( + job=partial(_run_equivalence_job, sharding_ir_modeling_file, sharding_ir_dist_config), + size=world_size, + ) From 5c896cd13b3c5f4f3e76470c41277bc72220ab50 Mon Sep 17 00:00:00 2001 From: NVShreyas <158103197+NVShreyas@users.noreply.github.com> Date: Thu, 28 May 2026 10:47:15 -0500 Subject: [PATCH 091/308] [TRTLLM-11410][feat] MoT World Model Support (#14012) Signed-off-by: Shreyas Misra Signed-off-by: Shreyas Mista --- .../visual_gen/attention_backend/vanilla.py | 9 +- .../_torch/visual_gen/models/__init__.py | 2 + .../visual_gen/models/cosmos3/__init__.py | 18 + .../visual_gen/models/cosmos3/defaults.py | 59 + .../visual_gen/models/cosmos3/guardrails.py | 67 + .../models/cosmos3/pipeline_cosmos3.py | 625 +++++++++ .../models/cosmos3/transformer_cosmos3.py | 1159 +++++++++++++++++ .../models/ltx2/transformer_ltx2.py | 1 + .../visual_gen/models/wan/transformer_wan.py | 1 + .../_torch/visual_gen/modules/attention.py | 3 +- .../_torch/visual_gen/pipeline_registry.py | 4 + 11 files changed, 1946 insertions(+), 2 deletions(-) create mode 100644 tensorrt_llm/_torch/visual_gen/models/cosmos3/__init__.py create mode 100644 tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py create mode 100644 tensorrt_llm/_torch/visual_gen/models/cosmos3/guardrails.py create mode 100644 tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py create mode 100644 tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py b/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py index bcd16893638a..ea6c3947fc78 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py @@ -99,7 +99,14 @@ def forward( f"Invalid v shape: expected [B={q.shape[0]}, H_kv, S_kv, D={self.head_dim}], got {v.shape}" ) - return F.scaled_dot_product_attention(q, k, v, is_causal=is_causal, scale=self.scale) + return F.scaled_dot_product_attention( + q, + k, + v, + is_causal=is_causal, + scale=self.scale, + enable_gqa=self.num_heads != self.num_kv_heads, + ) @property def preferred_layout(self) -> AttentionTensorLayout: diff --git a/tensorrt_llm/_torch/visual_gen/models/__init__.py b/tensorrt_llm/_torch/visual_gen/models/__init__.py index 235b5a65f3d5..4b425d2e7d89 100644 --- a/tensorrt_llm/_torch/visual_gen/models/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/models/__init__.py @@ -19,6 +19,7 @@ from ..pipeline import BasePipeline from ..pipeline_registry import AutoPipeline, register_pipeline +from .cosmos3 import Cosmos3OmniMoTPipeline from .flux import Flux2Pipeline, FluxPipeline from .ltx2 import LTX2Pipeline # noqa: F401 from .wan import WanImageToVideoPipeline, WanPipeline @@ -30,5 +31,6 @@ "Flux2Pipeline", "WanPipeline", "WanImageToVideoPipeline", + "Cosmos3OmniMoTPipeline", "register_pipeline", ] diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/__init__.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/__init__.py new file mode 100644 index 000000000000..98f83cee9b08 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .pipeline_cosmos3 import Cosmos3OmniMoTPipeline + +__all__ = ["Cosmos3OmniMoTPipeline"] diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py new file mode 100644 index 000000000000..e54e818356c4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Per-model default generation parameters for Cosmos3 pipelines. + +Shared by the Cosmos3 OmniMoT text-to-video and image-to-video generation paths. +""" + +from typing import Dict + +from tensorrt_llm._torch.visual_gen.pipeline import ExtraParamSchema + +# --------------------------------------------------------------------------- +# Constant tables +# --------------------------------------------------------------------------- + +COSMOS3_720P_PARAMS = { + "height": 720, + "width": 1280, + "num_inference_steps": 35, + "guidance_scale": 6.0, + "max_sequence_length": 1024, + "num_frames": 189, + "frame_rate": 24.0, +} + +COSMOS3_EXTRA_SPECS: Dict[str, ExtraParamSchema] = { + "use_duration_template": ExtraParamSchema( + type="bool", + default=True, + description="Whether to use the duration template.", + ), + "use_resolution_template": ExtraParamSchema( + type="bool", + default=True, + description="Whether to use the resolution template.", + ), + "use_system_prompt": ExtraParamSchema( + type="bool", + default=False, + description="Whether to use the system prompt.", + ), + "use_guardrails": ExtraParamSchema( + type="bool", + default=True, + description="Whether to use the guardrails.", + ), +} diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/guardrails.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/guardrails.py new file mode 100644 index 000000000000..881d830450fa --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/guardrails.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +import torch + +from tensorrt_llm.logger import logger + +GUARDRAIL_HF_REPO = "nvidia/Cosmos-1.0-Guardrail" +GUARDRAIL_REVISION = "cf03c0395fac8c4de386c0bdab12cc4fc8d66362" + + +def download_guardrail_checkpoint() -> str: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import GatedRepoError + + try: + return snapshot_download( + GUARDRAIL_HF_REPO, + revision=GUARDRAIL_REVISION, + local_files_only=True, + ) + except FileNotFoundError: + logger.warning(f"Guardrail checkpoint not found, downloading from {GUARDRAIL_HF_REPO}") + try: + return snapshot_download( + GUARDRAIL_HF_REPO, + revision=GUARDRAIL_REVISION, + ) + except GatedRepoError: + raise ValueError( + "Cosmos Guardrail checkpoint not found. " + "Please ensure " + "a) you have accepted the terms of use (https://huggingface.co/nvidia/Cosmos-1.0-Guardrail) " + "b) you have set a valid HF_TOKEN environment variable" + ) + + +def check_video_safety(video_tensor: torch.Tensor, safety_checker: Any) -> torch.Tensor | None: + v = video_tensor.detach().cpu() + was_batched = v.dim() == 5 + if was_batched: + v = v[0] + frames_np = v.numpy() + frames_np = safety_checker.check_video_safety(frames_np) + if frames_np is None: + return None + + result = torch.from_numpy(frames_np) + if was_batched: + result = result.unsqueeze(0) + return result.to(video_tensor.device) diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py new file mode 100644 index 000000000000..800006216bf2 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py @@ -0,0 +1,625 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os +import time +from typing import List, Optional, Union + +import PIL.Image +import torch +from diffusers import AutoencoderKLWan, UniPCMultistepScheduler +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from transformers import Qwen2Tokenizer + +from tensorrt_llm._torch.visual_gen.output import CudaPhaseTimer, PipelineOutput +from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline +from tensorrt_llm._torch.visual_gen.pipeline_registry import PipelineComponent, register_pipeline +from tensorrt_llm._torch.visual_gen.utils import postprocess_video_tensor +from tensorrt_llm._utils import nvtx_range +from tensorrt_llm.logger import logger + +from .defaults import COSMOS3_720P_PARAMS, COSMOS3_EXTRA_SPECS +from .guardrails import check_video_safety, download_guardrail_checkpoint +from .transformer_cosmos3 import Cosmos3VFMTransformer + +COSMOS3_DEFAULT_NEGATIVE_PROMPT = ( + "The video captures a series of frames showing ugly scenes, static with no motion, motion blur, " + "over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, " + "underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, jerky movements, " + "low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, fake elements, " + "unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. Overall, the video is of " + "poor quality." +) +COSMOS3_DEFAULT_SYSTEM_PROMPT = ( + "You are a helpful assistant who will generate videos from a given prompt." +) +COSMOS3_DURATION_TEMPLATE = "The video is {duration:.1f} seconds long and is of {fps:.0f} FPS." +COSMOS3_DEFAULT_RESOLUTION_TEMPLATE = "This video is of {height}x{width} resolution." +TRTLLM_DISABLE_COSMOS3_GUARDRAILS = os.environ.get("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", "0") == "1" + + +# TODO: add hf_ids +@register_pipeline("Cosmos3OmniMoTPipeline") +class Cosmos3OmniMoTPipeline(BasePipeline): + def __init__(self, model_config): + super().__init__(model_config) + + def _init_transformer(self) -> None: + logger.info("Initializing Cosmos3VFMTransformer") + self.transformer = Cosmos3VFMTransformer(self.model_config) + + def load_weights(self, weights: dict) -> None: + if self.transformer is not None and hasattr(self.transformer, "load_weights"): + transformer_weights = weights.get("transformer", weights) + self.transformer.load_weights(transformer_weights) + self.transformer.eval() + + def load_standard_components( + self, checkpoint_dir: str, device: torch.device, skip_components: Optional[list] = [] + ) -> None: + skip_components = skip_components or [] + + if PipelineComponent.TOKENIZER not in skip_components: + logger.info("Loading tokenizer...") + self.tokenizer = Qwen2Tokenizer.from_pretrained( + checkpoint_dir, + subfolder="text_tokenizer", + ) + + # Cosmos3 canonical defaults — overwritten if VAE is loaded + self.vae_scale_factor_temporal = 4 + self.vae_scale_factor_spatial = 16 + + if PipelineComponent.VAE not in skip_components: + logger.info("Loading VAE...") + self.vae = AutoencoderKLWan.from_pretrained( + checkpoint_dir, + subfolder=PipelineComponent.VAE, + torch_dtype=torch.bfloat16, # load VAE in BF16 for memory saving + ).to(device) + + self.vae_scale_factor_temporal = getattr( + self.vae.config, "scale_factor_temporal", self.vae_scale_factor_temporal + ) + self.vae_scale_factor_spatial = getattr( + self.vae.config, "scale_factor_spatial", self.vae_scale_factor_spatial + ) + self.transformer.temporal_compression_factor = self.vae_scale_factor_temporal + + if PipelineComponent.SCHEDULER not in skip_components: + logger.info("Loading scheduler...") + self.scheduler = UniPCMultistepScheduler.from_pretrained( + checkpoint_dir, + subfolder=PipelineComponent.SCHEDULER, + ) + + if not TRTLLM_DISABLE_COSMOS3_GUARDRAILS: + # lazy import + try: + from cosmos_guardrail import CosmosSafetyChecker + except (ImportError, ModuleNotFoundError): + raise ValueError( + "Cosmos Guardrail is not installed. This is in violation of the " + "[NVIDIA Open Model License Agreement]" + "(https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license). " + "Please run the following installation commands or " + "explicitly disable guardrails by setting TRTLLM_DISABLE_COSMOS3_GUARDRAILS=1 " + "(user is responsible for deploying the model without guardrails). " + "- `pip install cosmos_guardrail==0.3.0 && pip uninstall opencv-python`" + ) + # Guardrails are only evaluated on rank 0; load them only there to avoid + # dead model weights occupying GPU memory on every other rank. + if self.rank == 0: + # the download guardrail checkpoint will bypass CosmosSafetyChecker's checkpoint download. + # Both will use HF_HOME as the cache directory. + download_guardrail_checkpoint() + self.safety_checker = CosmosSafetyChecker() + self.safety_checker.to(device) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + @property + def default_warmup_resolutions(self): + return [(720, 1280)] + + @property + def default_warmup_num_frames(self): + return [189] + + @property + def default_generation_params(self): + return dict(COSMOS3_720P_PARAMS) + + @property + def extra_param_specs(self): + return dict(COSMOS3_EXTRA_SPECS) + + def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: + with torch.no_grad(): + self.forward( + prompt="warmup", + negative_prompt="", + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=steps, + guidance_scale=COSMOS3_720P_PARAMS["guidance_scale"], + seed=42, + max_sequence_length=COSMOS3_720P_PARAMS["max_sequence_length"], + use_guardrails=False, + image=None, + ) + + def infer(self, req): + return self.forward( + prompt=req.prompt, + negative_prompt=req.params.negative_prompt, + image=req.params.image, + height=req.params.height, + width=req.params.width, + num_frames=req.params.num_frames, + num_inference_steps=req.params.num_inference_steps, + guidance_scale=req.params.guidance_scale, + seed=req.params.seed, + max_sequence_length=req.params.max_sequence_length, + frame_rate=req.params.frame_rate, + use_duration_template=req.params.extra_params.get("use_duration_template", True), + use_resolution_template=req.params.extra_params.get("use_resolution_template", True), + use_system_prompt=req.params.extra_params.get("use_system_prompt", False), + use_guardrails=req.params.extra_params.get("use_guardrails", True), + ) + + def _format_prompt_with_template( + self, + prompt: str, + *, + height: int, + width: int, + num_frames: int, + frame_rate: float, + use_duration_template: bool = True, + use_resolution_template: bool = True, + ) -> str: + prompt = prompt.strip() + + if use_duration_template and num_frames > 1: + duration = num_frames / frame_rate + dur_text = COSMOS3_DURATION_TEMPLATE.format(duration=duration, fps=frame_rate) + prompt = prompt.rstrip(".") + ". " + dur_text + + prompt = prompt.strip() + if use_resolution_template: + res_text = COSMOS3_DEFAULT_RESOLUTION_TEMPLATE.format(height=height, width=width) + prompt = prompt.rstrip(".") + ". " + res_text + + return prompt + + def _resize_and_center_crop_image( + self, image: PIL.Image.Image, height: int, width: int + ) -> PIL.Image.Image: + """Match Cosmos3 reference preprocessing for conditioning images.""" + orig_w, orig_h = image.size + scaling_ratio = max(width / orig_w, height / orig_h) + resize_w = int(math.ceil(scaling_ratio * orig_w)) + resize_h = int(math.ceil(scaling_ratio * orig_h)) + + image = image.resize((resize_w, resize_h), PIL.Image.Resampling.LANCZOS) + + left = max((resize_w - width) // 2, 0) + top = max((resize_h - height) // 2, 0) + return image.crop((left, top, left + width, top + height)) + + @nvtx_range("_tokenize_prompt", color="blue") + def _tokenize_prompt( + self, text: str, max_sequence_length: int, use_system_prompt: bool = False + ): + """Tokenize a prompt using the Qwen2 chat template. + + Returns (input_ids, attention_mask) as [1, S] tensors on device. + """ + conversations = ( + [{"role": "system", "content": COSMOS3_DEFAULT_SYSTEM_PROMPT}] + if use_system_prompt + else [] + ) + conversations.append( + {"role": "user", "content": text}, + ) + token_ids = self.tokenizer.apply_chat_template( + conversations, + tokenize=True, + add_generation_prompt=True, + return_dict=False, + ) + reserved_tokens = 2 + if max_sequence_length < reserved_tokens: + raise ValueError( + f"max_sequence_length must be at least {reserved_tokens}, got {max_sequence_length}" + ) + token_ids = token_ids[: max_sequence_length - reserved_tokens] + token_ids.append(self.tokenizer.eos_token_id) # 151645 + token_ids.append(self.tokenizer.convert_tokens_to_ids("<|vision_start|>")) # 151652 + seq_len = len(token_ids) + + # Pad to max_sequence_length + pad_len = max_sequence_length - seq_len + attention_mask = [1] * seq_len + [0] * pad_len + token_ids = token_ids + [self.tokenizer.pad_token_id or 0] * pad_len + + input_ids = torch.tensor([token_ids], dtype=torch.long, device=self.device) + attention_mask = torch.tensor([attention_mask], dtype=torch.long, device=self.device) + return input_ids, attention_mask + + # ========================================================================= + # Latent preparation + # ========================================================================= + + @nvtx_range("_prepare_latents", color="blue") + def _prepare_latents(self, height, width, num_frames, generator): + num_channels_latents = self.transformer.latent_channel_size + num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + shape = ( + 1, + num_channels_latents, + num_latent_frames, + height // self.vae_scale_factor_spatial, + width // self.vae_scale_factor_spatial, + ) + return randn_tensor(shape, generator=generator, device=self.device, dtype=self.dtype) + + # -- I2V latent preparation ----------------------------------------------- + + def _encode_conditioning_video( + self, + image_tensor: torch.Tensor, + num_frames: int, + height: int, + width: int, + ) -> torch.Tensor: + """VAE-encode a conditioning image as a full-length video. + + The WAN VAE has temporal compression (factor 4), so encoding a single + frame produces degenerate temporal features. Following imaginaire4's + ``build_conditioned_video_batch``, we fill the entire pixel-space video + with the conditioning image (repeating it across all frames) so the + temporal encoder sees plausible content everywhere. The caller then + keeps only the conditioned latent frame(s) and replaces the rest with + noise. + + Args: + image_tensor: [1, 3, H, W] in [-1, 1] + num_frames: total pixel frames for the video + height: pixel height + width: pixel width + + Returns: + [1, C, T_latent, H_latent, W_latent] normalized latent of the + full conditioning video. + """ + # Build pixel-space video: repeat the conditioning image across all frames + # image_tensor: [1, 3, H, W] -> [1, 3, 1, H, W] -> [1, 3, num_frames, H, W] + video = image_tensor.unsqueeze(2).expand(-1, -1, num_frames, -1, -1).contiguous() + video = video.to(device=self.device, dtype=self.vae.dtype) + + latent = self.vae.encode(video).latent_dist.mode() + + # Normalize (inverse of _decode_latents denormalization) + if hasattr(self.vae.config, "latents_mean") and hasattr(self.vae.config, "latents_std"): + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, -1, 1, 1, 1) + .to(latent.device, latent.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, -1, 1, 1, 1) + .to(latent.device, latent.dtype) + ) + latent = (latent - latents_mean) / latents_std + else: + scaling_factor = getattr(self.vae.config, "scaling_factor", 1.0) + latent = latent * scaling_factor + + return latent.to(self.dtype) + + def _prepare_latents_i2v( + self, + image_tensor: torch.Tensor, + height: int, + width: int, + num_frames: int, + generator: torch.Generator, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepare initial latents with frame 0 conditioned on the input image. + + The conditioning image is repeated across all pixel frames before VAE + encoding so the temporal encoder sees plausible content everywhere + (avoids degenerate single-frame encoding with the WAN VAE's temporal + compression). Only frame 0 of the resulting latent is kept clean; + the rest is replaced with noise. + + Returns: + latents: [1, C, T_lat, H_lat, W_lat] with frame 0 = image, rest = noise + velocity_mask: [1, 1, T_lat, 1, 1] with frame 0 = 0, rest = 1 + image_latent: [1, C, 1, H_lat, W_lat] clean frame 0 for re-injection + """ + C = self.transformer.latent_channel_size + T_lat = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + + # Pure noise + noise = randn_tensor( + ( + 1, + C, + T_lat, + height // self.vae_scale_factor_spatial, + width // self.vae_scale_factor_spatial, + ), + generator=generator, + device=self.device, + dtype=self.dtype, + ) + + # Encode full conditioning video (image repeated across all frames) + cond_latent = self._encode_conditioning_video( + image_tensor, + num_frames, + height, + width, + ) # [1, C, T_lat, H_lat, W_lat] + + # Keep only frame 0 for conditioning; replace rest with noise + image_latent = cond_latent[:, :, 0:1, :, :] # [1, C, 1, H_lat, W_lat] + + condition_mask = torch.zeros(1, 1, T_lat, 1, 1, device=self.device, dtype=self.dtype) + condition_mask[:, :, 0, :, :] = 1.0 + + latents = condition_mask * cond_latent + (1.0 - condition_mask) * noise + + velocity_mask = 1.0 - condition_mask + return latents, velocity_mask, image_latent + + # ========================================================================= + # VAE decode + # ========================================================================= + + @nvtx_range("_decode_latents", color="blue") + def _decode_latents(self, latents): + latents = latents.to(self.vae.dtype) + + if hasattr(self.vae.config, "latents_mean") and hasattr(self.vae.config, "latents_std"): + if not hasattr(self, "_latents_mean"): + self._latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, -1, 1, 1, 1) + .to(self.device, self.vae.dtype) + ) + self._latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, -1, 1, 1, 1) + .to(self.device, self.vae.dtype) + ) + latents = (latents * self._latents_std) + self._latents_mean + else: + scaling_factor = self.vae.config.get("scaling_factor", 1.0) + latents = latents / scaling_factor + + video = self.vae.decode(latents, return_dict=False)[0] + video = postprocess_video_tensor(video) + return video + + # ========================================================================= + # Forward (main generation entry point) + # ========================================================================= + + @nvtx_range("Cosmos3OmniMoTPipeline.forward") + @torch.inference_mode() + def forward( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[str] = None, + image: Optional[Union[PIL.Image.Image, torch.Tensor, str]] = None, + height: int = COSMOS3_720P_PARAMS["height"], + width: int = COSMOS3_720P_PARAMS["width"], + num_frames: int = COSMOS3_720P_PARAMS["num_frames"], + num_inference_steps: int = COSMOS3_720P_PARAMS["num_inference_steps"], + guidance_scale: float = COSMOS3_720P_PARAMS["guidance_scale"], + seed: int = 42, + max_sequence_length: int = COSMOS3_720P_PARAMS["max_sequence_length"], + frame_rate: float = COSMOS3_720P_PARAMS["frame_rate"], + use_duration_template: bool = COSMOS3_EXTRA_SPECS["use_duration_template"].default, + use_resolution_template: bool = COSMOS3_EXTRA_SPECS["use_resolution_template"].default, + use_system_prompt: bool = COSMOS3_EXTRA_SPECS["use_system_prompt"].default, + use_guardrails: bool = COSMOS3_EXTRA_SPECS["use_guardrails"].default, + ): + pipeline_start = time.time() + timer = CudaPhaseTimer() + timer.mark_pre_start() + + use_guardrails = use_guardrails and not TRTLLM_DISABLE_COSMOS3_GUARDRAILS + + if isinstance(prompt, str): + prompt = [prompt] + batch_size = len(prompt) + + if batch_size > 1: + # TODO: support batch generation + raise ValueError("Batch generation is not supported for Cosmos3") + + # Validate image input — only single image is supported for batch generation + if image is not None and not isinstance(image, (PIL.Image.Image, torch.Tensor, str)): + raise ValueError( + f"`image` must be a PIL.Image, torch.Tensor, or file path string, " + f"got {type(image)}. Batch of different images is not supported; " + f"use a single image with multiple prompts instead." + ) + + # Text guardrail — check both positive and user-supplied negative prompts. + # None negative_prompt means the hardcoded default will be used (safe); skip it. + text_blocked = torch.zeros((), device=self.device, dtype=torch.int32) + if self.rank == 0 and use_guardrails and self.safety_checker is not None: + prompts_to_check = list(prompt) + if negative_prompt is not None: + prompts_to_check.append(negative_prompt) + for p in prompts_to_check: + is_safe = self.safety_checker.check_text_safety(p) + if not is_safe: + logger.warning("Text guardrail blocked prompt") + text_blocked.fill_(1) + break + + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.broadcast(text_blocked, src=0) + + if text_blocked.item(): + timer.mark_end() + return timer.fill(PipelineOutput()) + + generator = torch.Generator(device=self.device).manual_seed(seed) + + if negative_prompt is None: + negative_prompt = COSMOS3_DEFAULT_NEGATIVE_PROMPT + + negative_prompt = self._format_prompt_with_template( + negative_prompt, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + use_duration_template=use_duration_template, + use_resolution_template=use_resolution_template, + ) + + prompt = [ + self._format_prompt_with_template( + p, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + use_duration_template=use_duration_template, + use_resolution_template=use_resolution_template, + ) + for p in prompt + ] + logger.info(f"Prompt with metadata: '{prompt}'") + + prompt = prompt[0] + + # 1. Tokenize prompts (no separate text encoder — transformer embeds internally) + logger.info("Tokenizing prompts...") + cond_ids, cond_mask = self._tokenize_prompt(prompt, max_sequence_length, use_system_prompt) + uncond_ids, uncond_mask = self._tokenize_prompt( + negative_prompt, max_sequence_length, use_system_prompt + ) + + # 2. Prepare latents + if image is not None: + if isinstance(image, str): + image = PIL.Image.open(image).convert("RGB") + + if isinstance(image, PIL.Image.Image): + image = image.convert("RGB") + image = self._resize_and_center_crop_image(image, height=height, width=width) + image = self.video_processor.preprocess( + image, + height=height, + width=width, + ) + + latents, velocity_mask, image_latent = self._prepare_latents_i2v( + image, height=height, width=width, num_frames=num_frames, generator=generator + ) + else: + latents = self._prepare_latents(height, width, num_frames, generator) + velocity_mask = None + image_latent = None + + # Compute video shape in latent space + T_latent = latents.shape[2] + H_latent = latents.shape[3] + W_latent = latents.shape[4] + video_shape = (T_latent, H_latent, W_latent) + + # 3. Set up scheduler + self.scheduler.set_timesteps(num_inference_steps, device=self.device) + + # 4. Build forward_fn for the denoise loop + def forward_fn( + latent_input, extra_stream_latents, timestep, encoder_hidden_states, extra_tensors + ): + """Cosmos3 forward function for BasePipeline.denoise(). + + Since Cosmos3 embeds text internally, we pass token IDs via extra_tensors + rather than through encoder_hidden_states. + """ + noise_pred = self.transformer( + hidden_states=latent_input, + timestep=timestep, + text_ids=extra_tensors["text_ids"], + text_mask=extra_tensors["text_mask"], + video_shape=video_shape, + fps=frame_rate, + noisy_frame_mask=velocity_mask, + ) + if velocity_mask is not None: + noise_pred = noise_pred * velocity_mask + return noise_pred + + # 5. Build CFG tensors — text_ids and text_mask need to be split for CFG + # BasePipeline.denoise batches [uncond, cond] when guidance_scale > 1 + # We pass text IDs/masks through extra_cfg_tensors so they get split correctly + extra_cfg_tensors = { + "text_ids": (cond_ids, uncond_ids), + "text_mask": (cond_mask, uncond_mask), + } + + self.transformer.reset_cache() + + # 6. Denoise + timer.mark_denoise_start() + latents = self.denoise( + latents=latents, + scheduler=self.scheduler, + prompt_embeds=cond_ids, # placeholder — actual conditioning via extra_cfg_tensors + neg_prompt_embeds=uncond_ids, + guidance_scale=guidance_scale, + forward_fn=forward_fn, + extra_cfg_tensors=extra_cfg_tensors, + ) + timer.mark_post_start() + + # 7. Decode + logger.info("Decoding video...") + decode_start = time.time() + + if image_latent is not None: + latents = latents.clone() + latents[:, :, 0:1, :, :] = image_latent.to(device=latents.device, dtype=latents.dtype) + + video = self.decode_latents(latents, self._decode_latents) + + # Video guardrails + if self.rank == 0: + logger.info(f"Video decoded in {time.time() - decode_start:.2f}s") + logger.info(f"Total pipeline time: {time.time() - pipeline_start:.2f}s") + + if use_guardrails and self.safety_checker is not None: + video = check_video_safety(video, self.safety_checker) + + timer.mark_end() + return timer.fill(PipelineOutput(video=video, frame_rate=frame_rate)) diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py new file mode 100644 index 000000000000..d260b8e10ea1 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py @@ -0,0 +1,1159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from diffusers.models.embeddings import TimestepEmbedding + +from tensorrt_llm._torch.attention_backend.interface import PredefinedAttentionMask +from tensorrt_llm._torch.modules.embedding import Embedding +from tensorrt_llm._torch.modules.gated_mlp import GatedMLP +from tensorrt_llm._torch.modules.linear import Linear +from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig +from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode +from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader +from tensorrt_llm.logger import logger +from tensorrt_llm.models.modeling_utils import QuantConfig + + +class Qwen3VLTextRMSNorm(nn.Module): + def __init__( + self, hidden_size: int, eps: float = 1e-6, dtype: torch.dtype = torch.bfloat16 + ) -> None: + """ + Qwen3VLTextRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.dtype = dtype + + def post_load_weights(self): + self.weight.data = self.weight.data.to(self.dtype) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + output = self.weight * hidden_states.to(input_dtype) + return output + + +def compute_mrope_position_ids_text( + num_tokens: int, + temporal_offset: int, +) -> tuple[torch.Tensor, int]: + """Generate 3D mRoPE position IDs for text tokens. + + Text tokens: all three axes (T, H, W) share the same monotonically + increasing position IDs: (0,0,0), (1,1,1), (2,2,2), ... + + Returns: + (position_ids [3, num_tokens], next_temporal_offset) + """ + ids = torch.arange(num_tokens, dtype=torch.long) + temporal_offset + mrope_ids = ids.unsqueeze(0).expand(3, -1).contiguous() + return mrope_ids, temporal_offset + num_tokens + + +def compute_mrope_position_ids_vision( + grid_t: int, + grid_h: int, + grid_w: int, + temporal_offset: int | float, + fps: float | None = None, + base_fps: float = 24.0, + temporal_compression_factor: int = 4, + enable_fps_modulation: bool = False, +) -> tuple[torch.Tensor, int | float]: + """Generate 3D mRoPE position IDs for vision tokens. + + Creates a (T, H, W) position grid. Spatial indices reset to 0 + per vision segment (Qwen3VL-style, reset_spatial_indices=True). + Flattened in T-major order. + + When ``enable_fps_modulation`` is ``True``, temporal positions are scaled + to reflect real time so that videos at different frame rates get comparable + temporal embeddings. + + Returns: + (position_ids [3, grid_t * grid_h * grid_w], next_temporal_offset) + """ + if enable_fps_modulation and fps is not None: + tps = fps / temporal_compression_factor + base_tps = base_fps / temporal_compression_factor + frame_indices = torch.arange(grid_t, dtype=torch.float32) + t_index = ( + (frame_indices / tps * base_tps + temporal_offset) + .view(-1, 1) + .expand(-1, grid_h * grid_w) + .flatten() + ) + else: + t_index = torch.arange(grid_t, dtype=torch.long).view(-1, 1).expand( + -1, grid_h * grid_w + ).flatten() + int(temporal_offset) + + h_index = ( + torch.arange(grid_h, dtype=torch.long).view(1, -1, 1).expand(grid_t, -1, grid_w).flatten() + ) + w_index = ( + torch.arange(grid_w, dtype=torch.long).view(1, 1, -1).expand(grid_t, grid_h, -1).flatten() + ) + + if enable_fps_modulation: + mrope_ids = torch.stack( + [t_index, h_index.to(torch.float32), w_index.to(torch.float32)], dim=0 + ) + else: + mrope_ids = torch.stack([t_index, h_index, w_index], dim=0) + + next_offset = math.ceil(mrope_ids.max().item()) + 1 + return mrope_ids, next_offset + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__( + self, + hidden_size, + frequency_embedding_size=256, + max_period=10000, + target_dtype=torch.bfloat16, + ): + super().__init__() + self.mlp = TimestepEmbedding( + in_channels=frequency_embedding_size, time_embed_dim=hidden_size, act_fn="silu" + ) + self.frequency_embedding_size = frequency_embedding_size + self.hidden_size = hidden_size + + half = frequency_embedding_size // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=target_dtype) / half + ) + self.register_buffer("freqs", freqs, persistent=False) + + def _init_weights(self): + std = 1.0 / math.sqrt(self.frequency_embedding_size) + torch.nn.init.trunc_normal_(self.mlp.linear_1.weight, std=std, a=-3 * std, b=3 * std) + + std = 1.0 / math.sqrt(self.hidden_size) + torch.nn.init.trunc_normal_(self.mlp.linear_2.weight, std=std, a=-3 * std, b=3 * std) + + def forward(self, t): + # use .float() here if acc loss + args = t[:, None] * self.freqs[None] + t_freq = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + t_emb = self.mlp(t_freq) + return t_emb + + +def qwen3_rotate_half(x: torch.Tensor) -> torch.Tensor: + """Qwen3/Llama-style rotate_half: split first/second half of head_dim.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def qwen3_apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Qwen3-style RoPE: (x * cos) + (rotate_half(x) * sin). + + Args: + q: [B, S, H, D] + k: [B, S, H_kv, D] + cos: [1, S, 1, D] or broadcastable + sin: [1, S, 1, D] or broadcastable + """ + q_embed = (q * cos) + (qwen3_rotate_half(q) * sin) + k_embed = (k * cos) + (qwen3_rotate_half(k) * sin) + return q_embed, k_embed + + +class Cosmos3CausalAttention(Attention): + """Understanding pathway: causal self-attention on text tokens. + + Inherits from Attention for projections (SEPARATE_QKV), per-head QK norms, + and backend. Overrides forward to: + - Reshape to 4D before QK norm (per-head) + - Apply Qwen3-style RoPE (rotate_half, not interleaved) + - Pass causal mask to backend + """ + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + model_config: DiffusionModelConfig, + layer_idx: int = 0, + ): + super().__init__( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + head_dim=head_dim, + qkv_mode=QKVMode.SEPARATE_QKV, + qk_norm=True, + qk_norm_mode="per_head", + bias=False, + config=model_config, + layer_idx=layer_idx, + enable_ulysses=False, + ) + self.norm_q = Qwen3VLTextRMSNorm(hidden_size=head_dim, dtype=torch.bfloat16) + self.norm_k = Qwen3VLTextRMSNorm(hidden_size=head_dim, dtype=torch.bfloat16) + + def apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Per-head RMSNorm on 4D tensors [B, S, H, D].""" + q = F.rms_norm(q, (q.shape[-1],), self.norm_q.weight, self.norm_q.variance_epsilon) + k = F.rms_norm(k, (k.shape[-1],), self.norm_k.weight, self.norm_k.variance_epsilon) + return q, k + + def forward_with_kv( + self, + hidden_states: torch.Tensor, + freqs_cos: torch.Tensor, + freqs_sin: torch.Tensor, + ) -> torch.Tensor: + batch_size, seq_len = hidden_states.shape[:2] + + q, k, v = self.get_qkv(hidden_states) + + q = q.view(batch_size, seq_len, self.num_attention_heads, self.head_dim) + k = k.view(batch_size, seq_len, self.num_key_value_heads, self.head_dim) + v = v.view(batch_size, seq_len, self.num_key_value_heads, self.head_dim) + + q, k = self.apply_qk_norm(q, k) + q, k = qwen3_apply_rotary_pos_emb(q, k, freqs_cos, freqs_sin) + + out = self._attn_impl( + q, + k, + v, + attention_mask=PredefinedAttentionMask.CAUSAL, + ) + + return self.to_out[0](out), k, v + + def forward(self): + raise NotImplementedError( + "forward method not implemented for Cosmos3CausalAttention. Use forward_with_kv instead." + ) + + +class Cosmos3CrossAttention(Attention): + """Generation pathway: full attention where visual Q attends to all K/V. + + Inherits from Attention for gen-pathway projections, per-head QK norms, + and backend. Overrides forward to: + - Accept pre-computed und K/V for concatenation + - Reshape to 4D before QK norm (per-head) + - Apply Qwen3-style RoPE + - Full (non-causal) attention with Q_gen attending to [K_und, K_gen] + """ + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + model_config: DiffusionModelConfig, + layer_idx: int = 0, + ): + original_backend = model_config.attention.backend + if model_config.attention.backend == "TRTLLM": + # TRTLLM backend is not supported for Cosmos3CrossAttention + model_config.attention.backend = "VANILLA" + + super().__init__( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + head_dim=head_dim, + qkv_mode=QKVMode.FUSE_QKV, + qk_norm=True, + qk_norm_mode="per_head", + bias=False, + config=model_config, + layer_idx=layer_idx, + enable_ulysses=True, + ) + model_config.attention.backend = original_backend + + self.norm_q = Qwen3VLTextRMSNorm(hidden_size=head_dim, dtype=torch.bfloat16) + self.norm_k = Qwen3VLTextRMSNorm(hidden_size=head_dim, dtype=torch.bfloat16) + + def apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Per-head RMSNorm on 4D tensors [B, S, H, D].""" + q = F.rms_norm(q, (q.shape[-1],), self.norm_q.weight, self.norm_q.variance_epsilon) + k = F.rms_norm(k, (k.shape[-1],), self.norm_k.weight, self.norm_k.variance_epsilon) + return q, k + + def forward( + self, + hidden_states: torch.Tensor, + k_und: torch.Tensor, + v_und: torch.Tensor, + freqs_cos: torch.Tensor, + freqs_sin: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + hidden_states: [B, S_gen, hidden_size] visual tokens + k_und: [B, S_und, H_kv, D] pre-computed und keys (post-norm, post-RoPE) + v_und: [B, S_und, H_kv, D] pre-computed und values + freqs_cos: [B, S_gen, 1, D] cosine part of RoPE + freqs_sin: [B, S_gen, 1, D] sine part of RoPE + + Returns: + [B, S_gen, hidden_size] cross-attention output + """ + batch_size, seq_len_gen = hidden_states.shape[:2] + + q, k, v = self.get_qkv(hidden_states) + + q = q.view(batch_size, seq_len_gen, self.num_attention_heads, self.head_dim) + k = k.view(batch_size, seq_len_gen, self.num_key_value_heads, self.head_dim) + v = v.view(batch_size, seq_len_gen, self.num_key_value_heads, self.head_dim) + + q, k = self.apply_qk_norm(q, k) + q, k = qwen3_apply_rotary_pos_emb(q, k, freqs_cos, freqs_sin) + + k_all = torch.cat([k_und, k], dim=1).contiguous() + v_all = torch.cat([v_und, v], dim=1).contiguous() + + out = self._attn_impl( + q, + k_all, + v_all, + attention_mask=PredefinedAttentionMask.FULL, + ) + + return self.to_out[0](out) + + +class Cosmos3UndDecoderLayer(nn.Module): + """Understanding pathway decoder layer: causal self-attention + MLP.""" + + def __init__(self, model_config: DiffusionModelConfig, layer_idx: int): + super().__init__() + self.layer_idx = layer_idx + hidden_size = model_config.pretrained_config.hidden_size + intermediate_size = model_config.pretrained_config.intermediate_size + + self.self_attn = Cosmos3CausalAttention( + hidden_size=hidden_size, + num_attention_heads=model_config.pretrained_config.num_attention_heads, + num_key_value_heads=model_config.pretrained_config.num_key_value_heads, + head_dim=model_config.pretrained_config.head_dim, + model_config=model_config, + layer_idx=layer_idx, + ) + self.input_layernorm = Qwen3VLTextRMSNorm( + hidden_size=hidden_size, + eps=model_config.pretrained_config.rms_norm_eps, + dtype=torch.bfloat16, + ) + self.post_attention_layernorm = Qwen3VLTextRMSNorm( + hidden_size=hidden_size, + eps=model_config.pretrained_config.rms_norm_eps, + dtype=torch.bfloat16, + ) + self.mlp = GatedMLP( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + bias=False, + dtype=torch.bfloat16, + config=model_config, + layer_idx=layer_idx, + ) + + def forward( + self, + hidden_states: torch.Tensor, + freqs: Tuple[torch.Tensor, torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Returns: + (hidden_states, K, V) where K/V are post-QKnorm, post-RoPE + for consumption by the GEN cross-attention. + """ + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + cos, sin = freqs + attn_out, k, v = self.self_attn.forward_with_kv(hidden_states, cos, sin) + hidden_states = residual + attn_out + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + B, S, D = hidden_states.shape + hidden_states = self.mlp(hidden_states.view(-1, D)).view(B, S, D) + hidden_states = residual + hidden_states + + return hidden_states, k, v + + +class Cosmos3GenDecoderLayer(nn.Module): + """Generation pathway decoder layer: cross-attention (to UND K/V) + MLP.""" + + def __init__(self, model_config: DiffusionModelConfig, layer_idx: int): + super().__init__() + self.layer_idx = layer_idx + hidden_size = model_config.pretrained_config.hidden_size + intermediate_size = model_config.pretrained_config.intermediate_size + + self.cross_attention = Cosmos3CrossAttention( + hidden_size=hidden_size, + num_attention_heads=model_config.pretrained_config.num_attention_heads, + num_key_value_heads=model_config.pretrained_config.num_key_value_heads, + head_dim=model_config.pretrained_config.head_dim, + model_config=model_config, + layer_idx=layer_idx, + ) + self.input_layernorm = Qwen3VLTextRMSNorm( + hidden_size=hidden_size, + eps=model_config.pretrained_config.rms_norm_eps, + dtype=torch.bfloat16, + ) + self.post_attention_layernorm = Qwen3VLTextRMSNorm( + hidden_size=hidden_size, + eps=model_config.pretrained_config.rms_norm_eps, + dtype=torch.bfloat16, + ) + self.mlp = GatedMLP( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + bias=False, + dtype=torch.bfloat16, + config=model_config, + layer_idx=layer_idx, + ) + + def forward( + self, + hidden_states: torch.Tensor, + k_und: torch.Tensor, + v_und: torch.Tensor, + freqs: Tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + cos, sin = freqs + hidden_states = self.cross_attention( + hidden_states, + k_und=k_und, + v_und=v_und, + freqs_cos=cos, + freqs_sin=sin, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + B, S, D = hidden_states.shape + hidden_states = self.mlp(hidden_states.view(-1, D)).view(B, S, D) + hidden_states = residual + hidden_states + + return hidden_states + + +def _compute_default_rope_parameters( + model_config: DiffusionModelConfig, +) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PretrainedConfig`]): + The model configuration. This function assumes that the config will provide at least the following + properties: + + * rope_theta (`float`): The base wavelength from which the inverse frequencies will be derived. + * hidden_size (`int`): The numerator when deriving a head_dim, if not provided directly. + * num_attention_heads (`int`): The denominator when deriving a head_dim, if not provided directly. + + Additionally, this function will make use of the following properties if they are found in the config: + + * head_dim (`int`, *optional*): The size of the key-value heads in the model. If None, this value will be + derived as hidden_size // num_attention_heads. + * partial_rotary_factor (`float`, *optional*): If less than 1.0, inverse frequencies will be returned for + the first fraction of the head_dim. Defaults to 1.0. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = model_config.pretrained_config.rope_theta + partial_rotary_factor = 1 + head_dim = model_config.pretrained_config.head_dim + dim = int(head_dim * partial_rotary_factor) + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + +class Qwen3VLTextRotaryEmbedding(nn.Module): + def __init__(self, model_config: DiffusionModelConfig): + super().__init__() + self.rope_type = model_config.pretrained_config.rope_scaling["rope_type"] + self.max_seq_len_cached = model_config.pretrained_config.max_position_embeddings + self.original_max_seq_len = model_config.pretrained_config.max_position_embeddings + + self.mrope_section = model_config.pretrained_config.rope_scaling["mrope_section"] + + inv_freq, self.attention_scaling = _compute_default_rope_parameters(model_config) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def apply_interleaved_mrope(self, freqs, mrope_section): + """Apply interleaved MRoPE to 3D rotary embeddings. + Reorganizes frequency layout from chunked [TTT...HHH...WWW] to + interleaved [THTHWHTHW...TT], preserving frequency continuity. + args: + x: (3, bs, seq_len, head_dim // 2) + mrope_section: (3,) + returns: + x_t: (bs, seq_len, head_dim // 2) + """ + freqs_t = freqs[0] # just overwrite the first dimension T + for dim, offset in enumerate((1, 2), start=1): # H, W + length = mrope_section[dim] * 3 + idx = slice(offset, length, 3) + freqs_t[..., idx] = freqs[dim, ..., idx] + return freqs_t + + @torch.no_grad() + def forward(self, x, position_ids): + assert self.inv_freq.dtype == torch.float32, ( + f"inv_freq must be float32, but got {self.inv_freq.dtype}" + ) + + # In contrast to other models, Qwen3VL has different position ids for the grids + # So we expand the inv_freq to shape (3, ...) + if position_ids.ndim == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + inv_freq_expanded = ( + self.inv_freq[None, None, :, None] + .float() + .expand(3, position_ids.shape[1], -1, 1) + .to(x.device) + ) + position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) + + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) + freqs = self.apply_interleaved_mrope(freqs, self.mrope_section) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Cosmos3LanguageModel(nn.Module): + """Understanding pathway: a standard causal LM that processes text tokens. + + Returns per-layer K/V tensors for the generation pathway's cross-attention. + The UND pathway is independent of the denoising step, so its K/V can be + computed once and reused across all sampling steps. + """ + + def __init__(self, model_config: DiffusionModelConfig): + super().__init__() + hidden_size = model_config.pretrained_config.hidden_size + num_hidden_layers = model_config.pretrained_config.num_hidden_layers + + self.embed_tokens = Embedding( + model_config.pretrained_config.vocab_size, + hidden_size, + dtype=torch.bfloat16, + gather_output=True, + ) + self.rotary_emb = Qwen3VLTextRotaryEmbedding(model_config) + self.layers = nn.ModuleList( + [Cosmos3UndDecoderLayer(model_config, layer_idx=i) for i in range(num_hidden_layers)] + ) + + def forward( + self, + text_ids: torch.Tensor, + text_mask: torch.Tensor, + freqs: Tuple[torch.Tensor, torch.Tensor], + ) -> list[Tuple[torch.Tensor, torch.Tensor]]: + """ + Args: + text_ids: [B, S] token IDs + text_mask: [B, S] float mask (1=real, 0=pad) + freqs: (cos, sin) each [B, S, 1, D] — precomputed UND RoPE + + Returns: + List of (K, V) per layer. K/V are [B, S, H_kv, D], post-QKnorm + and post-RoPE, ready for GEN cross-attention. + """ + hidden = self.embed_tokens(text_ids) + mask_3d = text_mask.unsqueeze(-1) # [B, S, 1] + + cached_kv: list[Tuple[torch.Tensor, torch.Tensor]] = [] + for layer in self.layers: + hidden = hidden * mask_3d + hidden, k, v = layer(hidden, freqs) + cached_kv.append((k, v)) + + return cached_kv + + +class Cosmos3VFMTransformer(nn.Module): + def __init__(self, model_config: DiffusionModelConfig): + super().__init__() + self.model_config = model_config + pretrained_config = model_config.pretrained_config + + self.hidden_size = pretrained_config.hidden_size + self.num_hidden_layers = pretrained_config.num_hidden_layers + self.latent_patch_size = pretrained_config.latent_patch_size + self.latent_channel_size = pretrained_config.latent_channel + self.patch_latent_dim = (self.latent_patch_size**2) * self.latent_channel_size + self.timestep_scale = pretrained_config.timestep_scale + self.base_fps = pretrained_config.base_fps + + # Comes from VAE. Updated after VAE is loaded. + self.temporal_compression_factor = 4 + + self.unified_3d_mrope_temporal_modality_margin = ( + pretrained_config.unified_3d_mrope_temporal_modality_margin + ) + self.num_attention_heads = pretrained_config.num_attention_heads + self.num_kv_heads = pretrained_config.num_key_value_heads + self.enable_fps_modulation = pretrained_config.enable_fps_modulation + + if pretrained_config.position_embedding_type != "unified_3d_mrope": + raise ValueError( + f"Position embedding type {pretrained_config.position_embedding_type} not supported" + ) + + vgm = model_config.visual_gen_mapping + attn2d_row_size = vgm.attn2d_row_size if vgm else 1 + attn2d_col_size = vgm.attn2d_col_size if vgm else 1 + attn2d_mesh_size = attn2d_row_size * attn2d_col_size + ulysses_size = vgm.ulysses_size if vgm else 1 + use_attn2d = attn2d_mesh_size > 1 + use_ulysses = ulysses_size > 1 + + if vgm is not None and vgm.tp_size > 1: + raise ValueError( + f"Cosmos3 does not support tensor parallelism. Got tp_size={vgm.tp_size}" + ) + + if use_ulysses and ( + self.num_attention_heads % ulysses_size != 0 or self.num_kv_heads % ulysses_size != 0 + ): + raise ValueError( + f"num_attention_heads ({self.num_attention_heads}) and " + f"num_kv_heads ({self.num_kv_heads}) must be divisible by " + f"ulysses_size ({ulysses_size})" + ) + + if use_attn2d: + # Attention2D is not compatible with Cosmos3 cross-attention: its forward() + # TODO: Re-enable once Ring/Attn2D PRs with cross-attention support have landed. + raise NotImplementedError( + "Attention2D (Ring attention) is not supported for Cosmos3. " + "Use Ulysses sequence parallelism instead." + ) + elif use_ulysses: + self.use_seq_parallel = True + self.seq_parallel_size = ulysses_size + self.seq_parallel_pg = vgm.ulysses_group + self.seq_parallel_rank = vgm.ulysses_rank + else: + self.use_seq_parallel = False + self.seq_parallel_size = 1 + self.seq_parallel_pg = None + self.seq_parallel_rank = 0 + + self.language_model = Cosmos3LanguageModel(model_config) + + self.vae2llm = nn.Linear(self.patch_latent_dim, self.hidden_size) + self.llm2vae = nn.Linear(self.hidden_size, self.patch_latent_dim) + + # try timestep embedder in float32 if acc loss + self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=torch.bfloat16) + + self.gen_layers = nn.ModuleList( + [ + Cosmos3GenDecoderLayer(model_config, layer_idx=i) + for i in range(self.num_hidden_layers) + ] + ) + + self.norm_moe_gen = Qwen3VLTextRMSNorm( + hidden_size=self.hidden_size, + eps=pretrained_config.rms_norm_eps, + ) + + self.cached_kv = None + self.cached_freqs_gen = None + + self.__post_init__() + + @property + def device(self): + return next(self.parameters()).device + + def __post_init__(self): + # TODO: move this to pipeline loader under meta init so transformers' dont need to know about it here + self.apply_quant_config_exclude_modules() + + for _, module in self.named_modules(): + if callable(getattr(module, "create_weights", None)): + module.create_weights() + + def apply_quant_config_exclude_modules(self): + quant_config = self.model_config.quant_config + if quant_config is None or quant_config.exclude_modules is None: + return + + kv_cache_quant_algo = quant_config.kv_cache_quant_algo if quant_config else None + no_quant_config = QuantConfig(kv_cache_quant_algo=kv_cache_quant_algo) + + for name, module in self.named_modules(): + if isinstance(module, Linear): + is_excluded = quant_config.is_module_excluded_from_quantization(name) + if is_excluded and getattr(module, "quant_config", None) is not None: + module.quant_config = no_quant_config + + def _pad_to_patch_size(self, H: int, W: int) -> Tuple[int, int, int, int]: + """Compute padded spatial dims aligned to patch_size. + + Returns (Hp, Wp, H_padded, W_padded) where Hp/Wp are the patch grid + dimensions and H_padded/W_padded are the padded latent dimensions. + """ + p = self.latent_patch_size + H_padded = ((H + p - 1) // p) * p + W_padded = ((W + p - 1) // p) * p + return H_padded // p, W_padded // p, H_padded, W_padded + + def patchify(self, latents: torch.Tensor, T: int, H: int, W: int) -> torch.Tensor: + """[B, C, T, H, W] -> [B, T*Hp*Wp, p*p*C], padding H/W if needed.""" + B = latents.shape[0] + p = self.latent_patch_size + C = self.latent_channel_size + Hp, Wp, H_padded, W_padded = self._pad_to_patch_size(H, W) + + if H_padded != H or W_padded != W: + latents = F.pad(latents, (0, W_padded - W, 0, H_padded - H)) + + x = latents.reshape(B, C, T, Hp, p, Wp, p) + x = x.permute(0, 2, 3, 5, 4, 6, 1) # [B, T, Hp, Wp, p, p, C] + return x.reshape(B, T * Hp * Wp, p * p * C) + + def unpatchify(self, tokens: torch.Tensor, T: int, H: int, W: int) -> torch.Tensor: + """[B, T*Hp*Wp, p*p*C] -> [B, C, T, H, W], cropping padding if needed.""" + B = tokens.shape[0] + p = self.latent_patch_size + C = self.latent_channel_size + Hp, Wp, H_padded, W_padded = self._pad_to_patch_size(H, W) + + x = tokens.reshape(B, T, Hp, Wp, p, p, C) + x = x.permute(0, 6, 1, 2, 4, 3, 5) # [B, C, T, Hp, p, Wp, p] + x = x.reshape(B, C, T, H_padded, W_padded) + + if H_padded != H or W_padded != W: + x = x[:, :, :, :H, :W] + return x + + def _compute_rope_freqs( + self, + text_mask: torch.Tensor, + T: int, + Hp: int, + Wp: int, + fps: float | None, + device: torch.device, + dtype: torch.dtype, + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor]]: + """Compute mRoPE cos/sin for UND (text) and GEN (visual) pathways.""" + B = text_mask.shape[0] + S_text = text_mask.shape[1] + text_lengths = text_mask.sum(dim=1).long() + effective_fps = fps if fps is not None and T > 1 else None + + text_pos_list = [] + vis_pos_list = [] + for b in range(B): + real_len = int(text_lengths[b].item()) + t_pos, t_offset = compute_mrope_position_ids_text(real_len, temporal_offset=0) + v_pos, _ = compute_mrope_position_ids_vision( + T, + Hp, + Wp, + temporal_offset=t_offset + self.unified_3d_mrope_temporal_modality_margin, + fps=effective_fps, + base_fps=self.base_fps, + temporal_compression_factor=self.temporal_compression_factor, + enable_fps_modulation=self.enable_fps_modulation, + ) + if real_len < S_text: + t_pos = torch.cat( + [t_pos, torch.zeros(3, S_text - real_len, dtype=t_pos.dtype)], dim=1 + ) + text_pos_list.append(t_pos) + vis_pos_list.append(v_pos) + + text_pos_ids = torch.stack(text_pos_list, dim=1).to(device) # [3, B, S_text] + vis_pos_ids = torch.stack(vis_pos_list, dim=1).to(device) # [3, B, S_vis] + + rotary_emb = self.language_model.rotary_emb + _dummy = torch.tensor([], dtype=dtype, device=device) + cos_und, sin_und = rotary_emb(_dummy, position_ids=text_pos_ids) + cos_gen, sin_gen = rotary_emb(_dummy, position_ids=vis_pos_ids) + + freqs_und = (cos_und.unsqueeze(2), sin_und.unsqueeze(2)) # (B, S, 1, 128) + freqs_gen = (cos_gen.unsqueeze(2), sin_gen.unsqueeze(2)) + return freqs_und, freqs_gen + + def reset_cache(self): + self.cached_kv = None + self.cached_freqs_gen = None + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + text_ids: torch.Tensor, + text_mask: torch.Tensor, + video_shape: Tuple[int, int, int], + fps: float | None = None, + noisy_frame_mask: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + """ + Forward pass for parallel denoising. + + Args: + hidden_states: [B, C, T, H, W] noisy latents + timestep: [B] diffusion timestep per sample + text_ids: [B, S_text] tokenized text input + text_mask: [B, S_text] attention mask for text (1=real, 0=pad) + video_shape: (T, H, W) in latent space + fps: video frame rate; when provided, temporal mRoPE positions are + scaled to reflect real time (FPS modulation). + noisy_frame_mask: Optional [B, 1, T, 1, 1] mask where 1=noisy (add + timestep embedding, predict velocity) and 0=conditioned (clean + context, skip timestep embedding). None means all frames noisy + (T2V mode). + + Returns: + [B, C, T, H, W] velocity prediction + """ + T, H, W = video_shape + Hp, Wp, _, _ = self._pad_to_patch_size(H, W) + max_real_len = text_mask.sum(dim=1).max().item() + + hidden_gen = self.vae2llm(self.patchify(hidden_states, T, H, W)) + + with torch.autocast("cuda", enabled=True, dtype=torch.float32): + time_embed = self.time_embedder((timestep * self.timestep_scale)) + time_embed = time_embed.to(hidden_states.dtype) + + if noisy_frame_mask is not None: + # Build per-token mask from per-frame mask. + # noisy_frame_mask: [B, 1, T, 1, 1] → token mask: [B, T*Hp*Wp, 1] + noisy_frame_mask = noisy_frame_mask.expand(hidden_gen.shape[0], -1, -1, -1, -1) + token_noisy_mask = ( + noisy_frame_mask[:, 0, :, 0, 0] # [B, T] + .unsqueeze(-1) # [B, T, 1] + .expand(-1, -1, Hp * Wp) # [B, T, Hp*Wp] + .reshape(hidden_gen.shape[0], -1, 1) # [B, T*Hp*Wp, 1] + ) + hidden_gen = hidden_gen + time_embed.unsqueeze(1) * token_noisy_mask + else: + hidden_gen = hidden_gen + time_embed.unsqueeze(1) + + if self.cached_kv is None: + freqs_und, freqs_gen = self._compute_rope_freqs( + text_mask, + T, + Hp, + Wp, + fps, + hidden_states.device, + hidden_states.dtype, + ) + cached_kv_full = self.language_model(text_ids, text_mask, freqs_und) + self.cached_freqs_gen = freqs_gen + + if self.use_seq_parallel: + rank = self.seq_parallel_rank + # Round max_real_len up to next multiple of ulysses_size. + # At most seq_parallel_size-1 extra positions, negligible softmax dilution. + val = ( + self.seq_parallel_size - max_real_len % self.seq_parallel_size + ) % self.seq_parallel_size + S_text_shard_total = int(max_real_len) + val + S_text_shard = S_text_shard_total // self.seq_parallel_size + + self.cached_kv = [] + for k, v in cached_kv_full: + # Slice to S_text_shard_total; zero out the val padding positions + k = k[:, :S_text_shard_total].clone() + v = v[:, :S_text_shard_total].clone() + if val > 0: + k[:, int(max_real_len) :] = 0 + v[:, int(max_real_len) :] = 0 + self.cached_kv.append( + ( + k[:, rank * S_text_shard : (rank + 1) * S_text_shard], + v[:, rank * S_text_shard : (rank + 1) * S_text_shard], + ) + ) + else: + self.cached_kv = cached_kv_full + + if self.use_seq_parallel: + S_gen = hidden_gen.shape[1] + pad = (self.seq_parallel_size - S_gen % self.seq_parallel_size) % self.seq_parallel_size + if pad > 0: + # This will cause minor noise in softmax due to padding. + hidden_gen = F.pad(hidden_gen, (0, 0, 0, pad)) + cos, sin = self.cached_freqs_gen + cos_padded = F.pad(cos, (0, 0, 0, 0, 0, pad)) + sin_padded = F.pad(sin, (0, 0, 0, 0, 0, pad)) + else: + cos_padded, sin_padded = self.cached_freqs_gen + padded_s_gen = S_gen + pad + S_shard = padded_s_gen // self.seq_parallel_size + hidden_gen = hidden_gen[ + :, self.seq_parallel_rank * S_shard : (self.seq_parallel_rank + 1) * S_shard + ] + # Shard freqs_gen to match + freqs_gen = ( + cos_padded[ + :, self.seq_parallel_rank * S_shard : (self.seq_parallel_rank + 1) * S_shard + ], + sin_padded[ + :, self.seq_parallel_rank * S_shard : (self.seq_parallel_rank + 1) * S_shard + ], + ) + else: + freqs_gen = self.cached_freqs_gen + + for i, layer in enumerate(self.gen_layers): + k_und, v_und = self.cached_kv[i] + if self.seq_parallel_size <= 1: + k_und = k_und[:, :max_real_len] + v_und = v_und[:, :max_real_len] + hidden_gen = layer(hidden_gen, k_und, v_und, freqs_gen) + + if self.use_seq_parallel: + hidden_gen = hidden_gen.contiguous() + parts = [torch.empty_like(hidden_gen) for _ in range(self.seq_parallel_size)] + dist.all_gather(parts, hidden_gen, group=self.seq_parallel_pg) + hidden_gen = torch.cat(parts, dim=1)[:, :S_gen] # [B, S_gen, patch_latent_dim] + + hidden_gen = self.norm_moe_gen(hidden_gen) + return self.unpatchify(self.llm2vae(hidden_gen), T, H, W) + + def load_weights(self, weights: dict) -> None: + """Load weights with key remapping from Cosmos3-Nano / Diffusers checkpoints. + + Expects tensor names as in ``diffusion_pytorch_model.safetensors.index.json`` + (e.g. ``layers.{i}.self_attn.to_q.weight``, ``proj_in.weight``). + Maps UND vs GEN blocks into this module's layout (causal self-attn vs cross-attn + MLPs). + """ + remapped = {} + skip_prefixes = ( + "lm_head.", + "action_modality_embed", + "action_proj_", + "audio_modality_embed", + "audio_proj_", + ) + + for key, value in weights.items(): + k = key + + if k.startswith(skip_prefixes): + continue + + if k.startswith(("vae2llm.", "llm2vae.")): + remapped[k] = value + continue + + if k.startswith("proj_in."): + remapped[k.replace("proj_in.", "vae2llm.", 1)] = value + continue + if k.startswith("proj_out."): + remapped[k.replace("proj_out.", "llm2vae.", 1)] = value + continue + + if k.startswith("time_embedder.linear"): + k = k.replace("time_embedder.linear_1.", "time_embedder.mlp.linear_1.") + k = k.replace("time_embedder.linear_2.", "time_embedder.mlp.linear_2.") + remapped[k] = value + continue + + if k.startswith("model."): + k = k[len("model.") :] + + # embed_tokens and norm → language_model.* + if k.startswith("embed_tokens.") or k.startswith("norm."): + remapped[f"language_model.{k}"] = value + continue + + # norm_moe_gen stays at top level + if k.startswith("norm_moe_gen."): + remapped[k] = value + continue + + if not k.startswith("layers."): + logger.warning(f"Skipping unknown checkpoint key: {key}") + continue + + parts = k.split(".", 2) # ['layers', '{i}', '{rest}'] + layer_idx = parts[1] + rest = parts[2] + + # UND (language_model) prefix + und_lp = f"language_model.layers.{layer_idx}" + # GEN prefix + gen_lp = f"gen_layers.{layer_idx}" + + # --- UND attention → language_model.layers.{i}.self_attn.* --- + attn_und_map = { + "self_attn.to_q.": f"{und_lp}.self_attn.to_q.", + "self_attn.to_k.": f"{und_lp}.self_attn.to_k.", + "self_attn.to_v.": f"{und_lp}.self_attn.to_v.", + "self_attn.to_out.": f"{und_lp}.self_attn.to_out.0.", + "self_attn.norm_q.": f"{und_lp}.self_attn.norm_q.", + "self_attn.norm_k.": f"{und_lp}.self_attn.norm_k.", + } + + # --- GEN attention → gen_layers.{i}.cross_attention.* --- + attn_gen_map = { + "self_attn.add_q_proj.": f"{gen_lp}.cross_attention.to_q.", + "self_attn.add_k_proj.": f"{gen_lp}.cross_attention.to_k.", + "self_attn.add_v_proj.": f"{gen_lp}.cross_attention.to_v.", + "self_attn.to_add_out.": f"{gen_lp}.cross_attention.to_out.0.", + "self_attn.norm_added_q.": f"{gen_lp}.cross_attention.norm_q.", + "self_attn.norm_added_k.": f"{gen_lp}.cross_attention.norm_k.", + } + + # --- Norms --- + norm_map = { + "input_layernorm.": f"{und_lp}.input_layernorm.", + "post_attention_layernorm.": f"{und_lp}.post_attention_layernorm.", + "input_layernorm_moe_gen.": f"{gen_lp}.input_layernorm.", + "post_attention_layernorm_moe_gen.": f"{gen_lp}.post_attention_layernorm.", + } + + # --- MLPs --- + mlp_map = { + "mlp.gate_proj.": f"{und_lp}.mlp.gate_proj.", + "mlp.up_proj.": f"{und_lp}.mlp.up_proj.", + "mlp.down_proj.": f"{und_lp}.mlp.down_proj.", + "mlp_moe_gen.gate_proj.": f"{gen_lp}.mlp.gate_proj.", + "mlp_moe_gen.up_proj.": f"{gen_lp}.mlp.up_proj.", + "mlp_moe_gen.down_proj.": f"{gen_lp}.mlp.down_proj.", + } + + matched = False + for mapping in [attn_gen_map, attn_und_map, norm_map, mlp_map]: + for pattern, replacement in mapping.items(): + if rest.startswith(pattern): + suffix = rest[len(pattern) :] + remapped[replacement + suffix] = value + matched = True + break + if matched: + break + + if not matched: + logger.warning(f"Unmatched layer key: {key}") + + # --- Load using DynamicLinearWeightLoader (handles QKV/gate_up fusion + quantization) --- + params_map = { + "qkv_proj": ["to_q", "to_k", "to_v"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + loader = DynamicLinearWeightLoader(self.model_config, params_map=params_map) + + for param_name, param in self._parameters.items(): + if param is not None and param_name in remapped: + param.data.copy_(remapped[param_name].to(param.dtype)) + + loaded_linear = 0 + loaded_other = 0 + skipped_modules = [] + for name, module in self.named_modules(): + if len(module._parameters) == 0: + continue + + if isinstance(module, Linear): + weight_dicts = loader.get_linear_weights(module, name, remapped) + if weight_dicts: + loader.load_linear_weights(module, name, weight_dicts) + loaded_linear += 1 + else: + skipped_modules.append(f"{name}(Linear)") + else: + module_weights = loader.filter_weights(name, remapped) + if module_weights: + loaded_other += 1 + else: + has_params = any(p is not None for p in module._parameters.values()) + if has_params and name: + skipped_modules.append(f"{name}({type(module).__name__})") + for param_name, param in module._parameters.items(): + if param is not None and param_name in module_weights: + param.data.copy_(module_weights[param_name].to(param.dtype)) + + def post_load_weights(self) -> None: + """Post-load processing: dtype conversion and Linear finalization.""" + target_dtype = self.model_config.torch_dtype + + self.time_embedder.to(torch.float32) + self.language_model.embed_tokens.to(target_dtype) + self.vae2llm.to(target_dtype) + self.llm2vae.to(target_dtype) + + for _, module in self.named_modules(): + if isinstance(module, Linear) or isinstance(module, Qwen3VLTextRMSNorm): + module.post_load_weights() diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py index 647e73a554a6..9dbb91059dd1 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py @@ -120,6 +120,7 @@ def __init__( fuse_qk_norm_rope=True, config=config, layer_idx=layer_idx, + enable_ulysses=use_ulysses and not self._is_cross_attn, ) # For audio self-attention that may need a runtime Ulysses toggle diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py index 8b5ddaaf84e2..1f6e300c3b51 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py @@ -306,6 +306,7 @@ def __init__( eps=eps, config=model_config, layer_idx=_layer_idx, + enable_ulysses=False, ) if cross_attn_norm: diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index 676f3aec29fa..9cc8f6274bdb 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -51,6 +51,7 @@ def __init__( fuse_qk_norm_rope: Optional[bool] = None, config: Optional[DiffusionModelConfig] = None, layer_idx: Optional[int] = None, + enable_ulysses: bool = True, # make this enable sequence parallelism ): super().__init__() @@ -132,7 +133,7 @@ def __init__( # Currently kept as mutually exclusive. attn2d_size = (vgm.attn2d_row_size * vgm.attn2d_col_size) if vgm else 1 use_attn2d = attn2d_size > 1 and self.qkv_mode != QKVMode.SEPARATE_QKV - use_ulysses = ulysses_size > 1 and self.qkv_mode != QKVMode.SEPARATE_QKV + use_ulysses = ulysses_size > 1 and enable_ulysses # Compute head counts for the backend # Ulysses shards heads across workers; inner backend sees sharded count diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_registry.py b/tensorrt_llm/_torch/visual_gen/pipeline_registry.py index 120bbcd34273..8d804b8dc1a9 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_registry.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_registry.py @@ -182,6 +182,10 @@ def _detect_from_checkpoint(checkpoint_dir: str) -> str: if "Flux" in class_name: return "FluxPipeline" + if "Cosmos3" in class_name: + return "Cosmos3OmniMoTPipeline" + + ######################################################### # 2. Single-safetensors with embedded metadata (LTX-2 specific) detected = AutoPipeline._detect_from_single_safetensors(checkpoint_dir) if detected is not None: From 7443f1d05f0f75fa53241b0e80561424345788ae Mon Sep 17 00:00:00 2001 From: Yukun He <23156053+hyukn@users.noreply.github.com> Date: Thu, 28 May 2026 23:48:24 +0800 Subject: [PATCH 092/308] [https://nvbugs/6115036][fix] Fix NVFP4 engine size estimation and attention DP batch size in trtllm-bench (#13498) Signed-off-by: Yukun He <23156053+hyukn@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 --- .../_torch/pyexecutor/model_engine.py | 35 +++++++++++- tensorrt_llm/bench/benchmark/utils/general.py | 19 ++++--- tensorrt_llm/bench/build/build.py | 6 +++ tensorrt_llm/bench/build/dataclasses.py | 53 ++++++++++++++++--- tensorrt_llm/bench/build/tuning.py | 28 ++++++++-- 5 files changed, 122 insertions(+), 19 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 45aafe381b27..2c0c95ccb5ed 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -869,6 +869,30 @@ def _general_warmup(self, resource_manager: ResourceManager, with self.no_cuda_graph(): self._general_warmup_impl(resource_manager, warmup_requests_configs) + def _assert_all_tp_ranks_have_warmup_batch(self, batch, + num_tokens: int) -> None: + """Assert every TP rank has a valid warmup batch, or raise with diagnostics. + + Under attention-DP, each rank's KV cache available capacity can differ at + runtime, causing _create_warmup_request to return None on some ranks while + others proceed into forward() with tp_comm collectives — deadlocking the + job. This check prevents the deadlock by failing early with diagnostic info. + """ + if self.mapping.tp_size <= 1: + return + has_batch = int(batch is not None) + all_flags = list(self.dist.tp_allgather(has_batch)) + if any(all_flags) and not all(all_flags): + # Gather token counts for diagnostics + all_tokens = list(self.dist.tp_allgather(num_tokens)) + failed_ranks = [i for i, f in enumerate(all_flags) if not f] + raise RuntimeError( + f"Warmup batch creation failed on TP rank(s) {failed_ranks} " + f"but succeeded on others. This would cause a collective " + f"deadlock. Per-rank curr_max_num_tokens: {all_tokens}. " + f"This indicates asymmetric KV cache capacity across TP ranks. " + f"Consider increasing --kv_cache_free_gpu_mem_fraction.") + def _general_warmup_impl( self, resource_manager: ResourceManager, warmup_requests_configs: List[Tuple[int, int]]) -> None: @@ -882,8 +906,12 @@ def _general_warmup_impl( self._create_warmup_request(resource_manager, num_tokens, num_gen_tokens), resource_manager) as batch: + if batch is None and self.mapping.tp_size <= 1: + continue # Not enough KV cache space (single rank, safe to skip) + self._assert_all_tp_ranks_have_warmup_batch( + batch, num_tokens) if batch is None: - continue # Not enough KV cache space + continue # All ranks agree: not enough space logger.info( f"Run warmup with {num_tokens} tokens, include {num_gen_tokens} generation tokens" ) @@ -917,6 +945,11 @@ def _run_autotuner_warmup(self, resource_manager: ResourceManager): resource_manager, curr_max_num_tokens, 0) with self._release_batch_context(warmup_request, resource_manager) as batch: + if batch is None and self.mapping.tp_size <= 1: + pass # Single rank, safe to skip + else: + self._assert_all_tp_ranks_have_warmup_batch( + batch, curr_max_num_tokens) if batch is not None: # Reset the flag is_first_draft for the draft model. # This is necessary for overlap scheduler. diff --git a/tensorrt_llm/bench/benchmark/utils/general.py b/tensorrt_llm/bench/benchmark/utils/general.py index 8c4bf4a1b905..a4b6314324e5 100755 --- a/tensorrt_llm/bench/benchmark/utils/general.py +++ b/tensorrt_llm/bench/benchmark/utils/general.py @@ -79,6 +79,7 @@ def get_settings(params: dict, dataset_metadata: DatasetMetadata, model: str, """ extra_llm_api_options = params.get("extra_llm_api_options") enable_chunked_prefill = params.get("enable_chunked_prefill", False) + enable_attention_dp = False kv_cache_dtype = "auto" mamba_ssm_cache_dtype = params.get("mamba_ssm_cache_dtype", "auto") @@ -86,15 +87,20 @@ def get_settings(params: dict, dataset_metadata: DatasetMetadata, model: str, if extra_llm_api_options: with open(extra_llm_api_options, 'r') as f: llm_args_dict = yaml.safe_load(f) - kv_cache_config = llm_args_dict.get("kv_cache_config", { - "dtype": "auto", - }) - kv_cache_dtype = kv_cache_config.get("dtype", "auto") - mamba_ssm_cache_dtype = kv_cache_config.get("mamba_ssm_cache_dtype", - mamba_ssm_cache_dtype) + if not isinstance(llm_args_dict, dict): + raise TypeError( + f"extra_llm_api_options must contain a YAML mapping, " + f"got {type(llm_args_dict)}") + kv_cache_config = llm_args_dict.get("kv_cache_config", { + "dtype": "auto", + }) + kv_cache_dtype = kv_cache_config.get("dtype", "auto") + mamba_ssm_cache_dtype = kv_cache_config.get("mamba_ssm_cache_dtype", + mamba_ssm_cache_dtype) enable_chunked_prefill = llm_args_dict.get("enable_chunked_prefill", enable_chunked_prefill) + enable_attention_dp = llm_args_dict.get("enable_attention_dp", False) mapping = { "pp_size": params.get("pp"), @@ -137,6 +143,7 @@ def get_settings(params: dict, dataset_metadata: DatasetMetadata, model: str, dataset_metadata.avg_isl, dataset_metadata.avg_osl, params.get("kv_cache_free_gpu_mem_fraction"), + enable_attention_dp=enable_attention_dp, ) logger.info( diff --git a/tensorrt_llm/bench/build/build.py b/tensorrt_llm/bench/build/build.py index f8cb95962ddb..3290828107b4 100644 --- a/tensorrt_llm/bench/build/build.py +++ b/tensorrt_llm/bench/build/build.py @@ -33,6 +33,7 @@ def get_benchmark_engine_settings( target_input_len: int, target_output_len: int, kv_cache_gpu_mem_fraction: float = 0.95, + enable_attention_dp: bool = False, ) -> Tuple[int, int]: """ Retrieve benchmark settings for a specific model + configuration. @@ -43,6 +44,10 @@ def get_benchmark_engine_settings( pp_size (int): Number of pipeline parallel stages. target_input_len (int): Target input length to compile the engine. target_output_len (int): Target output length to compile the engine. + kv_cache_gpu_mem_fraction (float): Fraction of free memory to allocate + for KV cache. + enable_attention_dp (bool): Whether attention data parallelism is + enabled. Raises: ValueError: When the model_name is not supported. @@ -61,6 +66,7 @@ def get_benchmark_engine_settings( target_input_len, target_output_len, kv_cache_gpu_mem_fraction, + enable_attention_dp=enable_attention_dp, ) else: max_batch_size = DEFAULT_MAX_BATCH_SIZE diff --git a/tensorrt_llm/bench/build/dataclasses.py b/tensorrt_llm/bench/build/dataclasses.py index 3ba8e64f0984..39b6f8131f69 100755 --- a/tensorrt_llm/bench/build/dataclasses.py +++ b/tensorrt_llm/bench/build/dataclasses.py @@ -16,6 +16,22 @@ from tensorrt_llm._torch.pyexecutor.config_utils import ( load_pretrained_config, get_qwen3_hybrid_layer_types) +# Mapping from safetensors dtype strings to bytes per element. +# Used to compute checkpoint size from per-dtype element counts. +SAFETENSORS_DTYPE_BYTES = { + 'F64': 8, + 'F32': 4, + 'F16': 2, + 'BF16': 2, + 'I64': 8, + 'I32': 4, + 'I16': 2, + 'I8': 1, + 'U8': 1, + 'BOOL': 1, + 'F8_E4M3': 1, +} + def parse_safetensors_file_metadata(model_path, filename): @@ -121,6 +137,7 @@ class ModelConfig(BaseModel): name: str model_type: str param_count: int + checkpoint_size_in_gb: float = Field(default=0.0) num_hidden_layers: int = Field(validation_alias=AliasChoices( "num_hidden_layers", "n_layer", @@ -181,17 +198,32 @@ def set_values_if_none(self): return self @classmethod - def get_param_count(cls, model_hf_name, hf_model_path): - """ Read the parameter count from HF safetensor metadata. """ + def get_param_count_and_checkpoint_size(cls, model_hf_name, hf_model_path): + """Read parameter count and checkpoint size from safetensors metadata. + + Returns: + Tuple[int, float]: (param_count, checkpoint_size_in_gb). + """ if model_hf_name == "EleutherAI/gpt-j-6b": # GPT-J repo doesn't use safetensor format. param_count = 6053381344 + checkpoint_size_in_gb = param_count * 2 / (1024**3) else: model_name_or_path = hf_model_path or model_hf_name metadata = get_safetensors_metadata(model_name_or_path) param_count = sum(metadata.parameter_count.values()) - assert param_count, f"Can't get valid parameter count for model: {model_name_or_path}." - - return param_count + # Compute actual checkpoint size using per-dtype byte sizes. + # This is critical for quantized models (e.g. NVFP4) where + # different tensors have different dtypes (U8 for packed weights, + # BF16 for non-quantized layers, F8_E4M3 for scales, etc.). + checkpoint_size_in_bytes = sum( + count * SAFETENSORS_DTYPE_BYTES.get(dtype, 1) + for dtype, count in metadata.parameter_count.items()) + checkpoint_size_in_gb = checkpoint_size_in_bytes / (1024**3) + if not param_count: + raise ValueError(f"Can't get valid parameter count for model: " + f"{hf_model_path or model_hf_name}.") + + return param_count, checkpoint_size_in_gb @classmethod def from_hf(cls, model_hf_name, hf_model_path): @@ -199,9 +231,14 @@ def from_hf(cls, model_hf_name, hf_model_path): or model_hf_name, trust_remote_code=True) hf_config = pretrained_config.to_dict() - param_count = cls.get_param_count(model_hf_name, hf_model_path) - - return cls(name=model_hf_name, param_count=param_count, **hf_config) + param_count, checkpoint_size_in_gb = ( + cls.get_param_count_and_checkpoint_size(model_hf_name, + hf_model_path)) + + return cls(name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config) def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): return 0 diff --git a/tensorrt_llm/bench/build/tuning.py b/tensorrt_llm/bench/build/tuning.py index 8710e5e09ee2..d77cf6591dbb 100755 --- a/tensorrt_llm/bench/build/tuning.py +++ b/tensorrt_llm/bench/build/tuning.py @@ -26,6 +26,7 @@ def calc_engine_setting( target_input_len: int, target_output_len: int, kv_cache_gpu_mem_fraction: float = 0.95, + enable_attention_dp: bool = False, ) -> Tuple[int, int]: """ Calculate the engine build settings (max batch size and max num tokens) for a specific model + parallelism mapping + dataset configuration. @@ -44,6 +45,9 @@ def calc_engine_setting( target_output_len (int): Target output length to compile the engine. kv_cache_gpu_mem_fraction (float): Fraction of free memory to allocate for KV cache. + enable_attention_dp (bool): Whether attention data parallelism is + enabled. When True, each TP rank independently manages its own + KV cache and batch, so max_batch_size is computed per-rank. Raises: RuntimeError: When the number of GPUs or amount of KV cache is unable to @@ -67,9 +71,24 @@ def calc_engine_setting( # Number of GPU used for this run. n_gpus = tp_size * pp_size - # Total engine size. - engine_size = model_config.param_count * byte_per_elem / (1024**3) - total_gpu_memory = get_device_memory() * n_gpus + # Use checkpoint size from safetensors metadata for accurate estimation. + # This is critical for quantized models (e.g. NVFP4) where different + # tensors have different dtypes. + if model_config.checkpoint_size_in_gb > 0: + engine_size = model_config.checkpoint_size_in_gb + else: + # Fallback to param_count-based estimation for backward compatibility. + engine_size = model_config.param_count * byte_per_elem / (1024**3) + # With attention DP, each TP rank independently manages its own KV cache. + # Attention weights are replicated across TP ranks, while MoE/MLP weights + # are distributed via expert parallelism (EP). We approximate per-rank + # engine size as total / tp_size, which slightly underestimates because + # replicated attention weights are small relative to distributed MoE weights. + if enable_attention_dp: + total_gpu_memory = get_device_memory() * pp_size + engine_size = engine_size / tp_size + else: + total_gpu_memory = get_device_memory() * n_gpus # Available memory to allocate KV cache. available_memory = total_gpu_memory - engine_size logger.info(f"Estimated engine size: {engine_size:.2f} GB") @@ -136,7 +155,8 @@ def calc_engine_setting( if kv_cache_max_requests < 1: raise RuntimeError("The amount of KV cache memory is insufficient to " "run this model. Please try with more GPUs.") - if cache_memory / n_gpus < 10.0: + warning_gpu_count = pp_size if enable_attention_dp else n_gpus + if cache_memory / warning_gpu_count < 10.0: logger.warning( f"The KV cache memory per GPU is less than 10 GB. " "Performance may be undesirable. Please consider using a different " From 50ca49f8c53da3d703bca1c0f52fb621369ac6e2 Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Thu, 28 May 2026 12:05:43 -0400 Subject: [PATCH 093/308] [https://nvbugs/5972776][fix] Pass IPC HMAC key through file descriptor (#14378) Signed-off-by: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 26 +++++++-- tensorrt_llm/executor/utils.py | 56 +++++++++++++++++-- tensorrt_llm/llmapi/trtllm-llmapi-launch | 19 +++++-- tests/unittest/executor/test_launcher_envs.py | 54 ++++++++++++++++++ 4 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 tests/unittest/executor/test_launcher_envs.py diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index cf677a588a2e..a41664bcc424 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -24,7 +24,8 @@ from tensorrt_llm._tensorrt_engine import LLM from tensorrt_llm._utils import mpi_rank from tensorrt_llm.commands.utils import get_is_diffusion_model -from tensorrt_llm.executor.utils import LlmLauncherEnvs +from tensorrt_llm.executor.utils import (LlmLauncherEnvs, + set_spawn_proxy_process_ipc_hmac_key) from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import (BuildConfig, CapacitySchedulerPolicy, DynamicBatchConfig, KvCacheConfig, @@ -1403,11 +1404,13 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, # This mimics the behavior of trtllm-llmapi-launch # TODO: Make the port allocation atomic free_ipc_addr = find_free_ipc_addr() + ipc_hmac_key = secrets.token_hex(32) + set_spawn_proxy_process_ipc_hmac_key(ipc_hmac_key) + os.environ.pop( + LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, None) os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS] = "1" os.environ[ LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR.value] = free_ipc_addr - os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY. - value] = secrets.token_hex(32) os.environ[DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT. value] = "1" os.environ[DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX] = str(instance_idx) @@ -1423,7 +1426,6 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS in non_mpi_env assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR in non_mpi_env - assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT in non_mpi_env @@ -1446,13 +1448,24 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, signal.signal(signal.SIGTERM, _signal_handler_cleanup_child) signal.signal(signal.SIGINT, _signal_handler_cleanup_child) + read_fd = -1 + write_fd = -1 try: + read_fd, write_fd = os.pipe() + os.write(write_fd, ipc_hmac_key.encode("ascii")) + os.close(write_fd) + write_fd = -1 + non_mpi_env[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD. + value] = str(read_fd) _child_p_global = subprocess.Popen( command, env=non_mpi_env, stdout=sys.stdout, # Redirect to parent's stdout stderr=sys.stderr, # Redirect to parent's stderr + pass_fds=(read_fd, ), start_new_session=True) + os.close(read_fd) + read_fd = -1 logger.info( f"Parent process (PID {os.getpid()}) launched child process (PID {_child_p_global.pid})." @@ -1466,6 +1479,11 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, launch_remote_mpi_session_server(sub_comm) finally: + if write_fd != -1: + os.close(write_fd) + if read_fd != -1: + os.close(read_fd) + # Restore original signal handlers signal.signal(signal.SIGTERM, original_sigterm_handler) signal.signal(signal.SIGINT, original_sigint_handler) diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 12eaa9afefa9..421d943916fd 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -23,12 +23,49 @@ class LlmLauncherEnvs(StrEnum): # Spawn a process for the LLM-API Proxy TLLM_SPAWN_PROXY_PROCESS = "TLLM_SPAWN_PROXY_PROCESS" TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR = "TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR" - TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY" + TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD = ( + "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD") # Whether to use periodical responses handler in await_responses TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT = "TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT" +_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY: bytes | None = None + + +def _normalize_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> bytes: + if isinstance(key, bytes): + if len(key) == 32: + return key + key = key.decode("ascii") + + key_bytes = bytes.fromhex(key) + if len(key_bytes) != 32: + raise ValueError("IPC HMAC key must be 32 bytes.") + return key_bytes + + +def set_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> None: + global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY + _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( + _normalize_spawn_proxy_process_ipc_hmac_key(key)) + + +def _read_spawn_proxy_process_ipc_hmac_key_fd(fd_value: str) -> bytes: + fd = int(fd_value) + chunks: list[bytes] = [] + try: + while True: + chunk = os.read(fd, 4096) + if not chunk: + break + chunks.append(chunk) + finally: + os.close(fd) + + return _normalize_spawn_proxy_process_ipc_hmac_key(b"".join(chunks)) + + def get_spawn_proxy_process_ipc_addr_env() -> str | None: ''' Get the IPC address for the spawn proxy process dynamically. ''' return os.getenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR) @@ -36,11 +73,20 @@ def get_spawn_proxy_process_ipc_addr_env() -> str | None: def get_spawn_proxy_process_ipc_hmac_key_env() -> bytes: ''' Get the HMAC key for the spawn proxy process dynamically. ''' - key = os.getenv("TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY") - assert key is not None, ( - f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY} is not set. " + global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY + if _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY is not None: + return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY + + key_fd = os.environ.pop( + LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD, None) + if key_fd is not None: + _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( + _read_spawn_proxy_process_ipc_hmac_key_fd(key_fd)) + return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY + + raise AssertionError( + f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD} is not set. " "HMAC encryption is required for IPC communication.") - return bytes.fromhex(key) def get_spawn_proxy_process_env() -> bool: diff --git a/tensorrt_llm/llmapi/trtllm-llmapi-launch b/tensorrt_llm/llmapi/trtllm-llmapi-launch index c11a3bbeb171..d35b60046a26 100755 --- a/tensorrt_llm/llmapi/trtllm-llmapi-launch +++ b/tensorrt_llm/llmapi/trtllm-llmapi-launch @@ -40,7 +40,17 @@ function maybe_export_free_tcp_addr_for_spawn_proxy_process { export tllm_mpi_size=$(mpi_world_size) log_stderr "tllm_mpi_size: $tllm_mpi_size" -export TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY=$(openssl rand -hex 32) +unset TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD +ipc_hmac_key=$(openssl rand -hex 32) + +function run_with_ipc_hmac_key { + local fd + exec {fd}<<<"$ipc_hmac_key" + TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD="$fd" "$@" + local status=$? + exec {fd}<&- + return $status +} if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then @@ -74,12 +84,13 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then set +e # Execute the task with cleaned environment - "${task_with_command[@]}" + run_with_ipc_hmac_key "${task_with_command[@]}" task_exit_code=$? log_stderr "Rank${mpi_rank} Task exit code: $task_exit_code" # Stop the MPI Comm server - python3 -m tensorrt_llm.llmapi.mgmn_leader_node --action stop + run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node \ + --action stop mpi_exit_code=$? log_stderr "Rank${mpi_rank} MPI Comm server exit code: $mpi_exit_code" @@ -100,7 +111,7 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then log_stderr "Rank${mpi_rank} run mgmn leader node with mpi_world_size: $(mpi_world_size) ..." log_stderr "Rank0 host: $HOSTNAME" - python3 -m tensorrt_llm.llmapi.mgmn_leader_node + run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node mgmn_leader_node_exit_code=$? log_stderr "Rank${mpi_rank} MGMN leader node exit code: $mgmn_leader_node_exit_code" diff --git a/tests/unittest/executor/test_launcher_envs.py b/tests/unittest/executor/test_launcher_envs.py new file mode 100644 index 000000000000..0d1f96ef1620 --- /dev/null +++ b/tests/unittest/executor/test_launcher_envs.py @@ -0,0 +1,54 @@ +import os + +import pytest + +from tensorrt_llm.executor import utils as executor_utils +from tensorrt_llm.executor.utils import LlmLauncherEnvs + + +def _reset_ipc_hmac_key_env(monkeypatch): + monkeypatch.setattr(executor_utils, "_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY", None) + monkeypatch.delenv( + LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, raising=False + ) + + +def _write_key_fd(key_hex: str) -> int: + read_fd, write_fd = os.pipe() + os.write(write_fd, key_hex.encode("ascii")) + os.close(write_fd) + return read_fd + + +def test_get_spawn_proxy_process_ipc_hmac_key_from_fd(monkeypatch): + _reset_ipc_hmac_key_env(monkeypatch) + key_hex = "01" * 32 + read_fd = _write_key_fd(key_hex) + monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) + + key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() + + assert key == bytes.fromhex(key_hex) + assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ + with pytest.raises(OSError): + os.fstat(read_fd) + + +def test_get_spawn_proxy_process_ipc_hmac_key_caches_fd_key(monkeypatch): + _reset_ipc_hmac_key_env(monkeypatch) + key_hex = "02" * 32 + read_fd = _write_key_fd(key_hex) + monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) + + key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() + + assert key == bytes.fromhex(key_hex) + assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ + assert executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() == key + + +def test_get_spawn_proxy_process_ipc_hmac_key_missing(monkeypatch): + _reset_ipc_hmac_key_env(monkeypatch) + + with pytest.raises(AssertionError, match="HMAC encryption is required"): + executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() From e8a42a1f69baff7978cf2560cf7b868c56de69df Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Thu, 28 May 2026 12:12:01 -0400 Subject: [PATCH 094/308] [https://nvbugs/5911594][fix] Restrict HTTP cluster storage to loopback (#14161) Signed-off-by: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> --- tensorrt_llm/serve/cluster_storage.py | 25 +++++++++ tensorrt_llm/serve/openai_disagg_server.py | 13 ++++- tests/integration/defs/test_e2e.py | 2 +- .../test_lists/qa/llm_function_multinode.txt | 2 +- tests/integration/test_lists/waives.txt | 1 - .../disaggregated/test_cluster_storage.py | 56 +++++++++++++++++-- tests/unittest/llmapi/apps/openai_server.py | 1 + .../apps/test_disagg_serving_perf_metrics.py | 1 + 8 files changed, 90 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/serve/cluster_storage.py b/tensorrt_llm/serve/cluster_storage.py index d2ba0d911717..bb4f1902b6f4 100644 --- a/tensorrt_llm/serve/cluster_storage.py +++ b/tensorrt_llm/serve/cluster_storage.py @@ -2,6 +2,7 @@ import asyncio import importlib.metadata as importlib_metadata import importlib.util +import ipaddress import sys import time from dataclasses import dataclass @@ -18,6 +19,30 @@ from tensorrt_llm.logger import logger +def is_loopback_host(host: Optional[str]) -> bool: + if not isinstance(host, str) or not host: + return False + if host.lower() == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def validate_http_cluster_storage_scope(cluster_uri: str, + server_host: str) -> None: + parsed_uri = urlparse(cluster_uri) + if parsed_uri.scheme not in ("http", "https"): + return + if is_loopback_host(parsed_uri.hostname) and is_loopback_host(server_host): + return + raise ValueError( + "HTTP cluster storage is only supported for loopback-only " + "disaggregated serving. Use a loopback disagg_cluster.cluster_uri and " + "hostname, or use etcd for cluster storage.") + + def _find_module_file_in_distribution(dist, module_name: str): module_path = module_name.replace(".", "/") candidates = (f"{module_path}/__init__.py", f"{module_path}.py") diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index be5a65c0e45a..f57f251cab87 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -36,8 +36,9 @@ get_ctx_gen_server_addrs, get_global_disagg_request_id) from tensorrt_llm.logger import logger -from tensorrt_llm.serve.cluster_storage import (HttpClusterStorageServer, - create_cluster_storage) +from tensorrt_llm.serve.cluster_storage import ( + HttpClusterStorageServer, create_cluster_storage, + validate_http_cluster_storage_scope) from tensorrt_llm.serve.metadata_server import create_metadata_server from tensorrt_llm.serve.openai_client import OpenAIClient, OpenAIHttpClient from tensorrt_llm.serve.openai_disagg_service import ( @@ -105,7 +106,13 @@ def __init__(self, self._metadata_server = create_metadata_server(metadata_server_cfg) self._perf_metrics_collector = DisaggPerfMetricsCollector(config.perf_metrics_max_requests) - self._disagg_cluster_storage = create_cluster_storage(config.disagg_cluster_config.cluster_uri, config.disagg_cluster_config.cluster_name) if config.disagg_cluster_config else None + self._disagg_cluster_storage = None + if config.disagg_cluster_config: + validate_http_cluster_storage_scope( + config.disagg_cluster_config.cluster_uri, config.hostname) + self._disagg_cluster_storage = create_cluster_storage( + config.disagg_cluster_config.cluster_uri, + config.disagg_cluster_config.cluster_name) self._service = OpenAIDisaggregatedService( self._config, self._ctx_router, self._gen_router, self._create_client, diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 9c1f5b681926..947c191394ba 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1261,7 +1261,7 @@ def test_trtllm_multimodal_benchmark_serving(llm_root, llm_venv): @pytest.mark.skip_less_device(4) @pytest.mark.skip_less_device_memory(40000) -@pytest.mark.parametrize("service_discovery", ["etcd", "http"]) +@pytest.mark.parametrize("service_discovery", ["etcd"]) def test_openai_disagg_multi_nodes_completion_service_discovery( llm_root, llm_venv, service_discovery): test_root = unittest_path() / "llmapi" / "apps" diff --git a/tests/integration/test_lists/qa/llm_function_multinode.txt b/tests/integration/test_lists/qa/llm_function_multinode.txt index 898a65e59b89..a70db0ad96f3 100644 --- a/tests/integration/test_lists/qa/llm_function_multinode.txt +++ b/tests/integration/test_lists/qa/llm_function_multinode.txt @@ -5,4 +5,4 @@ test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp2pp1-gen_tp2pp1] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] -test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] +test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[etcd] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3fbc93110ee6..d77cacddaf45 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -320,7 +320,6 @@ test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-t test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] SKIP (https://nvbugs/6190759) -test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[http] SKIP (https://nvbugs/6115562) test_e2e.py::test_openai_kv_cache_contamination SKIP (https://nvbugs/6227203) test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] SKIP (https://nvbugs/5448523) diff --git a/tests/unittest/disaggregated/test_cluster_storage.py b/tests/unittest/disaggregated/test_cluster_storage.py index 9385fdec71fe..d72c5a38d1ce 100644 --- a/tests/unittest/disaggregated/test_cluster_storage.py +++ b/tests/unittest/disaggregated/test_cluster_storage.py @@ -10,11 +10,10 @@ import uvicorn from fastapi import FastAPI -from tensorrt_llm.serve.cluster_storage import (HttpClusterStorageServer, - StorageItem, WatchEvent, - WatchEventType, - create_cluster_storage, - create_cluster_storage_client) +from tensorrt_llm.serve.cluster_storage import ( + HttpClusterStorageServer, StorageItem, WatchEvent, WatchEventType, + create_cluster_storage, create_cluster_storage_client, is_loopback_host, + validate_http_cluster_storage_scope) _counter = 0 @@ -44,6 +43,53 @@ def run_in_thread(self): timeout = pytest.mark.timeout +@pytest.mark.parametrize( + "host", ["localhost", "LOCALHOST", "LocalHost", "127.0.0.1", "::1"]) +def test_is_loopback_host_accepts_loopback_hosts(host): + assert is_loopback_host(host) + + +@pytest.mark.parametrize("host", + [None, "", "0.0.0.0", "10.0.0.1", "example.com"]) +def test_is_loopback_host_rejects_non_loopback_hosts(host): + assert not is_loopback_host(host) + + +@pytest.mark.parametrize("scheme", ["http", "https"]) +@pytest.mark.parametrize("uri_host", ["localhost", "127.0.0.1", "[::1]"]) +@pytest.mark.parametrize("server_host", ["localhost", "127.0.0.1", "::1"]) +def test_http_cluster_storage_scope_allows_loopback_only( + scheme, uri_host, server_host): + validate_http_cluster_storage_scope(f"{scheme}://{uri_host}:18000", + server_host) + + +@pytest.mark.parametrize( + "cluster_uri, server_host", + [ + ("http://10.0.0.1:18000", "localhost"), + ("http://localhost:18000", "0.0.0.0"), + ("https://example.com:18000", "127.0.0.1"), + ("https://127.0.0.1:18000", "10.0.0.1"), + ], +) +def test_http_cluster_storage_scope_rejects_non_loopback_scope( + cluster_uri, server_host): + with pytest.raises(ValueError, match="loopback-only"): + validate_http_cluster_storage_scope(cluster_uri, server_host) + + +@pytest.mark.parametrize( + "cluster_uri", + [ + "etcd://10.0.0.1:2379", + "etcd://example.com:2379", + ], +) +def test_etcd_cluster_storage_scope_is_unchanged(cluster_uri): + validate_http_cluster_storage_scope(cluster_uri, "0.0.0.0") + + @pytest_asyncio.fixture(scope="function") async def storage_client(storage_server): _, cluster_uri = storage_server diff --git a/tests/unittest/llmapi/apps/openai_server.py b/tests/unittest/llmapi/apps/openai_server.py index ebbe0d5627fe..c5326a3f291a 100644 --- a/tests/unittest/llmapi/apps/openai_server.py +++ b/tests/unittest/llmapi/apps/openai_server.py @@ -164,6 +164,7 @@ def __init__(self, self.disagg_config = self._get_extra_config() if disagg_config: self.disagg_config.update(disagg_config) + self.host = self.disagg_config.get("hostname", self.host) self.log_path = log_path self.log_file = None self.extra_config_file = os.path.join( diff --git a/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py b/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py index 5fc9bdc0f142..9b18e3025d73 100644 --- a/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py +++ b/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py @@ -94,6 +94,7 @@ def worker(server_role: str, port: int): @pytest.fixture def disagg_server(disagg_cluster_config: dict, workers, disagg_port: int): disagg_config = { + "hostname": "localhost", "port": disagg_port, "disagg_cluster": disagg_cluster_config, "perf_metrics_max_requests": 1000, From e17344694238e6e148a2eb95ec2a3d51dd5f6207 Mon Sep 17 00:00:00 2001 From: Aurelien Chartier <2567591+achartier@users.noreply.github.com> Date: Thu, 28 May 2026 09:19:25 -0700 Subject: [PATCH 095/308] [None][fix] Exclude post-merge stages from CBTS force-keep filters (#14594) Signed-off-by: Aurelien Chartier <2567591+achartier@users.noreply.github.com> --- jenkins/L0_Test.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 43ef9179a80a..2b8ab89ed77e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4565,7 +4565,8 @@ def launchTestJobs(pipeline, testFilter) // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus // PackageSanityCheck (kept iff sanity_required) and PerfSanity (kept iff // perfsanity_required). Pure -Perf- stages always excluded (own trigger - // model, full-list benchmarks). + // model, full-list benchmarks). Post-Merge stages are never force-kept; + // they only run when explicitly listed in affected_stages. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { def affectedSet = (cbts.affected_stages ?: []) as Set @@ -4573,6 +4574,7 @@ def launchTestJobs(pipeline, testFilter) def needsPerfSanity = cbts.perfsanity_required parallelJobsFiltered = parallelJobs.findAll { key, _ -> if (key =~ /-Perf-/) return false + if (key =~ /Post-Merge/) return affectedSet.contains(key) return affectedSet.contains(key) || (needsSanity && key =~ /PackageSanityCheck/) || (needsPerfSanity && key =~ /PerfSanity/) From 776bebcd5881ecf563c096b494eac6b4f76faa13 Mon Sep 17 00:00:00 2001 From: tburt-nv <195370667+tburt-nv@users.noreply.github.com> Date: Thu, 28 May 2026 12:47:07 -0400 Subject: [PATCH 096/308] [TRTLLMINF-67][infra] use pre-configured idle GPU exemption (#14587) Signed-off-by: Tyler Burt <195370667+tburt-nv@users.noreply.github.com> --- jenkins/L0_Test.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 2b8ab89ed77e..0751daffcb99 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1168,8 +1168,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } def exemptionComment = "" - if (cluster.host.contains("oci-nrt") || cluster.host.contains("oci-hsg") || cluster.host.contains("lbd-lax")) { - exemptionComment = """--comment='{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"90","reason":"other","description":"Long data and model loading time and disaggregated serving tests"}}'""" + if (SlurmConfig.needsIdleGpuExemption(cluster)) { + exemptionComment = "--comment='${SlurmConfig.IDLE_GPU_EXEMPTION_PAYLOAD}'" } def envExportStatements = envVarsToExport.collect { varName, varValue -> From e96b710f9708930d097d16b1774565f39e3371cf Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 01:09:14 +0800 Subject: [PATCH 097/308] [https://nvbugs/6207749][fix] Replace the spec with `onnx>=1.21.0` in `requirements.txt`; mirror in `security_ (#14577) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- requirements.txt | 2 +- security_scanning/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3a1de400c452..a083e8cad59f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ ftfy lark mpi4py numpy>=2.0.0,<2.4 # numba 0.63.1 requires numpy<2.4 -onnx>=1.18.0,<1.20.0 +onnx>=1.21.0 onnx_graphsurgeon>=0.5.2 graphviz openai diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 50f0f045c028..afceed66e18b 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "lark (>=1.3.1,<2.0.0)", "mpi4py (>=4.1.2,<5.0.0)", "numpy (>=2.0.0,<2.4)", - "onnx (>=1.18.0,<1.20.0)", + "onnx (>=1.21.0)", "onnx-graphsurgeon (>=0.5.2)", "graphviz (>=0.21,<0.22)", "openai (>=2.38.0,<3.0.0)", From 22c79563fb9c744394072650c8f59e52d8da91f3 Mon Sep 17 00:00:00 2001 From: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Date: Thu, 28 May 2026 10:14:25 -0700 Subject: [PATCH 098/308] [https://nvbugs/6185480][fix] Autodeploy skip the GLM accuracy test for pre-hopper (#14656) Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_autodeploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index 632ee879409b..dda8a709331f 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -933,6 +933,7 @@ def get_default_sampling_params(self): n=beam_width, use_beam_search=beam_width > 1) + @skip_pre_hopper @pytest.mark.skip_less_device_memory(80000) @pytest.mark.parametrize("enable_chunked_prefill", [True, False]) @pytest.mark.parametrize("attn_backend", ["flashinfer", "trtllm"]) From da9db3a9527111e81ffeeb01884d783eada32a2f Mon Sep 17 00:00:00 2001 From: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Date: Thu, 28 May 2026 11:00:08 -0700 Subject: [PATCH 099/308] [https://nvbugs/6165866][infra] Waive 1 failed cases for main in pre-merge 40081 (#14653) Signed-off-by: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d77cacddaf45..27f37522b215 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -225,6 +225,7 @@ full:B200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) +full:B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:DGX_H100/kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked] SKIP (https://nvbugs/6136737) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) From 56df2009c645ea24bba243d48eed7c4eaf17d145 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 02:49:31 +0800 Subject: [PATCH 100/308] [https://nvbugs/6187185][fix] Apply the existing `low_memory_overrides()` helper in `TestNemotronV2.test_auto_ (#14584) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Co-authored-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_autodeploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index dda8a709331f..4490dbf76afa 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -498,7 +498,7 @@ def evaluate_tasks(self, llm, sampling_params): task = GSM8K(self.MODEL_NAME) task.evaluate(llm) - @pytest.mark.skip_less_device_memory(32000) + @pytest.mark.skip_less_device_memory(80000) @pytest.mark.parametrize("enable_chunked_prefill", [True, False]) def test_auto_dtype(self, enable_chunked_prefill): kwargs = self.get_default_kwargs(enable_chunked_prefill) From e6784d834b8d5b2a8514decfe74add196b69f8a1 Mon Sep 17 00:00:00 2001 From: Eran Geva <19514940+MrGeva@users.noreply.github.com> Date: Thu, 28 May 2026 22:01:40 +0300 Subject: [PATCH 101/308] [https://nvbugs/6192201][fix] AutoDeploy: unwaive llama perf test and increase its concurrency to 256 (#14691) Signed-off-by: Eran Geva <19514940+MrGeva@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - .../perf-sanity/aggregated/llama3_1_8b_fp8_ad_hopper.yaml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 27f37522b215..a050766806b3 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -289,7 +289,6 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236108) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6227472) -perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6192201) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) diff --git a/tests/scripts/perf-sanity/aggregated/llama3_1_8b_fp8_ad_hopper.yaml b/tests/scripts/perf-sanity/aggregated/llama3_1_8b_fp8_ad_hopper.yaml index 7f76e0b41b1b..8c30ff2b20bf 100644 --- a/tests/scripts/perf-sanity/aggregated/llama3_1_8b_fp8_ad_hopper.yaml +++ b/tests/scripts/perf-sanity/aggregated/llama3_1_8b_fp8_ad_hopper.yaml @@ -13,8 +13,8 @@ server_configs: extra_llm_api_config_path: "examples/auto_deploy/model_registry/configs/llama3_1_8b.yaml" world_size: 1 client_configs: - - name: "con64_iter10_1k1k" - concurrency: 64 + - name: "con256_iter10_1k1k" + concurrency: 256 iterations: 10 isl: 1024 osl: 1024 From f6ba936179b19e9dc8d3dcc05eefa0039cfdbe56 Mon Sep 17 00:00:00 2001 From: mpikulski <206748156+ixlmar@users.noreply.github.com> Date: Thu, 28 May 2026 21:08:55 +0200 Subject: [PATCH 102/308] [TRTLLM-12982][feat] improve attention backend selection (#14635) Signed-off-by: ixlmar <206748156+ixlmar@users.noreply.github.com> --- tensorrt_llm/_torch/attention_backend/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorrt_llm/_torch/attention_backend/utils.py b/tensorrt_llm/_torch/attention_backend/utils.py index 4a657b3d3958..3862e58da279 100644 --- a/tensorrt_llm/_torch/attention_backend/utils.py +++ b/tensorrt_llm/_torch/attention_backend/utils.py @@ -2,6 +2,8 @@ import torch +from tensorrt_llm.logger import logger + from ...models.modeling_utils import QuantConfig from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE from .interface import AttentionBackend, MLAParams, PositionalEmbeddingParams @@ -16,6 +18,7 @@ def get_attention_backend( backend_name: str, sparse_attn_config: Optional["SparseAttentionConfig"] = None ) -> Type[AttentionBackend]: + backend_name = backend_name.upper() if backend_name == "VANILLA": if sparse_attn_config is not None: return get_vanilla_sparse_attn_attention_backend(sparse_attn_config) @@ -34,6 +37,7 @@ def get_attention_backend( from .star_flashinfer import StarAttention return StarAttention + logger.warning("Falling back to TRTLLM attention backend") return TrtllmAttention From e35f0f492b4a1f9006ecb19315e8544986d56f41 Mon Sep 17 00:00:00 2001 From: Yao Yao Date: Fri, 29 May 2026 10:00:37 +0800 Subject: [PATCH 103/308] [None][test] Enable test for kv_cache_manager_v2 for A10 (#12885) Signed-off-by: Yao Yao --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index a050766806b3..65067488cd38 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -218,7 +218,6 @@ examples/test_visual_gen.py::test_wan21_t2v_lpips_against_golden SKIP (https://n examples/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) examples/test_visual_gen.py::test_wan_t2v_example SKIP (https://nvbugs/6215688) examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) -full:A10/unittest/kv_cache_manager_v2_tests/ SKIP (https://nvbugs/5841954) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6185480) full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] SKIP (https://nvbugs/6185480) full:B200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) From fed47f1fb10667cb7b90f167574e79a8574059eb Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Thu, 28 May 2026 19:39:47 -0700 Subject: [PATCH 104/308] [None][infra] Generate json with cmake fetched contents in build stage (#13607) Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- docker/Dockerfile.multi | 9 +- docker/Makefile | 4 + jenkins/Build.groovy | 7 +- jenkins/BuildDockerImage.groovy | 56 ++--- scripts/generate_container_oss_attribution.sh | 2 +- scripts/generate_cpp_dependency_json.py | 221 ++++++++++++++++++ scripts/get_wheel_from_package.py | 5 + 7 files changed, 274 insertions(+), 30 deletions(-) create mode 100644 scripts/generate_cpp_dependency_json.py diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index 6e20b8f3d733..ab8d8ede5c5e 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -119,8 +119,14 @@ ENV CCACHE_DIR=/root/.cache/ccache ARG GITHUB_MIRROR="" ARG BUILD_WHEEL_ARGS="--clean --benchmarks" ARG BUILD_WHEEL_SCRIPT="scripts/build_wheel.py" +ARG PREPARE_WHEEL_FROM_BUILD_STAGE="" RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACHE_DIR} \ - GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} + --mount=type=secret,id=GITLAB_TOKEN \ + GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} && \ + if [ "${PREPARE_WHEEL_FROM_BUILD_STAGE}" != "true" ]; then \ + GITLAB_TOKEN=$(cat /run/secrets/GITLAB_TOKEN 2>/dev/null) \ + python3 scripts/generate_cpp_dependency_json.py --deps-dir cpp/build/_deps --output-dir ./; \ + fi FROM ${DEVEL_IMAGE} AS release @@ -130,6 +136,7 @@ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,from=wheel,sour COPY README.md ./ COPY --from=wheel /src/tensorrt_llm/build/tensorrt_llm*.whl ./ +COPY --from=wheel /src/tensorrt_llm/third-party-sources.json ./ COPY docs docs COPY cpp/include include diff --git a/docker/Makefile b/docker/Makefile index 9a9c44ce8b56..e2a74c15ac36 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,3 +1,5 @@ +COMMA := , + # Default base image for the docker build as defined in Dockerfile.multi BASE_IMAGE ?= $(shell grep '^ARG BASE_IMAGE=' Dockerfile.multi | grep -o '=.*' | tr -d '="') BASE_TAG ?= $(shell grep '^ARG BASE_TAG=' Dockerfile.multi | grep -o '=.*' | tr -d '="') @@ -94,6 +96,7 @@ base_pull: $(if $(TRITON_BASE_TAG), --build-arg TRITON_BASE_TAG=$(TRITON_BASE_TAG)) \ $(if $(BUILD_WHEEL_ARGS), --build-arg BUILD_WHEEL_ARGS="$(BUILD_WHEEL_ARGS)") \ $(if $(BUILD_WHEEL_SCRIPT), --build-arg BUILD_WHEEL_SCRIPT="$(BUILD_WHEEL_SCRIPT)") \ + $(if $(PREPARE_WHEEL_FROM_BUILD_STAGE), --build-arg PREPARE_WHEEL_FROM_BUILD_STAGE="$(PREPARE_WHEEL_FROM_BUILD_STAGE)") \ $(if $(TORCH_INSTALL_TYPE), --build-arg TORCH_INSTALL_TYPE="$(TORCH_INSTALL_TYPE)") \ $(if $(CUDA_VERSION), --build-arg CUDA_VER="$(CUDA_VERSION)") \ $(if $(CUDNN_VERSION), --build-arg CUDNN_VER="$(CUDNN_VERSION)") \ @@ -108,6 +111,7 @@ base_pull: $(if $(SH_ENV), --build-arg SH_ENV="$(SH_ENV)") \ $(if $(BASH_ENV), --build-arg BASH_ENV="$(BASH_ENV)") \ $(if $(STAGE), --target $(STAGE)) \ + $(if $(GITLAB_TOKEN), --secret id=GITLAB_TOKEN$(COMMA)env=GITLAB_TOKEN) \ --file Dockerfile.multi \ --tag $(IMAGE_WITH_TAG) \ .. diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 405280496af8..a2cfc2bcb6c4 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -400,6 +400,9 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) withCredentials([usernamePassword(credentialsId: "urm-artifactory-creds", usernameVariable: 'CONAN_LOGIN_USERNAME', passwordVariable: 'CONAN_PASSWORD')]) { sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]} --benchmarks" } + withCredentials([string(credentialsId: 'svc_tensorrt_llm_oss_gitlab_token_secret', variable: 'GITLAB_TOKEN')]) { + sh "cd ${LLM_ROOT} && python3 scripts/generate_cpp_dependency_json.py --deps-dir cpp/build/_deps --output-dir ./ --token ${GITLAB_TOKEN}" + } if (is_linux_x86_64) { sh "cd ${LLM_ROOT} && python3 scripts/build_cpp_examples.py" } @@ -411,8 +414,10 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) def tritonShortTag = "r26.02" sh "cd ${LLM_ROOT}/triton_backend/inflight_batcher_llm && mkdir build && cd build && cmake .. -DTRTLLM_DIR=${llmPath} -DTRITON_COMMON_REPO_TAG=${tritonShortTag} -DTRITON_CORE_REPO_TAG=${tritonShortTag} -DTRITON_THIRD_PARTY_REPO_TAG=${tritonShortTag} -DTRITON_BACKEND_REPO_TAG=${tritonShortTag} -DUSE_CXX11_ABI=ON && make -j${buildJobs} install" - // Step 3: packaging wheels into tarfile + // Step 3.1: packaging wheels into tarfile sh "cp ${LLM_ROOT}/build/tensorrt_llm-*.whl TensorRT-LLM/" + // Step 3.2: packaging cmake dependency source json into tarfile + sh "cp ${LLM_ROOT}/third-party-sources.json TensorRT-LLM/" // Step 4: packaging tritonserver artifacts into tarfile sh "mkdir -p TensorRT-LLM/triton_backend/inflight_batcher_llm/" diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 595ff8893f2c..c2cc7c7c1813 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -236,7 +236,7 @@ def prepareWheelFromBuildStage(dockerfileStage, arch) { def wheelScript = 'scripts/get_wheel_from_package.py' def wheelArgs = "--arch ${arch} --timeout ${WAIT_TIME_FOR_BUILD_STAGE} --artifact_path " + env.uploadPath - return " BUILD_WHEEL_SCRIPT=${wheelScript} BUILD_WHEEL_ARGS='${wheelArgs}'" + return " BUILD_WHEEL_SCRIPT=${wheelScript} BUILD_WHEEL_ARGS='${wheelArgs}' PREPARE_WHEEL_FROM_BUILD_STAGE='true'" } def buildImage(config, imageKeyToTag) @@ -347,34 +347,36 @@ def buildImage(config, imageKeyToTag) sh "env | sort" def randomSleep = (Math.random() * 600 + 600).toInteger() trtllm_utils.llmExecStepWithRetry(this, script: "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}", sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) - try { - trtllm_utils.llmExecStepWithRetry(this, script: """ - cd ${LLM_ROOT} && make -C docker ${target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${imageWithTag} \ - STAGE=${dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} - """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) - } catch (InterruptedException ex) { - throw ex - } catch (Exception ex) { - if (buildWheelArgs.trim().isEmpty()) { + withCredentials([string(credentialsId: 'svc_tensorrt_llm_oss_gitlab_token_secret', variable: 'GITLAB_TOKEN')]) { + try { + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${imageWithTag} \ + STAGE=${dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} + """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) + } catch (InterruptedException ex) { throw ex + } catch (Exception ex) { + if (buildWheelArgs.trim().isEmpty()) { + throw ex + } + echo "Build failed with wheel arguments, retrying without them" + buildWheelArgs = "" + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${imageWithTag} \ + STAGE=${dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} + """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) } - echo "Build failed with wheel arguments, retrying without them" - buildWheelArgs = "" - trtllm_utils.llmExecStepWithRetry(this, script: """ - cd ${LLM_ROOT} && make -C docker ${target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${imageWithTag} \ - STAGE=${dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} - """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) - } + } // withCredentials if (target == "ngc-release") { imageKeyToTag["NGC Release Image ${config.arch}"] = imageWithTag } diff --git a/scripts/generate_container_oss_attribution.sh b/scripts/generate_container_oss_attribution.sh index f81a5127e2ea..978298891f99 100755 --- a/scripts/generate_container_oss_attribution.sh +++ b/scripts/generate_container_oss_attribution.sh @@ -47,7 +47,7 @@ mkdir -p "${OUTPUT_DIR}" # Generate the attribution file cat > "${OUTPUT_FILE}" << EOF -This container image includes open-source software whose source code is archived in the /third-party-source directory or at ${OSS_URL}. +This container image includes open-source software whose source code is linked in /third-party-sources.json. For further inquiries or assistance, contact us at oss-requests@nvidia.com EOF diff --git a/scripts/generate_cpp_dependency_json.py b/scripts/generate_cpp_dependency_json.py new file mode 100644 index 000000000000..934a5832328d --- /dev/null +++ b/scripts/generate_cpp_dependency_json.py @@ -0,0 +1,221 @@ +"""Generate a JSON manifest of third-party source URLs used in the CMake build. + +This script produces a record of third-party dependencies exactly as consumed +during the build. Each dependency is mirrored to the GitLab OSS components group +(https://gitlab.com/nvidia/tensorrt-llm/oss-components), and the resulting +third-party-sources.json is copied into the container image so that source +references are distributed alongside build artifacts, satisfying open-source +license obligations in a traceable and auditable way. +""" + +import argparse +import json +import logging +import os +import pathlib +import time +import urllib.parse +import urllib.request + +logger = logging.getLogger(__name__) + + +GITLAB_OSS_GROUP = "nvidia/tensorrt-llm/oss-components" +GITLAB_API_BASE = "https://gitlab.com/api/v4" + +REPO_URL_OVERWRITE = {"deep_ep_download": "https://github.com/deepseek-ai/DeepEP"} + +_FETCH_CONTENT_JSON = pathlib.Path(__file__).parent.parent / "3rdparty" / "fetch_content.json" + + +def _load_fetch_content_index() -> dict[str, dict]: + """Return a name->entry mapping from fetch_content.json.""" + data = json.loads(_FETCH_CONTENT_JSON.read_text()) + return {dep["name"]: dep for dep in data.get("dependencies", [])} + + +def get_source_info( + deps_dir: pathlib.Path, + package_name: str, + fetch_content_index: dict[str, dict] | None = None, +) -> dict[str, str]: + """Return {'url': ..., 'tag': ...} for package_name. + + Read directly from fetch_content.json. + """ + index = fetch_content_index or _load_fetch_content_index() + dep = index.get(package_name, {}) + repo = dep.get("git_repository", "").replace("${github_base_url}", "https://github.com") + return {"url": repo, "tag": dep.get("git_tag", "")} + + +def check_oss_components(package_name: str) -> tuple[str, int] | None: + """Return (web_url, project_id) if package_name exists in oss-components, else None.""" + project_path = urllib.parse.quote(f"{GITLAB_OSS_GROUP}/{package_name}", safe="") + url = f"{GITLAB_API_BASE}/projects/{project_path}" + req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + return data["web_url"], data["id"] + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + + +def commit_exists_in_project(project_id: int, ref: str) -> bool: + """Return True if ref (tag or commit SHA) exists in the GitLab project.""" + encoded_ref = urllib.parse.quote(ref, safe="") + url = f"{GITLAB_API_BASE}/projects/{project_id}/repository/commits/{encoded_ref}" + req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) + try: + with urllib.request.urlopen(req) as resp: + resp.read() + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise + + +def get_namespace_id() -> int: + """Return the numeric namespace ID for GITLAB_OSS_GROUP.""" + group_path = urllib.parse.quote(GITLAB_OSS_GROUP, safe="") + url = f"{GITLAB_API_BASE}/groups/{group_path}" + req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + return data["id"] + + +def create_oss_component( + package_name: str, namespace_id: int, upstream_url: str +) -> tuple[str, int]: + """Create a new project under oss-components with a pull mirror and return (web_url, project_id).""" + payload = json.dumps( + { + "name": package_name, + "namespace_id": namespace_id, + "visibility": "public", + "import_url": upstream_url, + "mirror": True, + } + ).encode() + req = urllib.request.Request( + f"{GITLAB_API_BASE}/projects", + data=payload, + headers={"PRIVATE-TOKEN": TOKEN, "Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + return data["web_url"], data["id"] + + +def wait_for_mirror(project_id: int, poll_interval: int = 10, timeout: int = 300) -> None: + """Poll until the project mirror import finishes; return False if it times out.""" + url = f"{GITLAB_API_BASE}/projects/{project_id}" + req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) + deadline = time.monotonic() + timeout + while True: + with urllib.request.urlopen(req) as resp: + status = json.loads(resp.read()).get("import_status") + if status == "finished": + return + if status == "failed": + raise RuntimeError(f"Mirror import failed for project {project_id}") + if time.monotonic() >= deadline: + raise TimeoutError( + f"Mirror import for project {project_id} still {status!r} after {timeout}s" + ) + logger.info("Mirror import status: %s, retrying in %ds...", status, poll_interval) + time.sleep(poll_interval) + + +def main(): + global TOKEN + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--deps-dir", + type=pathlib.Path, + required=True, + help="Path to the third party dependencies directory, e.g. ${CMAKE_BINARY_DIR}/_deps", + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + required=True, + help="Path to the output directory where third party sources will be copied", + ) + parser.add_argument( + "--token", + default=os.environ.get("GITLAB_TOKEN"), + help="GitLab private token (defaults to $GITLAB_TOKEN env var)", + ) + + args = parser.parse_args() + if not args.token: + parser.error("A GitLab token is required: pass --token or set $GITLAB_TOKEN") + TOKEN = args.token + + src_dirs = list(sorted(args.deps_dir.glob("*-src"))) + if not src_dirs: + raise ValueError(f"No source directories found in {args.deps_dir}") + + namespace_id: int | None = None + fetch_content = json.loads(_FETCH_CONTENT_JSON.read_text()) + dep_source_index = {dep["name"]: dep for dep in fetch_content.get("dependencies", [])} + third_party_source_list = [] + + for src_dir in src_dirs: + package_name = src_dir.name.removesuffix("-src") + source_info = get_source_info(args.deps_dir, package_name) + if package_name in REPO_URL_OVERWRITE: + source_info["url"] = REPO_URL_OVERWRITE[package_name] + logger.info( + "%s -> upstream url=%s tag=%s", package_name, source_info["url"], source_info["tag"] + ) + result = check_oss_components(package_name) + if result is not None: + oss_url, project_id = result + logger.info("%s -> found in oss-components: %s", package_name, oss_url) + else: + logger.info("%s -> NOT found in oss-components, creating repo", package_name) + if namespace_id is None: + namespace_id = get_namespace_id() + oss_url, project_id = create_oss_component( + package_name, namespace_id, source_info["url"] + ) + logger.info("%s -> created: %s", package_name, oss_url) + logger.info("%s -> waiting for mirror import to finish", package_name) + wait_for_mirror(project_id) + logger.info("%s -> mirror import finished", package_name) + + tag = source_info["tag"] + if tag and commit_exists_in_project(project_id, tag): + logger.info("%s -> ref %r confirmed in oss-components, updating url", package_name, tag) + if package_name not in dep_source_index: + logger.warning("%s -> not found in fetch_content.json, skipping", package_name) + continue + third_party_source_list.append( + {**dep_source_index[package_name], "git_repository": oss_url} + ) + else: + logger.warning( + "%s -> ref %r not found in oss-components repo, keeping upstream url", + package_name, + tag, + ) + + args.output_dir.mkdir(parents=True, exist_ok=True) + output_path = args.output_dir / "third-party-sources.json" + with open(output_path, "w", encoding="utf-8") as f: + json.dump(third_party_source_list, f, indent=2) + f.write("\n") + logger.info("Wrote oss fetch_content to %s", output_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/scripts/get_wheel_from_package.py b/scripts/get_wheel_from_package.py index cb604482c27b..043a3ab2bdcb 100644 --- a/scripts/get_wheel_from_package.py +++ b/scripts/get_wheel_from_package.py @@ -99,6 +99,11 @@ def get_wheel_from_package(arch, artifact_path, timeout): else: print(f"Warning: Benchmark file not found: {src_path}") + third_party_sources = tmp_dir / "third-party-sources.json" + if third_party_sources.exists(): + shutil.copy2(third_party_sources, llm_root / "third-party-sources.json") + print(f"Copied third-party-sources.json -> {build_dir}") + shutil.rmtree(tmp_dir) if os.path.exists(tarfile_name): From 1bb1a0206bb28ffc5bf33bd149ce72eff0382c0c Mon Sep 17 00:00:00 2001 From: RuQing Xu <7891482+xrq-phys@users.noreply.github.com> Date: Fri, 29 May 2026 11:58:01 +0900 Subject: [PATCH 105/308] [TRTLLM-12436][feat] visual_gen: add CuTe DSL attention via exported binaries (#13721) Signed-off-by: Ruqing Xu <7891482+xrq-phys@users.noreply.github.com> Signed-off-by: RuQing Xu <7891482+xrq-phys@users.noreply.github.com> Co-authored-by: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> --- .gitattributes | 1 + .gitignore | 1 + docs/source/models/visual-generation.md | 51 +- setup.py | 3 +- .../visual_gen/attention_backend/__init__.py | 2 + .../visual_gen/attention_backend/cute_dsl.py | 229 ++++++++ .../visual_gen/attention_backend/parallel.py | 6 +- .../visual_gen/attention_backend/utils.py | 17 +- .../visual_gen/cute_dsl_kernels/__init__.py | 14 + .../cute_dsl_kernels/blackwell/__init__.py | 14 + .../blackwell/attention/__init__.py | 18 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ..._causal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...h128_causal_nonpersistent_varlen_tvmffi.so | 3 + ..._nonpersistent_varlen_lse_skipsm_tvmffi.so | 3 + ...ocausal_nonpersistent_varlen_lse_tvmffi.so | 3 + ...usal_nonpersistent_varlen_skipsm_tvmffi.so | 3 + ...28_nocausal_nonpersistent_varlen_tvmffi.so | 3 + .../blackwell/attention/fmha.py | 494 ++++++++++++++++++ tensorrt_llm/visual_gen/args.py | 97 ++-- .../test_lists/test-db/l0_b200.yml | 3 +- .../visual_gen/test_attention_cute_dsl.py | 248 +++++++++ .../visual_gen/test_attention_integration.py | 151 ++++-- .../_torch/visual_gen/test_attention_perf.py | 109 +++- .../_torch/visual_gen/test_visual_gen_args.py | 22 +- 82 files changed, 1557 insertions(+), 115 deletions(-) create mode 100644 tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl.py create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/__init__.py create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/__init__.py create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so create mode 100644 tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py create mode 100644 tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py diff --git a/.gitattributes b/.gitattributes index cab2fa37863e..177818296355 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,4 @@ docs/source/blogs/media/tech_blog10_full_strategy_performance.png filter=lfs dif docs/source/blogs/media/tech_blog10_context_wait_performance.png filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/cubin/kernelMetaInfo_cubin.cpp filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/cubin/xqa_kernel_cubin.cpp filter=lfs diff=lfs merge=lfs -text +tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/*/*/*.so filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index a09c60676271..962f47d13419 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ tensorrt_llm/flash_mla/ tensorrt_llm/flash_mla_cpp_tllm.*.so tensorrt_llm/flash_mla_cpp_tllm.pyi tensorrt_llm/runtime/kv_cache_manager_v2/**/*.so +!tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so **/*__mypyc*.so tensorrt_llm/scripts *docs/cpp_docs* diff --git a/docs/source/models/visual-generation.md b/docs/source/models/visual-generation.md index 0c40ffe811a2..c1260f91de22 100644 --- a/docs/source/models/visual-generation.md +++ b/docs/source/models/visual-generation.md @@ -12,8 +12,9 @@ Visual generation models based on diffusion transformers (DiT) have become the s TensorRT-LLM **VisualGen** provides a unified inference stack for diffusion models, with a pipeline architecture separate from the LLM inference path. Key capabilities include: - A shared pipeline abstraction covering the denoising loop, guidance strategies, and component loading. -- Pluggable attention backends (PyTorch SDPA and TRT-LLM optimized kernels). +- Pluggable attention backends: PyTorch SDPA (`VANILLA`), TRT-LLM kernels (`TRTLLM`), TRT-LLM CuTe DSL kernels (`CUTEDSL`, Blackwell-class GPUs), and Flash Attention 4 (`FA4`). - Quantization support (dynamic and static) using the [ModelOpt](https://github.com/NVIDIA/TensorRT-Model-Optimizer) configuration format. +- Quantized attention support: `QK16PV8` to quantize Bmm2 on `CUTEDSL`, `SAGE` to run SageAttention on `TRTLLM` (requires Blackwell SM100). - Multi-GPU parallelism (CFG parallel, Ulysses sequence parallel). - **TeaCache** — a runtime caching optimization that skips transformer steps when timestep embeddings change slowly. - `trtllm-serve` integration with OpenAI-compatible API endpoints for image and video generation. @@ -107,6 +108,52 @@ args = VisualGenArgs( ) ``` +### Quantized Attention + +In addition to linear-layer quantization, VisualGen exposes two **attention-level** quantization presets that operate inside the attention kernel. They are configured through `AttentionConfig.quant_attention_config` (or the `--quant_attention_mode` flag in the example scripts) and are mutually exclusive with each other. + +- **QK16PV8** (`CUTEDSL` backend): Keeps Q & K in BF16 and quantizes only V to FP8 (E4M3, per-tensor), thus Bmm1 will be carried out in BF16 with Bmm2 in FP8. Targets Blackwell-class GPUs (`sm_100a` / `sm_103a`) with `head_dim = 128`. +- **SAGE** (`TRTLLM` backend): Quantizes Q, K, and V with per-block scaling factors. Q/K are stored as INT8 or FP8 (e4m3) and V as FP8 (e4m3); block sizes are tunable per axis (typically `(q, k, v) = (1, 4, 1)` for Wan-1.3B and `(1, 16, 1)` for larger Wan / FLUX checkpoints). Supported recipes are validated at runtime. + + +Python API for SageAttention: + +```python +from tensorrt_llm import VisualGenArgs + +args = VisualGenArgs( + model="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + attention_config={ + "backend": "TRTLLM", + "quant_attention_config": { + "qk_dtype": "int8", + "q_block_size": 1, + "k_block_size": 16, + "v_block_size": 1, + }, + }, +) +``` + +Python API for QK16PV8: + +```python +from tensorrt_llm import VisualGenArgs + +args = VisualGenArgs( + model="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + attention_config={ + "backend": "CUTEDSL", + "quant_attention_config": { + "qk_dtype": "bf16", + "q_block_size": 0, + "k_block_size": 0, + "v_block_size": 0, + }, + }, +) +``` + ### TeaCache TeaCache caches transformer outputs when timestep embeddings change slowly between denoising steps, skipping redundant computation. Enable via `VisualGenArgs.cache_config` (YAML or programmatic): @@ -126,7 +173,7 @@ The `teacache_thresh` parameter controls the similarity threshold. Cache-DiT is - **CFG Parallelism** (`--cfg_size 2`): Splits positive/negative guidance prompts across GPUs. - **Ulysses Parallelism** (`--ulysses_size N`): Splits the sequence dimension across GPUs for longer sequences. - **Parallel VAE** (`--parallel_vae_size N`): Shards the final VAE decode along a spatial axis across GPUs, useful to reduce VAE latency and improve GPU utilization (Constraint: `parallel_vae_size ≤ world_size`). Currently only supported for WAN models. -- **Attention Parallel**: There are 2 methods supported to run attention parallel. Both of these methods require the attention backend to support LSE (only FA4 currently) - +- **Attention Parallel**: There are 2 methods supported to run attention parallel. Both of these methods require the attention backend to support LSE (`FA4` and `CUTEDSL`) - - **Attention2D Parallelism** (`--attn2d_row_size N`, `--attn2d_col_size M`): Shards the sequence axis across a 2D `N x M` device mesh, all-gathering Q along rows and K/V along columns so each rank computes a sub-block of the attention matrix (total CP degree = `N * M`; not currently combinable with Ulysses). - **Ring Attention Parallelism** (`--ring_size N`): Shards the sequence axis across a 1D ring of `N` ranks and streams K/V blocks around the ring so each rank computes its attention output without materializing the full K/V (mutually exclusive with Attention2D). ## Developer Guide diff --git a/setup.py b/setup.py index c6f27bc0a229..5656531f5237 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -171,6 +171,7 @@ def has_ext_modules(self): # Include CUDA source for fused MoE align extension so runtime JIT can find it in wheels '_torch/auto_deploy/custom_ops/fused_moe/moe_align_kernel.cu', '_torch/auto_deploy/custom_ops/fused_moe/triton_fused_moe_configs/*', + '_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so', 'usage/schemas/*.json', ] diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/__init__.py b/tensorrt_llm/_torch/visual_gen/attention_backend/__init__.py index 1391a65a4259..eed0c24d8745 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/__init__.py @@ -20,6 +20,7 @@ simplified metadata that doesn't require KV caching. """ +from .cute_dsl import CuTeDSLAttention from .flash_attn4 import FlashAttn4Attention from .interface import AttentionBackend, AttentionTensorLayout from .parallel import Attention2DAttention, RingAttention, UlyssesAttention @@ -33,6 +34,7 @@ "AttentionTensorLayout", "get_visual_gen_attention_backend", "create_attention", + "CuTeDSLAttention", "FlashAttn4Attention", "TrtllmAttention", "TrtllmAttentionMetadata", diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl.py b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl.py new file mode 100644 index 000000000000..013065600207 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +CuTe DSL (NVIDIA kernels) Backend for Visual Generation Models + +Uses pre-compiled cubins derived from CUTLASS CuTe DSL FMHA. +Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16. +""" + +import math +from typing import Optional, Tuple + +import torch + +from tensorrt_llm.visual_gen.args import QuantAttentionConfig + +from ...attention_backend.interface import PredefinedAttentionMask +from .interface import AttentionBackend, AttentionTensorLayout + +_cute_dsl_import_error = None +try: + import tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention as cute_dsl + from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention.fmha import ( + _cute_runtime_import_error, + ) + + if _cute_runtime_import_error is not None: + raise ImportError(_cute_runtime_import_error) +except (ImportError, OSError) as e: + cute_dsl = None + _cute_dsl_import_error = e + + +class CuTeDSLAttention(AttentionBackend): + """ + CuTe DSL (NVIDIA kernels) backend for diffusion models. + + Uses pre-compiled cubin kernels (head_dim=128 only). + """ + + def __init__( + self, + layer_idx: int = 0, + num_heads: int = 8, + head_dim: int = 64, + num_kv_heads: Optional[int] = None, + dtype: Optional[torch.dtype] = None, + quant_attention_config: Optional[QuantAttentionConfig] = None, + skip_softmax_threshold_scale: Optional[float] = None, + **kwargs, + ): + # Only head_dim=128 cubins are packaged. + if head_dim != 128: + raise ValueError(f"CUTEDSL cubins require head_dim=128, got head_dim={head_dim}.") + self.layer_idx = layer_idx + self.num_heads = num_heads + self.head_dim = head_dim + self.num_kv_heads = num_kv_heads or num_heads + self.dtype = dtype + self.quant_attention_config = quant_attention_config + self.skip_softmax_threshold_scale = skip_softmax_threshold_scale + self.scale = 1.0 / math.sqrt(head_dim) + + # CuTe DSL expects [B, S, H, D] format + self._preferred_layout = AttentionTensorLayout.NHD + + def _prepare_inputs( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + attention_mask: PredefinedAttentionMask, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool, torch.dtype]: + """Cast inputs to CuTeDSL-compatible dtype and resolve causal flag.""" + if _cute_dsl_import_error is not None: + raise ImportError( + f"CuTe DSL kernels are not available. Import error: {_cute_dsl_import_error}" + ) from _cute_dsl_import_error + + is_causal = attention_mask == PredefinedAttentionMask.CAUSAL + + # Packaged cubins support float16 and bfloat16 only. + origin_dtype = q.dtype + if q.dtype not in (torch.float16, torch.bfloat16): + q = q.to(torch.bfloat16) + k = k.to(torch.bfloat16) + v = v.to(torch.bfloat16) + return q, k, v, is_causal, origin_dtype + + # cute_dsl.cute_dsl_fmha_fwd is already decorated with @torch.compiler.disable + # Allow torch.compile to fuse preceding linear/norm with quantization of V / seq-preprocess + def _fwd( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + is_causal: bool, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + batch_size, seq_len_q, num_heads, _ = q.shape + _, seq_len_kv, _, value_head_dim = v.shape + out = torch.empty( + batch_size, + seq_len_q, + num_heads, + value_head_dim, + dtype=q.dtype, + device=q.device, + ) + lse = torch.empty( + batch_size, + seq_len_q, + num_heads, + dtype=torch.float32, + device=q.device, + ) + + # Options that instructs quantization of V + scale_v = kwargs.get("scale_v", 1.0) + if self.quant_attention_config is not None: + v_qscale = 448.0 / v.abs().amax().clamp(min=1e-3) + v = (v * v_qscale).to(torch.float8_e4m3fn) + scale_v = scale_v / v_qscale + + # Sequence preproc. + qo_indptr_host = [i * seq_len_q for i in range(batch_size + 1)] + qo_indptr = torch.tensor(qo_indptr_host).to(device=q.device, dtype=torch.int32) + kv_indptr_host = [i * seq_len_kv for i in range(batch_size + 1)] + kv_indptr = torch.tensor(kv_indptr_host).to(device=q.device, dtype=torch.int32) + + # Skip softmax. + skip_softmax_threshold_scale = self.skip_softmax_threshold_scale + if skip_softmax_threshold_scale is not None and skip_softmax_threshold_scale <= 0.0: + skip_softmax_threshold_scale = None + + cute_dsl.cute_dsl_fmha_fwd( + q.flatten(0, 1).contiguous(), + k.flatten(0, 1).contiguous(), + v.flatten(0, 1).contiguous(), + out.flatten(0, 1), + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, + is_causal=is_causal, + sm_scale=self.scale, + lse=lse.flatten(0, 1).contiguous(), + scale_q=kwargs.get("scale_q", 1.0), + scale_k=kwargs.get("scale_k", 1.0), + scale_v=scale_v, + scale_o=kwargs.get("scale_o", 1.0), + max_qo_len=seq_len_q, + max_kv_len=seq_len_kv, + is_persistent=False, + skip_softmax_threshold_scale_factor=skip_softmax_threshold_scale, + ) + return out, lse + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.FULL, + **kwargs, + ) -> torch.Tensor: + """ + Forward pass using CuTe DSL (NVIDIA kernels). + + Dimensions are derived from tensor shapes (NHD layout: ``[B, S, H, D]``). + + Args: + q: Query tensor [batch_size, seq_len, num_heads, head_dim] + k: Key tensor [batch_size, seq_len_kv, num_kv_heads, head_dim] + v: Value tensor [batch_size, seq_len_kv, num_kv_heads, head_dim] + attention_mask: Attention mask type (CAUSAL or FULL) + + Returns: + Output tensor [batch_size, seq_len, num_heads, head_dim] + """ + output, _ = self.forward_with_lse(q, k, v, attention_mask=attention_mask, **kwargs) + return output + + def forward_with_lse( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.FULL, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Forward pass returning both output and log-sum-exp (LSE). + + Returns: + output: [batch_size, seq_len, num_heads, head_dim] + lse: [batch_size, num_heads, seq_len] - log-sum-exp per query position, + always in float32. Used for numerically stable combination of + partial attention results in Attention2D parallelism. + """ + q, k, v, is_causal, origin_dtype = self._prepare_inputs(q, k, v, attention_mask) + output, lse = self._fwd(q, k, v, is_causal, **kwargs) + if output.dtype != origin_dtype: + output = output.to(origin_dtype) + return output, lse.transpose(1, 2) + + @classmethod + def support_lse(cls) -> bool: + return True + + @property + def preferred_layout(self) -> AttentionTensorLayout: + """Return the preferred tensor layout for this backend.""" + return self._preferred_layout + + @classmethod + def support_fused_qkv(cls) -> bool: + return False diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py index 0cd02787cd8b..f55564a2acd6 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py @@ -16,7 +16,7 @@ Parallelism Wrappers Wraps any attention backend with a parallelism strategy. Not a standalone -backend — compose around a real backend (VANILLA/TRTLLM/FA4). +backend — compose around a real backend (VANILLA/TRTLLM/FA4/CUTEDSL). """ @@ -256,8 +256,8 @@ class Attention2DAttention(AttentionBackend): Supported inner backends ------------------------ The inner backend must support LSE output (``support_lse() -> True``) — required - for the reduce-scatter combine step. Currently only the FA4 backend - (``FlashAttn4Attention``) meets this requirement. + for the reduce-scatter combine step. Currently the FA4 and CUTEDSL + backends meet this requirement. Note: ``AttentionTensorLayout.NHD`` and ``AttentionTensorLayout.HND`` are both handled transparently; transposition is applied before the inner forward and diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py index 197374084c49..a3ce3df89cd9 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py @@ -37,7 +37,7 @@ def get_visual_gen_attention_backend( Get diffusion attention backend class by name. Args: - backend_name: Backend identifier ("VANILLA", "TRTLLM", "FA4") + backend_name: Backend identifier ("VANILLA", "TRTLLM", "FA4", "CUTEDSL") Returns: Diffusion attention backend class @@ -48,9 +48,12 @@ def get_visual_gen_attention_backend( - "TRTLLM": Optimized for self-attention (requires same Q/KV seq lengths) Better performance but requires fused QKV - "FA4": Flash Attention 4; provides higher speedup on Blackwell GPUs (sm100) - Requires flash-attn package with cute interface + Requires flash-attn package with cute interface + - "CUTEDSL": CuTe DSL FMHA kernels; uses packaged cubins when present, + otherwise compiles from CuTe DSL source """ # Lazy imports to avoid circular dependency + from .cute_dsl import CuTeDSLAttention from .flash_attn4 import FlashAttn4Attention from .trtllm import TrtllmAttention from .vanilla import VanillaAttention @@ -63,6 +66,8 @@ def get_visual_gen_attention_backend( return TrtllmAttention elif backend_name == "FA4": return FlashAttn4Attention + elif backend_name == "CUTEDSL": + return CuTeDSLAttention else: # Default to VANILLA for maximum compatibility return VanillaAttention @@ -89,7 +94,7 @@ def create_attention( internally, simplifying the forward() call. Args: - backend: Backend identifier ("VANILLA", "TRTLLM", "FA4") + backend: Backend identifier ("VANILLA", "TRTLLM", "FA4", "CUTEDSL") layer_idx: Layer index in the model num_heads: Number of attention heads head_dim: Dimension per head @@ -111,9 +116,9 @@ def create_attention( """ attn_cls = get_visual_gen_attention_backend(backend) - # Extract quant_attention_config from AttentionConfig and pass to TRTLLM backend. - # AttentionConfig validation disables unsupported recipes by normalizing - # quant_attention_config to None. + # Extract quant_attention_config from AttentionConfig and pass to backends + # that support it (TRTLLM SAGE recipes, CUTEDSL QK16PV8). AttentionConfig + # validation rejects unsupported (backend, recipe) combinations upstream. if attention_config is not None and attention_config.quant_attention_config is not None: kwargs["quant_attention_config"] = attention_config.quant_attention_config if backend.upper() == "TRTLLM": diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/__init__.py new file mode 100644 index 000000000000..467079831e16 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/__init__.py new file mode 100644 index 000000000000..467079831e16 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py new file mode 100644 index 000000000000..fdf84901e4c5 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .fmha import cute_dsl_fmha_fwd + +__all__ = ["cute_dsl_fmha_fwd"] diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..93e56313b689 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bde8e845611c0ff97738d3822da3ab406169596065bc703a8c8f1af58aeb4fbd +size 711184 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..8f7b825447a4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d962a68d6d0c89ca5e71d0514b2ea5f1b1da3099c7c89ea4c05cf7c5468f6b0c +size 686552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..6f2a475d0922 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc6c9ff687d1a49bdff39dffcf9e08f74c0114dc63921b458695d1b1add12df +size 702960 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..8c94270cef08 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d71eff9707d7e4464e6b2847ece251c352c783b60056086841cfc1ad5767489 +size 682416 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..b12c8c09af05 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117b99d4733a4cf500515a22f5fb2c87b5c89ffdf7736bc302fff48b27b68289 +size 645672 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..be106aa226d8 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed8d36370827808578ee1b2a7927a8bb964772ba1446a84d7eb66ae11857c28 +size 625128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..df218788b1ad --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:909f85670f5cfeae51fb7e7ca2ec6ee648f272c72c0e65545c2f94a084bbd8d7 +size 637440 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..67cd56345e01 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1bd11b767de8d4b039a4456f799a0461cc0cfa423e4ce97d6bddf1f289a81d +size 616896 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..05f14144f81e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4451124313741a07ee5c616c5fbaf3db110a1ab531bfc5955eeb2a126eb33d3e +size 711096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..86aef559d4ab --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bad02325167d25248a381c9e8cfa4479d93e4f7f2d2bbfff0816db8b03e52ea +size 690552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..2bd77cbe433b --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e71c573fad3a61051b4037cd38ca42daaa2cc6f2c1a007af74c7d5e0107b71 +size 702872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..a15c97e7c4ff --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40cb6caa255f53e90c8c84fb5b5cf13b4776863d76f1d0ead0553fb3b574b169 +size 686424 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..9b939a3c3917 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d48995a1b50324fb7f103a543bd8aa2d48ab8617a972e7bc594a834f78e7888 +size 645576 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6273d4c74ad7 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fff35882b5e0bfbebf2b313a67c7a80398a68df411a1fb30542f11a2e9212f1 +size 629136 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..ba3f0e067efe --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1436b06493487e07c260e5ce283dcd85bfd9551db272d96d2e395788ed34ba5b +size 637352 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..daf2c2df15b8 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:460134c26759d391ac0d4df94627e63794898425f887a210871259fc030d2157 +size 620904 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..10e981459b3c --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6ea5c21b35d5562eec533b601348f527da2ed159c8d6f2a413f02d8b3dfce4 +size 657936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..e153954ec5ee --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed589e4c6f369afb05903be7bee828ab5f8293736cca734390d6eab9ac609f56 +size 637400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..a8b2717c21dc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99801a8fb3c547c9f5778764b14049229e07a7395a2f26e098112ca3968884eb +size 653808 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..0691e9326f5a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:834109925ad81a8b50d44a436c34b9b9ff2f37bf87dc1c3420da10082f951f71 +size 629168 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..a33382575553 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962849852e70fd20d8f086b9257173ddda4f6ca654ed4fa165768a5398e50e75 +size 596520 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6a76fed84298 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a040d0f9333c99a78cdf2baa42f2dc71cf2d8252ac3d1b7005f3afd7b4b536 +size 575976 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..46d5cb25f686 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c241a35bb7ab27b9b4fe26c03f1a8ac4ba1eeb28cfc68fcc0f0d5133a74f393 +size 588288 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..073aa5c4860d --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285541f2d3d081f8726d79c7e6f484af55a06e52e7580fb58a360f9aad5e846b +size 567744 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..02af20762774 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7284ff86f4d04fedf14e87947411e1765b55851ee9d9f12450f0314644abd8e +size 657848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..4eb755d3c573 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f26d9fb521fea7260587558e4a5ec016c2c21b53f0fca580e65c46f7599c898 +size 641400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..a82de70cf9be --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d951a82391c4df1933bf1ff13bf7c455d90c95b19ebf3360c34a3151763ad9cc +size 653720 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..07ab962faa79 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce18821322bdb617a5c26e52465ff1c02d6bd03f6f24ce7604a538a2862e107a +size 633176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..5396cdeac4a3 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5af2f95b9ff7328f1dce2863345ddbf72a39ec4cc0c3ce7601ad904d1c279bb3 +size 592328 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..ec033517f17f --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afe75060cd7f3dd1a37704e2a4fe70aec59bcf60e83f6cc77c510fae8adb1209 +size 575888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..490ac3c2eec1 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:807fc151395aabe1f4fb2bd0451d6d4f8def14e591cff121dc5426c2ee24b841 +size 588200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..d708b622be2a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0faf35561762e27fbf861fe3385298f08da3aa9cc9564b9d3cd99c7fb2f74e5 +size 567656 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..334d4de2da9a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20f59a88225f55116a6269dde1eb677bd527cb91eb1200058765c8c64356b9fb +size 716248 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..ffb16280b537 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7666214fc5357f11f4db5cdae11c11f2c3a42679b30c1534535e12916aed578 +size 694232 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..7a3a7cf23675 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd18be0827da75961d58cb09d0c63fe0826ad9d1a83e16a44b74db43fb7b183b +size 705936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..86656cfff50e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a3da59ea39eb5d41e345a04ba0ab573fb6ef7d950091d4757ac27af2c736e5 +size 683984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..4ce0c3cc1752 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd8a2048373f40ddf71a9706f7a16aef9fb0e5b6487937be695f6e7d0744161 +size 651128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..4b6997fefec4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3407c3c3f177c8ab0b02c69c7cae9968710041d06acef9a10fc5ffa534ec274f +size 630200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..2c01e5f00009 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64ccf41ae8680369363660c1c350386a8b9784141853832fb49d2767e283a9e +size 639728 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..986c6a5db0d5 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eca236d41155a0e2c5b0f3289c5399b60589ffb5b679abff456202566e2bdf0 +size 618192 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..14609fab4bbc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5ad69ef4a76110e48c4f1b6642e3e8b321dc1d794e83957b6228a8c68edfd4 +size 714872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6c4ef0b8a4c7 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1896d588ebbbb6ffde8331432dafccbf996507504b6b86a0ba17bf539f2e8f9 +size 697240 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..40433af18781 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce41d196fa54fb89e5848891e1e402d6511d0e5635098e98f00b0312ce3ad77 +size 704296 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..14d97d8463c3 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f45b423c9980f32c116718dca45f1b05e65ad9abd1e0a70e37a07c9106c6827 +size 686888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..faabf383e21f --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:150cd0b8fa1834d00aa2de8ead15916bc08c4e4f5ba002683ceccca5dc15a793 +size 650496 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..2500b613f8e4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c0eab2d74850c29abd02aa927cc53da3ac730ebed1f8d62e5836c68c3c86a42 +size 635008 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..8c867ef9b986 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d044e658d6d3fda52a038a0d0d8b9cce502f7e383bb96deec344d10e217a6e98 +size 640056 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..5449e7917a22 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20bc7f069ea456ff8274d71dccaceeb979ba877e19a29d956753ebfa8e6000ad +size 624504 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..e22d76d563cc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fee6189e75f9b0233f7ccce9eec7d2f70b277f2f62d599c01f057e1c179235 +size 665400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..cc6eab46060c --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182a606d2ca126fc83c2f914a6e5547d84a9aa668e3526710ca3e7396aac2f8e +size 642776 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..9a06ad3879df --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96052f92a4a4e9b8edc152a2e4e60d0b3c9ffc3e8cad0f72716f4be1072cea5e +size 656112 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..04c07f32cdd9 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c4ca62124eb16d29903dde0f9cf9a46b8a72d3480fee3447736b6c4f2d71895 +size 632560 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..c27e7a6a19dc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b8919f036acd199b4aed790563b63de6d1cfa3ef37bfeec00751a25ee5c29d +size 600408 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..5f9f6cd54c9d --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eff71a1a8838b98ec684ada9e693db4f39a69527d8f072bb52ad3b136e2ad966 +size 580200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..62349117d032 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7d9c9e4ca723169a115029633811b97d6a57880274f668678218466bef9ee0 +size 591488 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..4b09aab76f59 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f0681109e2520f4135d1bb4910d31c11b6d0aae64e09efc6705ba461e2eede +size 568848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..17b50b8f2a9b --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a761c98700c565c98a0e7c7a67a569cf165cdf1fe1262c541ec066f60ffa7c1 +size 663704 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..a4d1b3ecae30 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21a9a2fc02f9ada3d126ee7c054f130fad0b16b62daadd63c881dc63991cf6b +size 644472 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..883e738091b4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9edebdc6f4c23e407f057fba17144122f625db87a29bdd61d14951b8d05aadf +size 655176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..a22d88458722 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9becebe7efb3c0342b41668b6d82888e5a100bccdf906a25c10402ee76b56115 +size 635160 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..836913616f49 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8239bc1f03bef805f46a356a9dcffa449dfa1f112ce8d39297f7cc272da60e48 +size 599536 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..226972ea4776 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb5f65d94e50b7466c027004c9fbfe0c7a5b9b67eec6c139496b70a7a0a85eb +size 581984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..bca39e1a5e2e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca9eeb79fe6a289db4cee5e9c2e8700dd9e138c149da1ec1299ea471efc3c8e +size 589096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..da1001774b1e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c24bd23172aa6d42fc3df1f7221e9ca3ce9c2b4e608386879f1734d58956fa45 +size 571544 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py new file mode 100644 index 000000000000..3de28390c569 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py @@ -0,0 +1,494 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import functools +import math +import platform +from pathlib import Path +from typing import Optional, Union + +import torch + +_cute_runtime_import_error = None +try: + import cutlass + from cuda.bindings import driver as cuda_driver + from cutlass import cute + from cutlass.cute import typing as cute_typing + from cutlass.cute.runtime import from_dlpack +except (ImportError, OSError) as e: + cutlass = None + cute = None + from_dlpack = None + cuda_driver = None + _cute_runtime_import_error = e + + +CUBINS_ROOT = Path(__file__).resolve().parent / "cubins" +SUPPORTED_GPU_ARCHS = ("sm_100a", "sm_103a") + + +def _dtype_to_str(dtype: torch.dtype) -> str: + float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + dtype_map = { + torch.float16: "fp16", + torch.bfloat16: "bf16", + torch.float32: "fp32", + } + if float8_e4m3fn is not None: + dtype_map[float8_e4m3fn] = "e4m3" + try: + return dtype_map[dtype] + except KeyError as exc: + raise ValueError(f"Unsupported CuTe DSL FMHA dtype: {dtype}") from exc + + +def _gpu_cpu_arch() -> str: + arch = platform.machine().lower() + if arch in ("amd64", "x64"): + return "x86_64" + if arch in ("arm64",): + return "aarch64" + return arch + + +def _get_gpu_arch(device: torch.device | str | None = None) -> str: + capability = torch.cuda.get_device_capability(device) + gpu_arch = f"sm_{capability[0]}{capability[1]}a" + if gpu_arch not in SUPPORTED_GPU_ARCHS: + supported = ", ".join(SUPPORTED_GPU_ARCHS) + raise ValueError( + f"Unsupported GPU architecture {gpu_arch}. Supported architectures: {supported}." + ) + return gpu_arch + + +def _get_cubins_dir(gpu_arch: str) -> Path: + return CUBINS_ROOT / _gpu_cpu_arch() / gpu_arch + + +def _get_variant_name( + qk_dtype: torch.dtype, + pv_dtype: torch.dtype, + out_dtype: torch.dtype, + head_dim: int, + is_causal: bool, + is_persistent: bool = True, + varlen: bool = False, + with_lse: bool = False, + enable_skip_softmax: bool = False, + enable_tvm_ffi: bool = False, +) -> str: + qk_str = _dtype_to_str(qk_dtype) + pv_str = _dtype_to_str(pv_dtype) + out_str = _dtype_to_str(out_dtype) + if qk_dtype != pv_dtype: + dtype_str = f"{qk_str}_{pv_str}_{out_str}" + elif qk_dtype != out_dtype: + dtype_str = f"{qk_str}_{out_str}" + else: + dtype_str = qk_str + + causal_str = "causal" if is_causal else "nocausal" + persist_str = "persistent" if is_persistent else "nonpersistent" + varlen_str = "_varlen" if varlen else "" + lse_str = "_lse" if with_lse else "" + skip_str = "_skipsm" if enable_skip_softmax else "" + ffi_str = "_tvmffi" if enable_tvm_ffi else "" + return ( + f"cute_dsl_fmha_{dtype_str}_h{head_dim}_{causal_str}_{persist_str}" + f"{varlen_str}{lse_str}{skip_str}{ffi_str}" + ) + + +def _get_candidate_paths(variant_name: str, gpu_arch: str | None) -> list[Path]: + names = [f"{variant_name}.so", f"{variant_name}.o"] + if gpu_arch is not None: + return [_get_cubins_dir(gpu_arch) / name for name in names] + + host_dir = CUBINS_ROOT / _gpu_cpu_arch() + return [ + host_dir / supported_gpu_arch / name + for supported_gpu_arch in SUPPORTED_GPU_ARCHS + for name in names + ] + + +def _resolve_cubin_path( + variant_name: str, + gpu_arch: str | None = None, +) -> Path: + tried = [] + for candidate in _get_candidate_paths(variant_name, gpu_arch): + tried.append(candidate) + if candidate.exists(): + return candidate.resolve() + + searched = "\n".join(f" - {path}" for path in tried) + default_dir = ( + _get_cubins_dir(gpu_arch) + if gpu_arch is not None + else CUBINS_ROOT / _gpu_cpu_arch() / "" + ) + raise FileNotFoundError( + f"Could not find packaged CuTe DSL FMHA cubins for '{variant_name}'.\n" + f"Expected a .so or .o under {default_dir}.\nSearched:\n{searched}" + ) + + +def _check_cute_runtime_available() -> None: + if cute is not None: + return + raise ImportError( + f"CuTe DSL runtime is not available. Import error: {_cute_runtime_import_error}" + ) from _cute_runtime_import_error + + +def _load_cubin_from_path( + path: str | Path, + variant_name: str | None = None, + enable_tvm_ffi: bool = True, +): + _check_cute_runtime_available() + + cubin_path = Path(path).expanduser().resolve() + if variant_name is None: + variant_name = cubin_path.stem + + module = cute.runtime.load_module(str(cubin_path), enable_tvm_ffi=enable_tvm_ffi) + try: + return getattr(module, variant_name) + except AttributeError as exc: + raise AttributeError( + f"Loaded {cubin_path}, but symbol '{variant_name}' was not found. " + "The symbol name must match the .so filename stem / function prefix." + ) from exc + + +@functools.lru_cache(maxsize=None) +def _load_cute_dsl_fmha_cubin_cached( + variant_name: str, + gpu_arch: str | None, + enable_tvm_ffi: bool, +): + path = _resolve_cubin_path(variant_name, gpu_arch) + return _load_cubin_from_path(path, variant_name, enable_tvm_ffi) + + +def get_cute_dsl_fmha_cubin( + qk_dtype: torch.dtype, + pv_dtype: torch.dtype, + out_dtype: torch.dtype, + head_dim: int, + is_causal: bool, + is_persistent: bool = True, + enable_tvm_ffi: bool = True, + varlen: bool = False, + with_lse: bool = False, + enable_skip_softmax: bool = False, + gpu_arch: str | None = None, +): + variant_name = _get_variant_name( + qk_dtype, + pv_dtype, + out_dtype, + head_dim, + is_causal, + is_persistent, + varlen, + with_lse, + enable_skip_softmax, + enable_tvm_ffi, + ) + return _load_cute_dsl_fmha_cubin_cached( + variant_name, + gpu_arch, + enable_tvm_ffi, + ) + + +def _check_inputs( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, +) -> bool: + if q.dim() not in (3, 4) or k.dim() != q.dim() or v.dim() != q.dim(): + raise ValueError("Expected 3D or 4D Q/K/V tensors with matching ranks.") + if q.dtype != k.dtype: + raise ValueError(f"Q/K dtype mismatch: {q.dtype} vs {k.dtype}") + if q.shape[-1] != k.shape[-1]: + raise ValueError(f"Q/K head dim mismatch: {q.shape[-1]} vs {k.shape[-1]}") + if k.shape[:-1] != v.shape[:-1]: + raise ValueError(f"K/V shape mismatch: {k.shape[:-1]} vs {v.shape[:-1]}") + expected_o_shape = (*q.shape[:-1], v.shape[-1]) + if tuple(o.shape) != expected_o_shape: + raise ValueError(f"Output shape mismatch: {tuple(o.shape)} vs {expected_o_shape}") + return q.dim() == 3 + + +def _to_cint_contiguous(tensor: torch.Tensor) -> torch.Tensor: + return tensor.to(torch.int32).contiguous() + + +def _to_cute_tensor(tensor: torch.Tensor, leading_dim: int): + float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + if tensor.dtype == float8_e4m3fn: + cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) + cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) + cute_tensor.element_type = cutlass.Float8E4M3FN + return cute_tensor + return from_dlpack(tensor, assumed_align=16).mark_layout_dynamic(leading_dim=leading_dim) + + +def _get_runtime_problem( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, + lse: Optional[torch.Tensor], + qo_indptr: Optional[torch.Tensor], + kv_indptr: Optional[torch.Tensor], + max_qo_len: Optional[int], + max_kv_len: Optional[int], + varlen: bool, +) -> tuple[ + int, + int, + int, + int, + int, + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, +]: + if varlen: + if qo_indptr is None or kv_indptr is None or max_qo_len is None or max_kv_len is None: + raise ValueError("Varlen FMHA requires indptr tensors and max sequence lengths.") + if qo_indptr.dim() != 1 or kv_indptr.dim() != 1: + raise ValueError("Varlen FMHA indptr tensors must be 1D.") + if qo_indptr.numel() != kv_indptr.numel() or qo_indptr.numel() < 2: + raise ValueError("Varlen FMHA indptr tensors must have matching non-empty sizes.") + total_q, num_heads_q, head_dim = q.shape + _, num_heads_kv, _ = k.shape + batch_size = qo_indptr.numel() - 1 + max_s_q = max_qo_len + max_s_k = max_kv_len + q_4d = q.unsqueeze(0) + k_4d = k.unsqueeze(0) + v_4d = v.unsqueeze(0) + o_4d = o.unsqueeze(0) + lse_3d = lse.unsqueeze(0) if lse is not None else None + else: + batch_size, max_s_q, num_heads_q, head_dim = q.shape + _, max_s_k, num_heads_kv, _ = k.shape + total_q = batch_size * max_s_q + q_4d = q + k_4d = k + v_4d = v + o_4d = o + lse_3d = lse + value_head_dim = v.shape[-1] + return ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + q_4d, + k_4d, + v_4d, + o_4d, + lse_3d, + ) + + +@torch.compiler.disable +def cute_dsl_fmha_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, + qo_indptr: Optional[torch.Tensor] = None, + kv_indptr: Optional[torch.Tensor] = None, + is_causal: bool = False, + sm_scale: Optional[float] = None, + window_left: int = -1, + window_right: int = -1, + lse: Optional[torch.Tensor] = None, + scale_q: Union[float, torch.Tensor] = 1.0, + scale_k: Union[float, torch.Tensor] = 1.0, + scale_v: Union[float, torch.Tensor] = 1.0, + scale_o: Union[float, torch.Tensor] = 1.0, + enable_tvm_ffi: bool = True, + is_persistent: bool = False, + max_qo_len: Optional[int] = None, + max_kv_len: Optional[int] = None, + kernel_fn=None, + skip_softmax_threshold_scale_factor: Optional[float] = None, +) -> None: + varlen = _check_inputs(q, k, v, o) + + scale_q = float(scale_q.item()) if isinstance(scale_q, torch.Tensor) else scale_q + scale_k = float(scale_k.item()) if isinstance(scale_k, torch.Tensor) else scale_k + scale_v = float(scale_v.item()) if isinstance(scale_v, torch.Tensor) else scale_v + scale_o = float(scale_o.item()) if isinstance(scale_o, torch.Tensor) else scale_o + + ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + q_4d, + k_4d, + v_4d, + o_4d, + lse_3d, + ) = _get_runtime_problem(q, k, v, o, lse, qo_indptr, kv_indptr, max_qo_len, max_kv_len, varlen) + use_skip_softmax = ( + skip_softmax_threshold_scale_factor is not None and skip_softmax_threshold_scale_factor > 0 + ) + problem_size = ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + ) + + if kernel_fn is None: + kernel_fn = get_cute_dsl_fmha_cubin( + q.dtype, + v.dtype, + o.dtype, + head_dim, + is_causal, + is_persistent=is_persistent, + varlen=varlen, + enable_tvm_ffi=enable_tvm_ffi, + with_lse=lse is not None, + enable_skip_softmax=use_skip_softmax, + gpu_arch=_get_gpu_arch(q.device), + ) + + if sm_scale is None: + sm_scale = 1.0 / math.sqrt(head_dim) + scale_softmax = scale_q * scale_k * sm_scale + scale_softmax_log2 = scale_softmax * math.log2(math.exp(1.0)) + scale_output = scale_v / scale_o + + skip_threshold_log2 = None + if use_skip_softmax: + threshold = skip_softmax_threshold_scale_factor / max_s_k + skip_threshold_log2 = cute_typing.Float32(math.log2(threshold)) + + ws_left = None if window_left == -1 else cute_typing.Int32(window_left) + ws_right = None if window_right == -1 else cute_typing.Int32(window_right) + if is_causal and ws_right is None: + ws_right = cute_typing.Int32(0) + + # CUBIN path + num_head_groups = num_heads_q // num_heads_kv + q_5d = q_4d.unflatten(2, (num_heads_kv, num_head_groups)) + k_5d = k_4d.unsqueeze(3) + v_5d = v_4d.unsqueeze(3) + o_5d = o_4d.unflatten(2, (num_heads_kv, num_head_groups)) + lse_4d = lse_3d.unflatten(2, (num_heads_kv, num_head_groups)) if lse is not None else None + + if enable_tvm_ffi: + qo_indptr_i32 = _to_cint_contiguous(qo_indptr) if varlen else None + kv_indptr_i32 = _to_cint_contiguous(kv_indptr) if varlen else None + kernel_fn( + q_5d, + k_5d, + v_5d, + o_5d, + problem_size, + qo_indptr_i32, + kv_indptr_i32, + lse_4d, + None, # sink + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + skip_threshold_log2, + ws_left, + ws_right, + None, + None, + False, # reserved + ) + return + + q_cute = _to_cute_tensor(q_5d, leading_dim=4) + k_cute = _to_cute_tensor(k_5d, leading_dim=4) + v_cute = _to_cute_tensor(v_5d, leading_dim=4) + o_cute = _to_cute_tensor(o_5d, leading_dim=4) + + cum_seqlen_q_cute = None + cum_seqlen_k_cute = None + if varlen: + qo_indptr_i32 = _to_cint_contiguous(qo_indptr) + kv_indptr_i32 = _to_cint_contiguous(kv_indptr) + cum_seqlen_q_cute = from_dlpack(qo_indptr_i32, assumed_align=16).mark_layout_dynamic( + leading_dim=0 + ) + cum_seqlen_k_cute = from_dlpack(kv_indptr_i32, assumed_align=16).mark_layout_dynamic( + leading_dim=0 + ) + + lse_iter = None + if lse is not None: + lse_cute = from_dlpack(lse_4d, assumed_align=16).mark_layout_dynamic(leading_dim=2) + lse_iter = lse_cute.iterator + + stream = cuda_driver.CUstream(torch.cuda.current_stream(q).cuda_stream) + kernel_fn( + q_cute.iterator, + k_cute.iterator, + v_cute.iterator, + o_cute.iterator, + problem_size, + cum_seqlen_q_cute, + cum_seqlen_k_cute, + lse_iter, + None, # sink_iter + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + skip_threshold_log2, + ws_left, + ws_right, + None, + None, + False, # reserved + stream, + ) diff --git a/tensorrt_llm/visual_gen/args.py b/tensorrt_llm/visual_gen/args.py index 8f66a5ad31ab..c4a324779f0c 100644 --- a/tensorrt_llm/visual_gen/args.py +++ b/tensorrt_llm/visual_gen/args.py @@ -44,51 +44,43 @@ class QuantAttentionConfig(StrictBaseModel): - """Attention quantization recipe (TRTLLM backend only). + """Attention quantization recipe (TRTLLM / CUTEDSL backends). - Describes user intent for quantized attention: per-axis dtype and - per-block layout for Q, K, V. Providing this config to - ``AttentionConfig`` enables quantized attention; setting - ``AttentionConfig.quant_attention_config = None`` disables it. + Describes user intent for quantized attention: per-bmm dtype and per-block layout for Q, K, V. + Providing this config to AttentionConfig enables quantized attention; setting + AttentionConfig.quant_attention_config = None disables it. - Bare ``QuantAttentionConfig()`` is a valid finest-granularity recipe - (``qk_dtype="int8"``, ``v_dtype="fp8"``, ``q=k=v=1``); tune - ``k_block_size`` up (4 or 16) for speed. - - The ``"fp8"`` dtype maps to ``e4m3`` (``torch.float8_e4m3fn``) in - the current kernel; ``e5m2`` is not supported on this path. - - Unsupported recipes are rejected by ``AttentionConfig``'s validator - with a ``ValueError``. + Bare QuantAttentionConfig() is a valid Qk16Pv8 recipe. + Unsupported recipes are rejected by AttentionConfig's validator with a ValueError. """ - qk_dtype: Literal["int8", "fp8"] = Field( - "int8", + qk_dtype: Literal["bf16", "int8", "fp8"] = Field( + "bf16", status="prototype", - description="Q/K quantization dtype: 'int8' or 'fp8' (e4m3 in practice).", + description="Q/K quantization dtype; bf16 leaves Q/K unquantized.", ) v_dtype: Literal["fp8"] = Field( "fp8", status="prototype", - description="V quantization dtype. The current kernel always stores V in FP8 (e4m3).", + description="V quantization dtype. The current kernels always load V in FP8 (e4m3).", ) q_block_size: int = Field( - 1, - ge=1, + 0, + ge=0, status="prototype", - description="Elements per quantization block for Q.", + description="Elements per quantization block for Q; 0 for per-tensor quantization.", ) k_block_size: int = Field( - 1, - ge=1, + 0, + ge=0, status="prototype", - description="Elements per quantization block for K.", + description="Elements per quantization block for K; 0 for per-tensor quantization.", ) v_block_size: int = Field( - 1, - ge=1, + 0, + ge=0, status="prototype", - description="Elements per quantization block for V.", + description="Elements per quantization block for V; 0 for per-tensor quantization.", ) @@ -102,16 +94,16 @@ class QuantAttentionConfig(StrictBaseModel): class AttentionConfig(StrictBaseModel): """Configuration for Attention layers.""" - backend: Literal["VANILLA", "TRTLLM", "FA4"] = Field( + backend: Literal["VANILLA", "TRTLLM", "FA4", "CUTEDSL"] = Field( "VANILLA", status="prototype", - description="Attention backend: VANILLA (PyTorch SDPA), TRTLLM, FA4", + description="Attention backend: VANILLA (PyTorch SDPA), TRTLLM, FA4, CUTEDSL", ) quant_attention_config: Optional[QuantAttentionConfig] = Field( None, status="prototype", description=( - "Quantized-attention recipe (TRTLLM backend only). " + "Quantized-attention recipe (TRTLLM / CUTEDSL backends). " "Set to a QuantAttentionConfig instance to enable quantized " "attention; leave as None to disable." ), @@ -132,35 +124,48 @@ class AttentionConfig(StrictBaseModel): @model_validator(mode="after") def _validate_quant_attention_config(self) -> "AttentionConfig": - SUPPORTED_QUANT_RECIPES = { + # SAGE recipes target the TRTLLM backend (per-block Q/K/V scales). + SAGE_RECIPES = { ("int8", "fp8", (1, 1, 1)), ("int8", "fp8", (1, 4, 1)), ("int8", "fp8", (1, 16, 1)), ("fp8", "fp8", (1, 1, 1)), ("fp8", "fp8", (1, 4, 1)), } + # QK16PV8 (CUTEDSL backend): Q/K kept in bf16, V quantized to FP8. + QK16PV8_DTYPES = { + ("bf16", "fp8", (0, 0, 0)), + } if self.quant_attention_config is None: return self - if self.backend != "TRTLLM": - raise ValueError( - f"quant_attention_config requires backend='TRTLLM', " - f"got backend='{self.backend}'. Either set backend='TRTLLM' " - f"or remove quant_attention_config." - ) - - q = self.quant_attention_config + q_config = self.quant_attention_config recipe = ( - q.qk_dtype, - q.v_dtype, - (q.q_block_size, q.k_block_size, q.v_block_size), + q_config.qk_dtype, + q_config.v_dtype, + (q_config.q_block_size, q_config.k_block_size, q_config.v_block_size), ) - if recipe not in SUPPORTED_QUANT_RECIPES: + if self.backend == "TRTLLM": + if recipe not in SAGE_RECIPES: + raise ValueError( + f"Unsupported quant_attention_config={self.quant_attention_config!r} " + f"for backend='TRTLLM'. Supported SAGE recipes " + f"(qk_dtype, v_dtype, (q_block, k_block, v_block)): " + f"{sorted(SAGE_RECIPES)}." + ) + elif self.backend == "CUTEDSL": + if recipe not in QK16PV8_DTYPES: + raise ValueError( + f"Unsupported quant_attention_config={self.quant_attention_config!r} " + f"for backend='CUTEDSL'. Supported (qk_dtype, v_dtype): " + f"{sorted(QK16PV8_DTYPES)}." + ) + else: raise ValueError( - f"Unsupported quant_attention_config={self.quant_attention_config!r}. " - f"Supported recipes (qk_dtype, v_dtype, (q_block, k_block, v_block)): " - f"{sorted(SUPPORTED_QUANT_RECIPES)}." + f"quant_attention_config requires backend in ('TRTLLM', 'CUTEDSL'), " + f"got backend='{self.backend}'. Either change backend or " + f"remove quant_attention_config." ) return self diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 3b04ba55c708..1d1ebe9e2ae2 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -187,9 +187,10 @@ l0_b200: - unittest/_torch/visual_gen/test_teacache.py - unittest/_torch/visual_gen/test_cache_dit.py - unittest/_torch/visual_gen/test_quant_ops.py + - unittest/_torch/visual_gen/test_attention_cute_dsl.py + - unittest/_torch/visual_gen/test_attention_trtllm_sage.py - unittest/_torch/visual_gen/test_attention_integration.py - unittest/_torch/visual_gen/test_attention_perf.py - - unittest/_torch/visual_gen/test_attention_trtllm_sage.py - unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py - unittest/_torch/visual_gen/test_trtllm_serve_e2e.py - unittest/_torch/visual_gen/test_model_loader.py diff --git a/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py new file mode 100644 index 000000000000..eb6a6ec1093a --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch +import torch.nn.functional as F + +from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention import cute_dsl_fmha_fwd +from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention.fmha import ( + get_cute_dsl_fmha_cubin, +) + + +def test_cute_dsl_cubin_kernel_can_import_and_load() -> None: + gpu_arch = _require_supported_gpu_arch() + try: + kernel = get_cute_dsl_fmha_cubin( + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + 128, + is_causal=False, + is_persistent=False, + varlen=True, + enable_tvm_ffi=True, + gpu_arch=gpu_arch, + ) + except ImportError as exc: + pytest.skip(str(exc)) + + assert kernel is not None + + +def _require_supported_gpu_arch() -> str: + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for CuTe DSL FMHA kernels.") + + compute_capability = torch.cuda.get_device_capability() + gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" + if gpu_arch not in ("sm_100a", "sm_103a"): + pytest.skip("CuTe DSL FMHA smoke tests require a supported Blackwell-class GPU.") + + return gpu_arch + + +def _make_indptr(lens: list[int], device: torch.device) -> torch.Tensor: + lens_tensor = torch.tensor(lens, dtype=torch.int32, device=device) + return torch.cat( + [ + torch.zeros(1, dtype=torch.int32, device=device), + lens_tensor.cumsum(0).int(), + ] + ) + + +def _make_tensor( + max_len: int, + total_len: int, + num_heads: int, + head_dim: int, + dtype: torch.dtype, + ref_dtype: torch.dtype, + scale: float, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor]: + tensor_f32 = torch.randn( + max_len + total_len, + num_heads, + head_dim, + dtype=torch.float32, + device=device, + ) + tensor_f32 *= 0.1 + if dtype == torch.float8_e4m3fn: + tensor = (tensor_f32 / scale).to(dtype) + ref = (tensor.float() * scale).to(ref_dtype) + else: + tensor = tensor_f32.to(dtype) + ref = tensor.to(ref_dtype) + return tensor[max_len:], ref[max_len:] + + +def _sdpa_ref( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + is_causal: bool, + sm_scale: float, +) -> torch.Tensor: + out = [] + for batch_idx in range(qo_indptr.numel() - 1): + q_start = int(qo_indptr[batch_idx].item()) + q_end = int(qo_indptr[batch_idx + 1].item()) + kv_start = int(kv_indptr[batch_idx].item()) + kv_end = int(kv_indptr[batch_idx + 1].item()) + + q_i = q[q_start:q_end].transpose(0, 1).unsqueeze(0) + k_i = k[kv_start:kv_end].transpose(0, 1).unsqueeze(0) + v_i = v[kv_start:kv_end].transpose(0, 1).unsqueeze(0) + if q_i.shape[1] != k_i.shape[1]: + repeat_factor = q_i.shape[1] // k_i.shape[1] + k_i = k_i.repeat_interleave(repeat_factor, dim=1) + v_i = v_i.repeat_interleave(repeat_factor, dim=1) + out_i = F.scaled_dot_product_attention( + q_i, + k_i, + v_i, + is_causal=is_causal, + scale=sm_scale, + ) + out.append(out_i.squeeze(0).transpose(0, 1)) + return torch.cat(out, dim=0) + + +@pytest.mark.parametrize( + ("q_lens", "kv_lens", "is_causal"), + [ + pytest.param([256], [256], False, id="single_nocausal"), + pytest.param([512], [512], True, id="single_causal"), + pytest.param([64, 128], [128, 512], False, id="varlen_nocausal"), + ], +) +@pytest.mark.parametrize( + ("qk_dtype", "pv_dtype", "out_dtype", "atol", "rtol"), + [ + pytest.param( + torch.bfloat16, + torch.float8_e4m3fn, + torch.bfloat16, + 5e-2, + 5e-2, + id="bf16_fp8_bf16", + ), + pytest.param( + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + 2e-2, + 2e-2, + id="bf16", + ), + ], +) +@pytest.mark.parametrize( + ("num_heads", "num_heads_kv"), + [ + pytest.param(1, 1, id="mha_1h"), + pytest.param(4, 4, id="mha_4h"), + pytest.param(4, 2, id="gqa_4h_2kv"), + ], +) +@pytest.mark.parametrize("head_dim", [128]) +def test_cute_dsl_fmha_context_forward_cubin_smoke( + q_lens: list[int], + kv_lens: list[int], + is_causal: bool, + qk_dtype: torch.dtype, + pv_dtype: torch.dtype, + out_dtype: torch.dtype, + num_heads: int, + num_heads_kv: int, + head_dim: int, + atol: float, + rtol: float, +) -> None: + gpu_arch = _require_supported_gpu_arch() + device = torch.device("cuda:0") + sm_scale = head_dim**-0.5 + scale_v = 0.06 if pv_dtype == torch.float8_e4m3fn else 1.0 + + torch.manual_seed(42) + torch.cuda.manual_seed_all(42) + qo_indptr = _make_indptr(q_lens, device) + kv_indptr = _make_indptr(kv_lens, device) + total_q = int(qo_indptr[-1].item()) + total_kv = int(kv_indptr[-1].item()) + max_qo_len = max(q_lens) + max_kv_len = max(kv_lens) + + q, q_ref = _make_tensor( + max_qo_len, total_q, num_heads, head_dim, qk_dtype, out_dtype, 1.0, device + ) + k, k_ref = _make_tensor( + max_kv_len, total_kv, num_heads_kv, head_dim, qk_dtype, out_dtype, 1.0, device + ) + v, v_ref = _make_tensor( + max_kv_len, total_kv, num_heads_kv, head_dim, pv_dtype, out_dtype, scale_v, device + ) + out_storage = torch.empty( + max_qo_len + total_q, + num_heads, + head_dim, + dtype=out_dtype, + device=device, + ) + out = out_storage[max_qo_len:] + kernel_fn = get_cute_dsl_fmha_cubin( + qk_dtype, + pv_dtype, + out_dtype, + head_dim, + is_causal, + is_persistent=False, + varlen=True, + enable_tvm_ffi=True, + gpu_arch=gpu_arch, + ) + + cute_dsl_fmha_fwd( + q, + k, + v, + out, + qo_indptr, + kv_indptr, + is_causal=is_causal, + sm_scale=sm_scale, + scale_v=scale_v, + max_qo_len=max_qo_len, + max_kv_len=max_kv_len, + kernel_fn=kernel_fn, + ) + torch.cuda.synchronize() + + out_ref = _sdpa_ref( + q_ref, + k_ref, + v_ref, + qo_indptr, + kv_indptr, + is_causal, + sm_scale, + ) + torch.testing.assert_close(out, out_ref, atol=atol, rtol=rtol) diff --git a/tests/unittest/_torch/visual_gen/test_attention_integration.py b/tests/unittest/_torch/visual_gen/test_attention_integration.py index 893304b4088f..52ebb45062c2 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_integration.py +++ b/tests/unittest/_torch/visual_gen/test_attention_integration.py @@ -7,6 +7,7 @@ """ from types import SimpleNamespace +from typing import Optional import pytest import torch @@ -18,6 +19,7 @@ # ============================================================================ # Flash Attention 4 availability # ============================================================================ +from tensorrt_llm._torch.visual_gen.attention_backend.cute_dsl import _cute_dsl_import_error from tensorrt_llm._torch.visual_gen.attention_backend.flash_attn4 import _flash_attn_fwd as _fa4_fwd from tensorrt_llm._torch.visual_gen.config import ( DiffusionModelConfig, @@ -29,6 +31,7 @@ from tensorrt_llm.visual_gen.args import AttentionConfig, QuantAttentionConfig _flash_attn4_available = _fa4_fwd is not None +_cute_dsl_available = _cute_dsl_import_error is None # ============================================================================ # Original naive implementations for comparison @@ -142,6 +145,20 @@ def create_model_config( return config +def _require_attention_backend(attn_backend: str, head_dim: Optional[int] = None) -> None: + if attn_backend == "FA4" and not _flash_attn4_available: + pytest.fail("FlashAttention 4 backend is required for FA4 attention test") + if attn_backend == "CUTEDSL" and not _cute_dsl_available: + pytest.fail("CuTe DSL backend is required for CUTEDSL attention test") + if attn_backend == "CUTEDSL": + compute_capability = torch.cuda.get_device_capability() + gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" + if gpu_arch not in ("sm_100a", "sm_103a"): + pytest.skip("CUTEDSL attention test requires a supported Blackwell-class GPU") + if head_dim is not None and head_dim != 128: + pytest.skip("CUTEDSL attention test requires head_dim=128") + + def copy_weights_self_attention(naive: NaiveWanSelfAttention, integrated: Attention): """Copy weights from naive to integrated self-attention.""" # QKV projection: naive has to_qkv, integrated has qkv_proj @@ -213,11 +230,22 @@ def generate_rope_embeddings( # ============================================================================ # Test functions # ============================================================================ -@pytest.mark.parametrize("attn_backend", ["VANILLA", "TRTLLM", "FA4"]) -def test_self_attention_equivalence(attn_backend: str): +@pytest.mark.parametrize("head_dim", [32, 128]) +@pytest.mark.parametrize( + ("attn_backend", "quant_attention_config"), + [ + ("VANILLA", None), + ("TRTLLM", None), + ("FA4", None), + ("CUTEDSL", None), + ("CUTEDSL", QuantAttentionConfig(qk_dtype="bf16", v_dtype="fp8")), + ], +) +def test_self_attention_equivalence( + head_dim: int, attn_backend: str, quant_attention_config: "QuantAttentionConfig | None" +): """Test that integrated self-attention produces same output as naive.""" - if attn_backend == "FA4" and not _flash_attn4_available: - pytest.fail("FlashAttention 4 backend is required for FA4 self-attention test") + _require_attention_backend(attn_backend, head_dim) print("\n" + "=" * 60) print("Testing Self-Attention Equivalence") @@ -226,9 +254,8 @@ def test_self_attention_equivalence(attn_backend: str): # Config batch_size = 2 seq_len = 16 - hidden_size = 128 num_heads = 4 - head_dim = hidden_size // num_heads + hidden_size = head_dim * num_heads device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dtype = torch.bfloat16 # Use bf16 since flashinfer doesn't support fp32 @@ -238,7 +265,13 @@ def test_self_attention_equivalence(attn_backend: str): # Create models naive = NaiveWanSelfAttention(hidden_size, num_heads, head_dim, dtype=dtype).to(device) - model_config = create_model_config(hidden_size, num_heads, head_dim, attn_backend=attn_backend) + model_config = create_model_config( + hidden_size, + num_heads, + head_dim, + attn_backend=attn_backend, + quant_attention_config=quant_attention_config, + ) integrated = Attention( hidden_size, num_heads, qkv_mode=QKVMode.FUSE_QKV, config=model_config ).to(device) # self attention @@ -266,13 +299,14 @@ def test_self_attention_equivalence(attn_backend: str): # Compare (using looser tolerance for bf16) max_diff = (out_naive - out_integrated).abs().max().item() mean_diff = (out_naive - out_integrated).abs().mean().item() - is_close = torch.allclose(out_naive, out_integrated, rtol=1e-2, atol=1e-2) + tol = 1e-2 if quant_attention_config is None else 2e-2 + is_close = torch.allclose(out_naive, out_integrated, rtol=tol, atol=tol) print("\nResults:") print(f" Output shape: naive={out_naive.shape}, integrated={out_integrated.shape}") print(f" Max absolute difference: {max_diff:.2e}") print(f" Mean absolute difference: {mean_diff:.2e}") - print(f" Outputs match (rtol=1e-2, atol=1e-2): {is_close}") + print(f" Outputs match (rtol={tol}, atol={tol}): {is_close}") if is_close: print(" ✅ PASS: Self-attention outputs match!") @@ -300,6 +334,10 @@ def test_sage_attention_self_attention(qk_dtype: str, batch_size: int, seq_len: 3. Outputs are finite (no NaN/Inf) 4. Approximate agreement with naive (cosine similarity > 0.99) """ + compute_capability = torch.cuda.get_device_capability() + gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" + if qk_dtype == "int8" and gpu_arch not in ["sm_100a"]: + pytest.skip("Int8 kernels are only available for SM100 devices.") print("\n" + "=" * 60) print(f"Testing SageAttention (qk_dtype={qk_dtype}, B={batch_size}, S={seq_len})") print("=" * 60) @@ -386,11 +424,21 @@ def test_sage_attention_self_attention(qk_dtype: str, batch_size: int, seq_len: return cos_sim > 0.99 -@pytest.mark.parametrize("attn_backend", ["VANILLA", "FA4"]) -def test_cross_attention_equivalence(attn_backend: str): +@pytest.mark.parametrize("head_dim", [32, 128]) +@pytest.mark.parametrize( + ("attn_backend", "quant_attention_config"), + [ + ("VANILLA", None), + ("FA4", None), + ("CUTEDSL", None), + ("CUTEDSL", QuantAttentionConfig(qk_dtype="bf16", v_dtype="fp8")), + ], +) +def test_cross_attention_equivalence( + head_dim: int, attn_backend: str, quant_attention_config: "QuantAttentionConfig | None" +): """Test that integrated cross-attention produces same output as naive.""" - if attn_backend == "FA4" and not _flash_attn4_available: - pytest.fail("FlashAttention 4 backend is required for FA4 cross-attention test") + _require_attention_backend(attn_backend, head_dim) print("\n" + "=" * 60) print("Testing Cross-Attention Equivalence") @@ -400,9 +448,8 @@ def test_cross_attention_equivalence(attn_backend: str): batch_size = 2 seq_len = 16 encoder_seq_len = 24 # Different from query seq_len - hidden_size = 128 num_heads = 4 - head_dim = hidden_size // num_heads + hidden_size = num_heads * head_dim device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dtype = torch.bfloat16 # Use bf16 since flashinfer doesn't support fp32 @@ -414,7 +461,13 @@ def test_cross_attention_equivalence(attn_backend: str): # Create models naive = NaiveWanCrossAttention(hidden_size, num_heads, head_dim, dtype=dtype).to(device) - model_config = create_model_config(hidden_size, num_heads, head_dim, attn_backend=attn_backend) + model_config = create_model_config( + hidden_size, + num_heads, + head_dim, + attn_backend=attn_backend, + quant_attention_config=quant_attention_config, + ) integrated = Attention( hidden_size, num_heads, qkv_mode=QKVMode.SEPARATE_QKV, config=model_config ).to(device) # cross attention @@ -441,13 +494,14 @@ def test_cross_attention_equivalence(attn_backend: str): # Compare (using looser tolerance for bf16) max_diff = (out_naive - out_integrated).abs().max().item() mean_diff = (out_naive - out_integrated).abs().mean().item() - is_close = torch.allclose(out_naive, out_integrated, rtol=1e-2, atol=1e-2) + tol = 1e-2 if quant_attention_config is None else 2e-2 + is_close = torch.allclose(out_naive, out_integrated, rtol=tol, atol=tol) print("\nResults:") print(f" Output shape: naive={out_naive.shape}, integrated={out_integrated.shape}") print(f" Max absolute difference: {max_diff:.2e}") print(f" Mean absolute difference: {mean_diff:.2e}") - print(f" Outputs match (rtol=1e-2, atol=1e-2): {is_close}") + print(f" Outputs match (rtol={tol}, atol={tol}): {is_close}") if is_close: print(" ✅ PASS: Cross-attention outputs match!") @@ -467,12 +521,25 @@ def test_cross_attention_equivalence(attn_backend: str): (1, 2048, 512, 12, 128), ], ) -def test_fa4_cross_attention_wan_shapes( - batch: int, seq_len_q: int, seq_len_kv: int, num_heads: int, head_dim: int +@pytest.mark.parametrize( + ("attn_backend", "quant_attention_config"), + [ + ("FA4", None), + ("CUTEDSL", None), + ("CUTEDSL", QuantAttentionConfig(qk_dtype="bf16", v_dtype="fp8")), + ], +) +def test_fast_cross_attention_wan_shapes( + batch: int, + seq_len_q: int, + seq_len_kv: int, + num_heads: int, + head_dim: int, + attn_backend: str, + quant_attention_config: "QuantAttentionConfig | None", ): - """Test FA4 cross-attention correctness at Wan-realistic shapes.""" - if not _flash_attn4_available: - pytest.fail("FlashAttention 4 backend is required for FA4 Wan-shape tests") + """Test fast cross-attention correctness at Wan-realistic shapes.""" + _require_attention_backend(attn_backend, head_dim) hidden_size = num_heads * head_dim device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -489,15 +556,21 @@ def test_fa4_cross_attention_wan_shapes( device ) - cfg_fa4 = create_model_config(hidden_size, num_heads, head_dim, attn_backend="FA4") - fa4_model = Attention(hidden_size, num_heads, qkv_mode=QKVMode.SEPARATE_QKV, config=cfg_fa4).to( - device + cfg_fast = create_model_config( + hidden_size, + num_heads, + head_dim, + attn_backend=attn_backend, + quant_attention_config=quant_attention_config, ) + fast_model = Attention( + hidden_size, num_heads, qkv_mode=QKVMode.SEPARATE_QKV, config=cfg_fast + ).to(device) copy_weights_cross_attention(naive, ref) - copy_weights_cross_attention(naive, fa4_model) + copy_weights_cross_attention(naive, fast_model) ref.eval() - fa4_model.eval() + fast_model.eval() torch.manual_seed(42) hidden_states = torch.randn(batch, seq_len_q, hidden_size, device=device, dtype=dtype) @@ -505,12 +578,13 @@ def test_fa4_cross_attention_wan_shapes( with torch.no_grad(): out_ref = ref(hidden_states, encoder_hidden_states) - out_fa4 = fa4_model(hidden_states, encoder_hidden_states) + out_fast = fast_model(hidden_states, encoder_hidden_states) - max_diff = (out_ref - out_fa4).abs().max().item() - is_close = torch.allclose(out_ref, out_fa4, rtol=1e-2, atol=1e-2) + max_diff = (out_ref - out_fast).abs().max().item() + tol = 1e-2 if quant_attention_config is None else 2e-2 + is_close = torch.allclose(out_ref, out_fast, rtol=tol, atol=tol) print(f" Max diff: {max_diff:.2e}, match: {is_close}") - assert is_close, f"FA4 cross-attn mismatch at Wan shapes: max_diff={max_diff:.2e}" + assert is_close, f"{attn_backend} cross-attn mismatch at Wan shapes: max_diff={max_diff:.2e}" def test_trtllm_cached_prepare(): @@ -686,8 +760,11 @@ def run_all_tests(): results = {} # Run self-attention tests with different backends - for backend in ["VANILLA", "TRTLLM"] + (["FA4"] if _flash_attn4_available else []): - results[f"self_attention_{backend}"] = test_self_attention_equivalence(backend) + fast_backends = ["FA4"] if _flash_attn4_available else [] + if _cute_dsl_available: + fast_backends.append("CUTEDSL") + for backend in ["VANILLA", "TRTLLM"] + fast_backends: + results[f"self_attention_{backend}"] = test_self_attention_equivalence(backend, "NO_QUANT") # Run SageAttention self-attention tests (subset for manual runner) for batch_size in [1, 2]: @@ -699,9 +776,11 @@ def run_all_tests(): ) # Run cross-attention tests - results["cross_attention_VANILLA"] = test_cross_attention_equivalence("VANILLA") + results["cross_attention_VANILLA"] = test_cross_attention_equivalence("VANILLA", "NO_QUANT") if _flash_attn4_available: - results["cross_attention_FA4"] = test_cross_attention_equivalence("FA4") + results["cross_attention_FA4"] = test_cross_attention_equivalence("FA4", "NO_QUANT") + if _cute_dsl_available: + results["cross_attention_CUTEDSL"] = test_cross_attention_equivalence("CUTEDSL", "NO_QUANT") # Run TRTLLM-specific caching tests results["trtllm_cached_prepare"] = test_trtllm_cached_prepare() diff --git a/tests/unittest/_torch/visual_gen/test_attention_perf.py b/tests/unittest/_torch/visual_gen/test_attention_perf.py index 27cffc6213d8..90a8ac3608c0 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_perf.py +++ b/tests/unittest/_torch/visual_gen/test_attention_perf.py @@ -39,6 +39,7 @@ # ============================================================================ # Flash Attention 4 availability # ============================================================================ +from tensorrt_llm._torch.visual_gen.attention_backend.cute_dsl import _cute_dsl_import_error from tensorrt_llm._torch.visual_gen.attention_backend.flash_attn4 import _flash_attn_fwd as _fa4_fwd from tensorrt_llm._torch.visual_gen.attention_backend.flash_attn4 import ( _flash_attn_fwd_import_error as _fa4_import_error, @@ -51,6 +52,30 @@ from tensorrt_llm.visual_gen.args import AttentionConfig, QuantAttentionConfig _flash_attn4_available = _fa4_fwd is not None +_cute_dsl_available = _cute_dsl_import_error is None + + +def _require_attention_backend(backend: str, head_dim: Optional[int] = None) -> None: + if backend == "FA4" and not _flash_attn4_available: + pytest.fail( + "FlashAttention 4 backend is required for FA4 attention perf test" + + (f": {_fa4_import_error}" if _fa4_import_error else "") + ) + if backend == "CUTEDSL" and not _cute_dsl_available: + pytest.fail( + "CuTe DSL backend is required for CUTEDSL attention perf test" + + (f": {_cute_dsl_import_error}" if _cute_dsl_import_error else "") + ) + if backend == "CUTEDSL": + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for CUTEDSL attention perf test") + compute_capability = torch.cuda.get_device_capability() + gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" + if gpu_arch not in ("sm_100a", "sm_103a"): + pytest.skip("CUTEDSL attention perf test requires a supported Blackwell-class GPU") + if head_dim is not None and head_dim != 128: + pytest.skip("CUTEDSL attention perf test requires head_dim=128") + # NVTX support for profiling try: @@ -277,10 +302,21 @@ def create_attention_model( return model def create_cross_attention_model( - self, hidden_size: int, num_heads: int, head_dim: int, backend: str + self, + hidden_size: int, + num_heads: int, + head_dim: int, + backend: str, + quant_attention_config: "QuantAttentionConfig | None" = None, ) -> Attention: """Create a WAN cross-attention model with specified backend.""" - config = create_model_config(hidden_size, num_heads, head_dim, attn_backend=backend) + config = create_model_config( + hidden_size, + num_heads, + head_dim, + attn_backend=backend, + quant_attention_config=quant_attention_config, + ) model = Attention(hidden_size, num_heads, qkv_mode=QKVMode.SEPARATE_QKV, config=config).to( self.device ) @@ -318,6 +354,7 @@ def benchmark_cross_attn_single( seq_len_kv: int, head_dim: int, backend: str, + quant_attention_config: "QuantAttentionConfig | None" = None, verbose: bool = True, ) -> Optional[Dict]: """Benchmark a single cross-attention configuration. @@ -328,7 +365,13 @@ def benchmark_cross_attn_single( hidden_size = num_heads * head_dim try: - model = self.create_cross_attention_model(hidden_size, num_heads, head_dim, backend) + model = self.create_cross_attention_model( + hidden_size, + num_heads, + head_dim, + backend, + quant_attention_config=quant_attention_config, + ) hidden_states = torch.randn( batch_size, seq_len_q, hidden_size, device=self.device, dtype=self.dtype @@ -473,6 +516,7 @@ def benchmark_comparison( seq_len: int, head_dim: int, description: str = "", + quant_attention_config: "QuantAttentionConfig | None" = None, verbose: bool = True, ) -> Dict[str, Optional[Dict]]: """Benchmark and compare all backends for a given configuration.""" @@ -486,7 +530,13 @@ def benchmark_comparison( results = {} for backend in self.backends: results[backend] = self.benchmark_single( - batch_size, num_heads, seq_len, head_dim, backend, verbose + batch_size, + num_heads, + seq_len, + head_dim, + backend, + quant_attention_config=quant_attention_config, + verbose=verbose, ) # Print comparison @@ -643,19 +693,36 @@ def setup(self): benchmark_iterations=20, ) - @pytest.mark.parametrize("backend", ["VANILLA", "TRTLLM", "FA4"]) - def test_self_attention_perf(self, backend: str): + @pytest.mark.parametrize("head_dim", [64, 128]) + @pytest.mark.parametrize( + ("backend", "quant_attention_config"), + [ + ("VANILLA", None), + ("TRTLLM", None), + ("FA4", None), + ("CUTEDSL", None), + ("CUTEDSL", QuantAttentionConfig(qk_dtype="bf16", v_dtype="fp8")), + ], + ) + def test_self_attention_perf( + self, + head_dim: int, + backend: str, + quant_attention_config: "QuantAttentionConfig | None", + ): """Test that attention backend runs without errors.""" - if backend == "FA4" and not _flash_attn4_available: - pytest.fail( - "FlashAttention 4 backend is required for FA4 self-attention perf test" - + (f": {_fa4_import_error}" if _fa4_import_error else "") - ) + _require_attention_backend(backend, head_dim) - batch_size, num_heads, seq_len, head_dim = 1, 24, 1024, 64 + batch_size, num_heads, seq_len = 1, 24, 1024 result = self.benchmark.benchmark_single( - batch_size, num_heads, seq_len, head_dim, backend, verbose=True + batch_size, + num_heads, + seq_len, + head_dim, + backend, + quant_attention_config=quant_attention_config, + verbose=True, ) assert result is not None, f"{backend} benchmark failed to produce results" @@ -814,7 +881,13 @@ def test_fa4_vs_vanilla_cross_attn_wan_shapes( results = {} for backend in ["VANILLA", "FA4"]: results[backend] = self.benchmark.benchmark_cross_attn_single( - batch, num_heads, seq_len_q, seq_len_kv, head_dim, backend, verbose=True + batch, + num_heads, + seq_len_q, + seq_len_kv, + head_dim, + backend, + verbose=True, ) vanilla = results.get("VANILLA") @@ -850,7 +923,13 @@ def test_fa4_cross_attn_quick( """Quick FA4 cross-attention correctness and timing check.""" for backend in ["VANILLA", "FA4"]: result = self.benchmark.benchmark_cross_attn_single( - batch, num_heads, seq_len_q, seq_len_kv, head_dim, backend, verbose=True + batch, + num_heads, + seq_len_q, + seq_len_kv, + head_dim, + backend, + verbose=True, ) assert result is not None, f"{backend} cross-attn failed" assert result["avg_ms"] > 0 diff --git a/tests/unittest/_torch/visual_gen/test_visual_gen_args.py b/tests/unittest/_torch/visual_gen/test_visual_gen_args.py index 1b64933a4f00..cc3328740223 100644 --- a/tests/unittest/_torch/visual_gen/test_visual_gen_args.py +++ b/tests/unittest/_torch/visual_gen/test_visual_gen_args.py @@ -73,8 +73,8 @@ def test_legacy_linear_field_rejected(self): class TestAttentionConfigQuantValidation: """Unsupported quantized-attention recipes are rejected with ValueError.""" - def test_quant_config_rejected_on_non_trtllm_backend(self): - with pytest.raises(ValidationError, match="requires backend='TRTLLM'"): + def test_quant_config_rejected_on_unsupported_backend(self): + with pytest.raises(ValidationError, match="requires backend in"): AttentionConfig( backend="VANILLA", quant_attention_config=QuantAttentionConfig(), @@ -84,13 +84,25 @@ def test_quant_config_rejected_when_unsupported(self): with pytest.raises(ValidationError, match="Unsupported quant_attention_config"): AttentionConfig( backend="TRTLLM", - quant_attention_config=QuantAttentionConfig(k_block_size=127), + quant_attention_config=QuantAttentionConfig( + qk_dtype="int8", q_block_size=1, k_block_size=127, v_block_size=1 + ), ) - def test_supported_quant_configs(self): + def test_supported_quant_config_sage(self): attention = AttentionConfig( backend="TRTLLM", - quant_attention_config=QuantAttentionConfig(), + quant_attention_config=QuantAttentionConfig( + qk_dtype="int8", q_block_size=1, k_block_size=16, v_block_size=1 + ), + ) + + assert attention.quant_attention_config is not None + + def test_supported_quant_config_cute(self): + attention = AttentionConfig( + backend="CUTEDSL", + quant_attention_config=QuantAttentionConfig(qk_dtype="bf16", v_dtype="fp8"), ) assert attention.quant_attention_config is not None From a471435b35d832b2a1b4adc912b61890f3e44a41 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 28 May 2026 20:03:35 -0700 Subject: [PATCH 106/308] [https://nvbugs/6229221][fix] Add a reasoning parser for qwen3_5 (#14659) Signed-off-by: Michal Guzek --- tensorrt_llm/llmapi/reasoning_parser.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index 77c3fda95fa3..0aa2ed16fe65 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -98,6 +98,15 @@ def finish(self) -> ReasoningParserResult: @register_reasoning_parser("deepseek-r1", reasoning_at_start=True) @register_reasoning_parser("laguna") @register_reasoning_parser("qwen3") +# Qwen3.5 (and forced-thinking Qwen3 variants) use a chat template that +# pre-injects `\n` into the assistant prompt prefix, so the model +# output begins inside the reasoning block with no opening tag to search +# for. That requires `reasoning_at_start=True`. The existing `qwen3` key +# keeps `reasoning_at_start=False` for back-compat, and `parse()` is +# binary on this flag (it either requires `` to be present in the +# output, or assumes the output begins at the start of reasoning) - so +# the two behaviors must be registered under separate keys. +@register_reasoning_parser("qwen3_5", reasoning_at_start=True) @register_reasoning_parser("minimax_m2", reasoning_at_start=True) @register_reasoning_parser("minimax_m2_append_think", reasoning_at_start=True) class DeepSeekR1Parser(BaseReasoningParser): From 8cdde83c16bb44b16a7440cab82d82cae52de459 Mon Sep 17 00:00:00 2001 From: Aurelien Chartier <2567591+achartier@users.noreply.github.com> Date: Thu, 28 May 2026 20:06:29 -0700 Subject: [PATCH 107/308] [None][fix] Reuse batch_indices_cuda across CUDA graph captures in EAGLE3 (#14381) Signed-off-by: Aurelien Chartier <2567591+achartier@users.noreply.github.com> --- tensorrt_llm/_torch/speculative/eagle3.py | 30 +++++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index 56151fb01867..10ecd90966cd 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -64,6 +64,11 @@ def __init__(self, (max_num_tokens, self.hidden_size * config.num_capture_layers), dtype=self.dtype, device='cuda') + self.batch_indices_cuda = torch.empty( + [max_num_requests], + dtype=torch.int, + device='cuda', + ) # sequence length, only used for metadata preparation self.seq_lens = {i: 0 for i in range(slot_size)} # start indices of each slot @@ -131,9 +136,15 @@ class Eagle3OneModelDynamicTreeResourceManager(BaseResourceManager): """ hidden_states: Optional[torch.Tensor] = None + batch_indices_cuda: Optional[torch.Tensor] = None def __init__(self, config: "EagleDecodingConfig", max_num_requests: int): self.max_num_requests = max_num_requests + self.batch_indices_cuda = torch.empty( + [max_num_requests], + dtype=torch.int, + device='cuda', + ) self.spec_tree_manager = SpecTreeManager( max_num_requests=max_num_requests, use_dynamic_tree=config.use_dynamic_tree, @@ -389,11 +400,20 @@ def __post_init__(self): self.hidden_size * len(self.layers_to_capture)), dtype=self.dtype, device='cuda') - self.batch_indices_cuda = torch.empty( - [self.max_num_requests], - dtype=torch.int, - device='cuda', - ) + if (self.spec_resource_manager is not None + and self.spec_resource_manager.batch_indices_cuda is not None): + self.batch_indices_cuda = self.spec_resource_manager.batch_indices_cuda + assert self.batch_indices_cuda.shape[0] >= self.max_num_requests, ( + f"batch_indices_cuda shape mismatch: " + f"{type(self.spec_resource_manager).__name__} has " + f"{self.batch_indices_cuda.shape}, but metadata expects " + f"at least ({self.max_num_requests},)") + else: + self.batch_indices_cuda = torch.empty( + [self.max_num_requests], + dtype=torch.int, + device='cuda', + ) # Set tree flags based on config if self.use_dynamic_tree: From dedd826651704b872d5e1bc69c5d607d4619fa47 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 03:12:35 +0000 Subject: [PATCH 108/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- security_scanning/docs/poetry.lock | 6 +- security_scanning/examples/apps/poetry.lock | 6 +- .../examples/auto_deploy/poetry.lock | 12 +- .../examples/draft_target_model/poetry.lock | 12 +- security_scanning/examples/eagle/poetry.lock | 12 +- .../llm-eval/lm-eval-harness/poetry.lock | 12 +- .../examples/lookahead/poetry.lock | 12 +- security_scanning/examples/medusa/poetry.lock | 12 +- .../models/contrib/baichuan/poetry.lock | 12 +- .../examples/models/contrib/bloom/poetry.lock | 12 +- .../models/contrib/chatglm-6b/poetry.lock | 12 +- .../models/contrib/chatglm2-6b/poetry.lock | 12 +- .../contrib/chatglm3-6b-32k/poetry.lock | 12 +- .../examples/models/contrib/dbrx/poetry.lock | 12 +- .../models/contrib/deepseek_v1/poetry.lock | 12 +- .../models/contrib/deepseek_v2/poetry.lock | 12 +- .../models/contrib/falcon/poetry.lock | 12 +- .../examples/models/contrib/gptj/poetry.lock | 12 +- .../models/contrib/gptneox/poetry.lock | 12 +- .../examples/models/contrib/grok/poetry.lock | 12 +- .../models/contrib/hyperclovax/poetry.lock | 12 +- .../models/contrib/internlm/poetry.lock | 12 +- .../examples/models/contrib/jais/poetry.lock | 12 +- .../examples/models/contrib/mmdit/poetry.lock | 12 +- .../examples/models/contrib/mpt/poetry.lock | 12 +- .../examples/models/contrib/opt/poetry.lock | 12 +- .../models/contrib/skywork/poetry.lock | 12 +- .../examples/models/contrib/smaug/poetry.lock | 12 +- .../examples/models/contrib/stdit/poetry.lock | 12 +- .../examples/models/core/commandr/poetry.lock | 12 +- .../examples/models/core/gemma/poetry.lock | 12 +- .../examples/models/core/glm-4-9b/poetry.lock | 12 +- .../examples/models/core/gpt/poetry.lock | 12 +- .../examples/models/core/llama/poetry.lock | 12 +- .../examples/models/core/mamba/poetry.lock | 12 +- .../examples/models/core/mixtral/poetry.lock | 6 +- .../examples/models/core/mllama/poetry.lock | 6 +- .../examples/models/core/nemotron/poetry.lock | 12 +- .../examples/models/core/phi/poetry.lock | 12 +- .../examples/models/core/qwen/poetry.lock | 50 ++-- .../models/core/qwen2audio/poetry.lock | 12 +- .../examples/models/core/qwenvl/poetry.lock | 12 +- .../models/core/recurrentgemma/poetry.lock | 12 +- .../examples/models/core/whisper/poetry.lock | 18 +- security_scanning/examples/ngram/poetry.lock | 12 +- .../examples/quantization/poetry.lock | 12 +- .../examples/ray_orchestrator/poetry.lock | 160 +++++++++++- .../examples/redrafter/poetry.lock | 12 +- security_scanning/examples/serve/poetry.lock | 24 +- .../examples/trtllm-eval/poetry.lock | 12 +- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 229 ++++++++++++++---- security_scanning/triton_backend/poetry.lock | 12 +- 53 files changed, 653 insertions(+), 372 deletions(-) diff --git a/security_scanning/docs/poetry.lock b/security_scanning/docs/poetry.lock index ac828c7555b2..997d590c1232 100644 --- a/security_scanning/docs/poetry.lock +++ b/security_scanning/docs/poetry.lock @@ -318,14 +318,14 @@ files = [ [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/apps/poetry.lock b/security_scanning/examples/apps/poetry.lock index e4da6d6cc1c4..793fdbb76a67 100644 --- a/security_scanning/examples/apps/poetry.lock +++ b/security_scanning/examples/apps/poetry.lock @@ -148,14 +148,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 2f60d0996032..47c0c73a4922 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -1137,14 +1137,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1174,14 +1174,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/draft_target_model/poetry.lock b/security_scanning/examples/draft_target_model/poetry.lock index 181c780b1fff..368d809f4267 100644 --- a/security_scanning/examples/draft_target_model/poetry.lock +++ b/security_scanning/examples/draft_target_model/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/eagle/poetry.lock b/security_scanning/examples/eagle/poetry.lock index bed68fee3f9a..73d25b03ea36 100644 --- a/security_scanning/examples/eagle/poetry.lock +++ b/security_scanning/examples/eagle/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index f09c1a263393..8592cbac562b 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -992,14 +992,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1029,14 +1029,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/lookahead/poetry.lock b/security_scanning/examples/lookahead/poetry.lock index 181c780b1fff..368d809f4267 100644 --- a/security_scanning/examples/lookahead/poetry.lock +++ b/security_scanning/examples/lookahead/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/medusa/poetry.lock b/security_scanning/examples/medusa/poetry.lock index 181c780b1fff..368d809f4267 100644 --- a/security_scanning/examples/medusa/poetry.lock +++ b/security_scanning/examples/medusa/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/baichuan/poetry.lock b/security_scanning/examples/models/contrib/baichuan/poetry.lock index 697bdb39fdeb..03c1f0e7ce8e 100644 --- a/security_scanning/examples/models/contrib/baichuan/poetry.lock +++ b/security_scanning/examples/models/contrib/baichuan/poetry.lock @@ -839,14 +839,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -876,14 +876,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/bloom/poetry.lock b/security_scanning/examples/models/contrib/bloom/poetry.lock index d44bb3c7605b..dc682784cc28 100644 --- a/security_scanning/examples/models/contrib/bloom/poetry.lock +++ b/security_scanning/examples/models/contrib/bloom/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock index c32a0fbf6d9b..eaaf906664b5 100644 --- a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock index c32a0fbf6d9b..eaaf906664b5 100644 --- a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock index c32a0fbf6d9b..eaaf906664b5 100644 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/dbrx/poetry.lock b/security_scanning/examples/models/contrib/dbrx/poetry.lock index c5ed9ef17aeb..e8979a0c7455 100644 --- a/security_scanning/examples/models/contrib/dbrx/poetry.lock +++ b/security_scanning/examples/models/contrib/dbrx/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock index f878729b3a47..83b7157abd35 100644 --- a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/falcon/poetry.lock b/security_scanning/examples/models/contrib/falcon/poetry.lock index d5fe9085af98..804f96084df6 100644 --- a/security_scanning/examples/models/contrib/falcon/poetry.lock +++ b/security_scanning/examples/models/contrib/falcon/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/gptj/poetry.lock b/security_scanning/examples/models/contrib/gptj/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/contrib/gptj/poetry.lock +++ b/security_scanning/examples/models/contrib/gptj/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/gptneox/poetry.lock b/security_scanning/examples/models/contrib/gptneox/poetry.lock index bed68fee3f9a..73d25b03ea36 100644 --- a/security_scanning/examples/models/contrib/gptneox/poetry.lock +++ b/security_scanning/examples/models/contrib/gptneox/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/grok/poetry.lock b/security_scanning/examples/models/contrib/grok/poetry.lock index 087b1c6d8610..bd28544ad248 100644 --- a/security_scanning/examples/models/contrib/grok/poetry.lock +++ b/security_scanning/examples/models/contrib/grok/poetry.lock @@ -981,14 +981,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1033,14 +1033,14 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index 393281306e37..5308046106bd 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -394,14 +394,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -431,14 +431,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/internlm/poetry.lock b/security_scanning/examples/models/contrib/internlm/poetry.lock index 181c780b1fff..368d809f4267 100644 --- a/security_scanning/examples/models/contrib/internlm/poetry.lock +++ b/security_scanning/examples/models/contrib/internlm/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/jais/poetry.lock b/security_scanning/examples/models/contrib/jais/poetry.lock index d44bb3c7605b..dc682784cc28 100644 --- a/security_scanning/examples/models/contrib/jais/poetry.lock +++ b/security_scanning/examples/models/contrib/jais/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/mmdit/poetry.lock b/security_scanning/examples/models/contrib/mmdit/poetry.lock index dfbe1bd0fcfb..cdaa56ecf86e 100644 --- a/security_scanning/examples/models/contrib/mmdit/poetry.lock +++ b/security_scanning/examples/models/contrib/mmdit/poetry.lock @@ -420,14 +420,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -457,14 +457,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/mpt/poetry.lock b/security_scanning/examples/models/contrib/mpt/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/contrib/mpt/poetry.lock +++ b/security_scanning/examples/models/contrib/mpt/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/opt/poetry.lock b/security_scanning/examples/models/contrib/opt/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/contrib/opt/poetry.lock +++ b/security_scanning/examples/models/contrib/opt/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/skywork/poetry.lock b/security_scanning/examples/models/contrib/skywork/poetry.lock index d44bb3c7605b..dc682784cc28 100644 --- a/security_scanning/examples/models/contrib/skywork/poetry.lock +++ b/security_scanning/examples/models/contrib/skywork/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/smaug/poetry.lock b/security_scanning/examples/models/contrib/smaug/poetry.lock index d44bb3c7605b..dc682784cc28 100644 --- a/security_scanning/examples/models/contrib/smaug/poetry.lock +++ b/security_scanning/examples/models/contrib/smaug/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/contrib/stdit/poetry.lock b/security_scanning/examples/models/contrib/stdit/poetry.lock index de50ba70e20e..7104f94def36 100644 --- a/security_scanning/examples/models/contrib/stdit/poetry.lock +++ b/security_scanning/examples/models/contrib/stdit/poetry.lock @@ -944,14 +944,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -981,14 +981,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/commandr/poetry.lock b/security_scanning/examples/models/core/commandr/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/core/commandr/poetry.lock +++ b/security_scanning/examples/models/core/commandr/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/gemma/poetry.lock b/security_scanning/examples/models/core/gemma/poetry.lock index d40358cfdd00..bdd43e9bee06 100644 --- a/security_scanning/examples/models/core/gemma/poetry.lock +++ b/security_scanning/examples/models/core/gemma/poetry.lock @@ -951,14 +951,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1003,14 +1003,14 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/glm-4-9b/poetry.lock b/security_scanning/examples/models/core/glm-4-9b/poetry.lock index c32a0fbf6d9b..eaaf906664b5 100644 --- a/security_scanning/examples/models/core/glm-4-9b/poetry.lock +++ b/security_scanning/examples/models/core/glm-4-9b/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/gpt/poetry.lock b/security_scanning/examples/models/core/gpt/poetry.lock index d44bb3c7605b..dc682784cc28 100644 --- a/security_scanning/examples/models/core/gpt/poetry.lock +++ b/security_scanning/examples/models/core/gpt/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/llama/poetry.lock b/security_scanning/examples/models/core/llama/poetry.lock index 5c12350d21cb..a0bd689a9c87 100644 --- a/security_scanning/examples/models/core/llama/poetry.lock +++ b/security_scanning/examples/models/core/llama/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/mamba/poetry.lock b/security_scanning/examples/models/core/mamba/poetry.lock index 67392b7a282e..7feb4e0f77e1 100644 --- a/security_scanning/examples/models/core/mamba/poetry.lock +++ b/security_scanning/examples/models/core/mamba/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock index dbe8a56f13ba..760f2f82a92b 100644 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ b/security_scanning/examples/models/core/mixtral/poetry.lock @@ -426,14 +426,14 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/mllama/poetry.lock b/security_scanning/examples/models/core/mllama/poetry.lock index 791e5ef97b6e..f90323bf02f5 100644 --- a/security_scanning/examples/models/core/mllama/poetry.lock +++ b/security_scanning/examples/models/core/mllama/poetry.lock @@ -419,14 +419,14 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/nemotron/poetry.lock b/security_scanning/examples/models/core/nemotron/poetry.lock index 4bc5cf76045c..297d2dac7d24 100644 --- a/security_scanning/examples/models/core/nemotron/poetry.lock +++ b/security_scanning/examples/models/core/nemotron/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/phi/poetry.lock b/security_scanning/examples/models/core/phi/poetry.lock index bb3b6bc9b6d9..dce1be46e2fd 100644 --- a/security_scanning/examples/models/core/phi/poetry.lock +++ b/security_scanning/examples/models/core/phi/poetry.lock @@ -840,14 +840,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -877,14 +877,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/qwen/poetry.lock b/security_scanning/examples/models/core/qwen/poetry.lock index db818eb8ca56..8078881ef87d 100644 --- a/security_scanning/examples/models/core/qwen/poetry.lock +++ b/security_scanning/examples/models/core/qwen/poetry.lock @@ -996,14 +996,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1033,14 +1033,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] @@ -2732,31 +2732,31 @@ six = ">=1.14.0" [[package]] name = "ruff" -version = "0.15.14" +version = "0.15.15" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["main"] markers = "sys_platform != \"emscripten\"" files = [ - {file = "ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108"}, - {file = "ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b"}, - {file = "ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba"}, - {file = "ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f"}, - {file = "ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581"}, - {file = "ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93"}, - {file = "ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61"}, - {file = "ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553"}, - {file = "ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6"}, - {file = "ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902"}, - {file = "ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826"}, - {file = "ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f"}, + {file = "ruff-0.15.15-py3-none-linux_armv6l.whl", hash = "sha256:cf93e5388f412e1b108b1f8b34a6e036b70fe8aff89393befad96fe48670311b"}, + {file = "ruff-0.15.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac5a646d1f6a7dadd5d50842dae2c1f9862ac887ef5d1b1375e02def791fde6e"}, + {file = "ruff-0.15.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:77d955a431430c66f72dd94e379ad38a16daea3d25094872ac4edf9e797be530"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7614ee79c69788cf6cedd568069ade9cecc22a1ad20494efe8d0c9ebb4b622d4"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cdb1679e06a1f6b47bc384714ae96f6e2fb65ca441eb78c43d2ca554176ce1f"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2728b93d7b23a603ea2c0ac6eb73d760bd38ec9de35f35fb41e18f7a3fee7622"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be582fcc0db438902c7792b08d6ddf6c9b9e21addaa10092c2c741cfb09e5a45"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7aa77465b8ecaf1a27bea098d696f7fed5e1eccbd10b321b682d6de586ae5627"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48decfa11d740de4889de623be1463308346312f2409a56e24aa280c86162dc4"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a5015088452ca0081387063649ec67f06d3d1d6b8b936a1f836b5e9657ecd48c"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5294aab6356c81600fcdea3a62bb1b924dfd5e91767c12318d3f68f86af57cd"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:db5bd4d802415cca656dc1616070b725952d6ae95eb5d4831e49fbd94a38f75f"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:587a6278ed42059191c1a466e490bd7930fb50bd2e255398bc29616c895a61cb"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:df0c1c084f5f4be9812f61518a45c440d3c30d69ce4bf6c5270e66d38338f02a"}, + {file = "ruff-0.15.15-py3-none-win32.whl", hash = "sha256:29428ea79694afbe756d45fd59b36f22b6b020dc0443cf7de0173046236964b9"}, + {file = "ruff-0.15.15-py3-none-win_amd64.whl", hash = "sha256:8df0323902e15e24bc4bf246da830573d3cf3352bd0b9a164eab335d111ff4a4"}, + {file = "ruff-0.15.15-py3-none-win_arm64.whl", hash = "sha256:3c8ceca6792f38196b8f589bc92eccd03eef286602da92e5dc05cc42ef6441b7"}, + {file = "ruff-0.15.15.tar.gz", hash = "sha256:b8dff018130b46d8e5bf0f926ef6b60cf871d6d5ae45fc9334e09632daa741d6"}, ] [[package]] diff --git a/security_scanning/examples/models/core/qwen2audio/poetry.lock b/security_scanning/examples/models/core/qwen2audio/poetry.lock index e5295d68ae75..f3132bb8b010 100644 --- a/security_scanning/examples/models/core/qwen2audio/poetry.lock +++ b/security_scanning/examples/models/core/qwen2audio/poetry.lock @@ -840,14 +840,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -877,14 +877,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 9fb5b37651e4..8ee0142f486a 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -1199,14 +1199,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1236,14 +1236,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/recurrentgemma/poetry.lock b/security_scanning/examples/models/core/recurrentgemma/poetry.lock index 32685dc86b4f..58cb2b6ffe4b 100644 --- a/security_scanning/examples/models/core/recurrentgemma/poetry.lock +++ b/security_scanning/examples/models/core/recurrentgemma/poetry.lock @@ -971,14 +971,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1008,14 +1008,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/models/core/whisper/poetry.lock b/security_scanning/examples/models/core/whisper/poetry.lock index 5ffb3a92555d..f8da1fe1653a 100644 --- a/security_scanning/examples/models/core/whisper/poetry.lock +++ b/security_scanning/examples/models/core/whisper/poetry.lock @@ -1005,14 +1005,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1042,14 +1042,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] @@ -2320,14 +2320,14 @@ xml = ["lxml (>=5.3.0)"] [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917"}, - {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] diff --git a/security_scanning/examples/ngram/poetry.lock b/security_scanning/examples/ngram/poetry.lock index e35d521f0dd4..7b20c0a00de0 100644 --- a/security_scanning/examples/ngram/poetry.lock +++ b/security_scanning/examples/ngram/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/quantization/poetry.lock b/security_scanning/examples/quantization/poetry.lock index 75f9f276fff5..a513fa11f384 100644 --- a/security_scanning/examples/quantization/poetry.lock +++ b/security_scanning/examples/quantization/poetry.lock @@ -792,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -829,14 +829,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 0bb993d8e616..cfd8cf12a2ab 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -893,14 +893,14 @@ protobuf = ["grpcio-tools (>=1.80.0)"] [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] @@ -1300,14 +1300,14 @@ files = [ [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917"}, - {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] @@ -1934,6 +1934,7 @@ description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.10" groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, @@ -2052,6 +2053,147 @@ files = [ {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] +[[package]] +name = "rpds-py" +version = "2026.5.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version == \"3.12\"" +files = [ + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3397a5ed7174dc2786bb214030232fc36fe8e5584fec43a9952cc542b1a12036"}, + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:99ab6ba7bfa2cb0f96a04e3652355bf04e3f51aceb1e943b8541dab7ba4828cc"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0efbe45632665e53e3db8fe1e5692db58fc5cb9bab4459d570b83efefe11164"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01d17b29c0c23d82b1f4751147ec49cf451f1fc2554eb9ef5f957e55d2656ead"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7559f72b94ae52659086c595dfa017cde03155f7832071d30959049052cb3ece"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e25b7088f9ccbfc0dfcaa52bf969300ca229e10ecf758974ebcbb080a4b37bb"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613fc4ee9eaef26dc5840666214dd6fbcebcf32f46e76f4abc473059f4e13dda"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:85264a90ff4c05c1568dd65f5921c837614b67c60358fb4c17df3b7f2e90690a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe71bca7d547acb17027c7fd1624ff8aae623499c498d3e7011182c4de5c25e0"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05fa4f41f37ec97c9c260441a940450a192f78d774d2b097eee1379f1e1246a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df1d2a1996755b24b9ecee92cb4d36c28f86f464a6a173349c26bab41e94b8c2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8895840ac4809e5f60c88fd07617cd71326e73d6e5a8aa783c5c0f7c24985de2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win32.whl", hash = "sha256:3684a59b158a7683aaeb8e25352e9a9dd2122cec78f2d8530266e4f91b4c7b3f"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:7bd530e6a530bb3ea892f194fafa455f3516ac25ecf7143fd33c09be62b0470a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:0a5ae4dbe43c1076983b72616496919872ae7bbe7a1e21cc48336bc3154d130b"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:edf2765d84e42447f112ad877af8fe1db0089aaec5b28e88d6eab45e7fe99cea"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad3773236e95f7f33991eb125224b7da66f206504d032a253a02da7e134519fb"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a04df86b3f0fade39ec8fd0e0aab089b1da9fbd2b48df778a57ef96f5e7d38df"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6142dbd80c4df62a5d899f0d616d417f84e0bc8d32526c8e5589019d75d028a7"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b35217adefe87f2fe4db7e9766cabe84744bfe9616d9667be18988928c7f2dc"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b95d5e11fc712b752081183a55a244c03cd00570489edd7014d8899f8ceb8162"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141c9498daf2ace9eda35d2b0e376f9ea8b058d84f2aef4f96fccfd449a2f251"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:6f249f8b860a200ad35193af961183ebe9132710484e6f6ce0cf89fd83c63a9a"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4abbf391a70be864920858bf360f4fb380577c9a0f732438a1996726e2c195b"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c74005a7bb87752acf351c93897ec63ad77a07a0da7ecad9c050e32e7286ba34"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:8213afbe8a3a906fb9acb2014423fe3359ee783d0bf90995f70623a3217bfa6c"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049"}, + {file = "rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256"}, +] + [[package]] name = "six" version = "1.17.0" @@ -2137,14 +2279,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.4.0" +version = "21.4.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "virtualenv-21.4.0-py3-none-any.whl", hash = "sha256:334fb2e774003e53a889c4808c5392114020813faf8a6c291d0d97dd3cb26b7a"}, - {file = "virtualenv-21.4.0.tar.gz", hash = "sha256:8d401ab6ee040a7c679f2d3947faa9ed07edf284c0a4e7bda9edd2f34b1fe500"}, + {file = "virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12"}, + {file = "virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2"}, ] [package.dependencies] diff --git a/security_scanning/examples/redrafter/poetry.lock b/security_scanning/examples/redrafter/poetry.lock index 181c780b1fff..368d809f4267 100644 --- a/security_scanning/examples/redrafter/poetry.lock +++ b/security_scanning/examples/redrafter/poetry.lock @@ -828,14 +828,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -865,14 +865,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index fc859e5055c6..49dcf063e28d 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -1664,14 +1664,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1701,14 +1701,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] @@ -2964,14 +2964,14 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917"}, - {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] @@ -4620,14 +4620,14 @@ numpy = "*" [[package]] name = "starlette" -version = "1.1.0" +version = "1.2.0" description = "The little ASGI library that shines." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382"}, - {file = "starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f"}, + {file = "starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7"}, + {file = "starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553"}, ] [package.dependencies] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index ecf22f55a4b4..e259f4424228 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -992,14 +992,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1029,14 +1029,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 8b7acebaa41e..0e9006bba4c3 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "fd5fa6169d5e9313d9cc281038d25de769037462", - "timestamp": "2026-05-28T02:47:45Z" + "commit_hash": "fed47f1fb10667cb7b90f167574e79a8574059eb", + "timestamp": "2026-05-29T02:46:33Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index e8dc15033ea2..85b3bb692288 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -2138,14 +2138,14 @@ files = [ [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -2175,14 +2175,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] @@ -3985,51 +3985,48 @@ PyYAML = ">=5.1.0" [[package]] name = "onnx" -version = "1.19.1" +version = "1.21.0" description = "Open Neural Network Exchange" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "onnx-1.19.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7343250cc5276cf439fe623b8f92e11cf0d1eebc733ae4a8b2e86903bb72ae68"}, - {file = "onnx-1.19.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1fb8f79de7f3920bb82b537f3c6ac70c0ce59f600471d9c3eed2b5f8b079b748"}, - {file = "onnx-1.19.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:92b9d2dece41cc84213dbbfd1acbc2a28c27108c53bd28ddb6d1043fbfcbd2d5"}, - {file = "onnx-1.19.1-cp310-cp310-win32.whl", hash = "sha256:c0b1a2b6bb19a0fc9f5de7661a547136d082c03c169a5215e18ff3ececd2a82f"}, - {file = "onnx-1.19.1-cp310-cp310-win_amd64.whl", hash = "sha256:1c0498c00db05fcdb3426697d330dcecc3f60020015065e2c76fa795f2c9a605"}, - {file = "onnx-1.19.1-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:17aaf5832126de0a5197a5864e4f09a764dd7681d3035135547959b4b6b77a09"}, - {file = "onnx-1.19.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01b292a4d0b197c45d8184545bbc8ae1df83466341b604187c1b05902cb9c920"}, - {file = "onnx-1.19.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1839af08ab4a909e4af936b8149c27f8c64b96138981024e251906e0539d8bf9"}, - {file = "onnx-1.19.1-cp311-cp311-win32.whl", hash = "sha256:0bdbb676e3722bd32f9227c465d552689f49086f986a696419d865cb4e70b989"}, - {file = "onnx-1.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:1346853df5c1e3ebedb2e794cf2a51e0f33759affd655524864ccbcddad7035b"}, - {file = "onnx-1.19.1-cp311-cp311-win_arm64.whl", hash = "sha256:2d69c280c0e665b7f923f499243b9bb84fe97970b7a4668afa0032045de602c8"}, - {file = "onnx-1.19.1-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:3612193a89ddbce5c4e86150869b9258780a82fb8c4ca197723a4460178a6ce9"}, - {file = "onnx-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c2fd2f744e7a3880ad0c262efa2edf6d965d0bd02b8f327ec516ad4cb0f2f15"}, - {file = "onnx-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:485d3674d50d789e0ee72fa6f6e174ab81cb14c772d594f992141bd744729d8a"}, - {file = "onnx-1.19.1-cp312-cp312-win32.whl", hash = "sha256:638bc56ff1a5718f7441e887aeb4e450f37a81c6eac482040381b140bd9ba601"}, - {file = "onnx-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:bc7e2e4e163e679721e547958b5a7db875bf822cad371b7c1304aa4401a7c7a4"}, - {file = "onnx-1.19.1-cp312-cp312-win_arm64.whl", hash = "sha256:17c215b1c0f20fe93b4cbe62668247c1d2294b9bc7f6be0ca9ced28e980c07b7"}, - {file = "onnx-1.19.1-cp313-cp313-macosx_12_0_universal2.whl", hash = "sha256:4e5f938c68c4dffd3e19e4fd76eb98d298174eb5ebc09319cdd0ec5fe50050dc"}, - {file = "onnx-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:86e20a5984b017feeef2dbf4ceff1c7c161ab9423254968dd77d3696c38691d0"}, - {file = "onnx-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d9c467f0f29993c12f330736af87972f30adb8329b515f39d63a0db929cb2c"}, - {file = "onnx-1.19.1-cp313-cp313-win32.whl", hash = "sha256:65eee353a51b4e4ca3e797784661e5376e2b209f17557e04921eac9166a8752e"}, - {file = "onnx-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:c3bc87e38b53554b1fc9ef7b275c81c6f5c93c90a91935bb0aa8d4d498a6d48e"}, - {file = "onnx-1.19.1-cp313-cp313-win_arm64.whl", hash = "sha256:e41496f400afb980ec643d80d5164753a88a85234fa5c06afdeebc8b7d1ec252"}, - {file = "onnx-1.19.1-cp313-cp313t-macosx_12_0_universal2.whl", hash = "sha256:5f6274abf0fd74e80e78ecbb44bd44509409634525c89a9b38276c8af47dc0a2"}, - {file = "onnx-1.19.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:07dcd4d83584eb4bf8f21ac04c82643712e5e93ac2a0ed10121ec123cb127e1e"}, - {file = "onnx-1.19.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1975860c3e720db25d37f1619976582828264bdcc64fa7511c321ac4fc01add3"}, - {file = "onnx-1.19.1-cp313-cp313t-win_amd64.whl", hash = "sha256:9807d0e181f6070ee3a6276166acdc571575d1bd522fc7e89dba16fd6e7ffed9"}, - {file = "onnx-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6ee83e6929d75005482d9f304c502ac7c9b8d6db153aa6b484dae74d0f28570"}, - {file = "onnx-1.19.1-cp39-cp39-macosx_12_0_universal2.whl", hash = "sha256:2980de39df1f5afd005a8aeb0b35703dbbab8e4012bcec1634febbdfb8654da8"}, - {file = "onnx-1.19.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bf35f7abc7096df2bb0171102fa7d89ba4a5f5407e3b352ee27bb5e1867e0f19"}, - {file = "onnx-1.19.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc81f200ed98bd0ced53c3f0fdb8164a42e2b8582a1fa9cb8aeb01b64367c7f4"}, - {file = "onnx-1.19.1-cp39-cp39-win32.whl", hash = "sha256:a2e51118c3db00b169cac8170d94d832c2ffe80935563ced596182d4baa6fcb4"}, - {file = "onnx-1.19.1-cp39-cp39-win_amd64.whl", hash = "sha256:4650d053c7c26e40a080b7378d61446958d6da4e217e1d0d422eb9264f8064ae"}, - {file = "onnx-1.19.1.tar.gz", hash = "sha256:737524d6eb3907d3499ea459c6f01c5a96278bb3a0f2ff8ae04786fb5d7f1ed5"}, + {file = "onnx-1.21.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:e0c21cc5c7a41d1a509828e2b14fe9c30e807c6df611ec0fd64a47b8d4b16abd"}, + {file = "onnx-1.21.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1931bfcc222a4c9da6475f2ffffb84b97ab3876041ec639171c11ce802bee6a"}, + {file = "onnx-1.21.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9b56ad04039fac6b028c07e54afa1ec7f75dd340f65311f2c292e41ed7aa4d9"}, + {file = "onnx-1.21.0-cp310-cp310-win32.whl", hash = "sha256:3abd09872523c7e0362d767e4e63bd7c6bac52a5e2c3edbf061061fe540e2027"}, + {file = "onnx-1.21.0-cp310-cp310-win_amd64.whl", hash = "sha256:f2c7c234c568402e10db74e33d787e4144e394ae2bcbbf11000fbfe2e017ad68"}, + {file = "onnx-1.21.0-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:2aca19949260875c14866fc77ea0bc37e4e809b24976108762843d328c92d3ce"}, + {file = "onnx-1.21.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82aa6ab51144df07c58c4850cb78d4f1ae969d8c0bf657b28041796d49ba6974"}, + {file = "onnx-1.21.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c3185a232089335581fabb98fba4e86d3e8246b8140f2e406082438100ebda"}, + {file = "onnx-1.21.0-cp311-cp311-win32.whl", hash = "sha256:f53b3c15a3b539c16b99655c43c365622046d68c49b680c48eba4da2a4fb6f27"}, + {file = "onnx-1.21.0-cp311-cp311-win_amd64.whl", hash = "sha256:5f78c411743db317a76e5d009f84f7e3d5380411a1567a868e82461a1e5c775d"}, + {file = "onnx-1.21.0-cp311-cp311-win_arm64.whl", hash = "sha256:ab6a488dabbb172eebc9f3b3e7ac68763f32b0c571626d4a5004608f866cc83d"}, + {file = "onnx-1.21.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:fc2635400fe39ff37ebc4e75342cc54450eadadf39c540ff132c319bf4960095"}, + {file = "onnx-1.21.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9003d5206c01fa2ff4b46311566865d8e493e1a6998d4009ec6de39843f1b59b"}, + {file = "onnx-1.21.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9261bd580fb8548c9c37b3c6750387eb8f21ea43c63880d37b2c622e1684285"}, + {file = "onnx-1.21.0-cp312-abi3-win32.whl", hash = "sha256:9ea4e824964082811938a9250451d89c4ec474fe42dd36c038bfa5df31993d1e"}, + {file = "onnx-1.21.0-cp312-abi3-win_amd64.whl", hash = "sha256:458d91948ad9a7729a347550553b49ab6939f9af2cddf334e2116e45467dc61f"}, + {file = "onnx-1.21.0-cp312-abi3-win_arm64.whl", hash = "sha256:ca14bc4842fccc3187eb538f07eabeb25a779b39388b006db4356c07403a7bbb"}, + {file = "onnx-1.21.0-cp313-cp313t-macosx_12_0_universal2.whl", hash = "sha256:257d1d1deb6a652913698f1e3f33ef1ca0aa69174892fe38946d4572d89dd94f"}, + {file = "onnx-1.21.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7cd7cb8f6459311bdb557cbf6c0ccc6d8ace11c304d1bba0a30b4a4688e245f8"}, + {file = "onnx-1.21.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b58a4cfec8d9311b73dc083e4c1fa362069267881144c05139b3eba5dc3a840"}, + {file = "onnx-1.21.0-cp313-cp313t-win_amd64.whl", hash = "sha256:1a9baf882562c4cebf79589bebb7cd71a20e30b51158cac3e3bbaf27da6163bd"}, + {file = "onnx-1.21.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bba12181566acf49b35875838eba49536a327b2944664b17125577d230c637ad"}, + {file = "onnx-1.21.0-cp314-cp314t-macosx_12_0_universal2.whl", hash = "sha256:7ee9d8fd6a4874a5fa8b44bbcabea104ce752b20469b88bc50c7dcf9030779ad"}, + {file = "onnx-1.21.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5489f25fe461e7f32128218251a466cabbeeaf1eaa791c79daebf1a80d5a2cc9"}, + {file = "onnx-1.21.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db17fc0fec46180b6acbd1d5d8650a04e5527c02b09381da0b5b888d02a204c8"}, + {file = "onnx-1.21.0-cp314-cp314t-win_amd64.whl", hash = "sha256:19d9971a3e52a12968ae6c70fd0f86c349536de0b0c33922ecdbe52d1972fe60"}, + {file = "onnx-1.21.0-cp314-cp314t-win_arm64.whl", hash = "sha256:efba467efb316baf2a9452d892c2f982b9b758c778d23e38c7f44fa211b30bb9"}, + {file = "onnx-1.21.0.tar.gz", hash = "sha256:4d8b67d0aaec5864c87633188b91cc520877477ec0254eda122bef8be43cd764"}, ] [package.dependencies] -ml_dtypes = ">=0.5.0" -numpy = ">=1.22" +ml_dtypes = [ + {version = ">=0.5.0", markers = "platform_machine != \"s390x\""}, + {version = ">=0.5.4", markers = "platform_machine == \"s390x\""}, +] +numpy = ">=1.23.2" protobuf = ">=4.25.1" typing_extensions = ">=4.7.1" @@ -5727,6 +5724,7 @@ description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.10" groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, @@ -5845,6 +5843,147 @@ files = [ {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] +[[package]] +name = "rpds-py" +version = "2026.5.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3397a5ed7174dc2786bb214030232fc36fe8e5584fec43a9952cc542b1a12036"}, + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:99ab6ba7bfa2cb0f96a04e3652355bf04e3f51aceb1e943b8541dab7ba4828cc"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0efbe45632665e53e3db8fe1e5692db58fc5cb9bab4459d570b83efefe11164"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01d17b29c0c23d82b1f4751147ec49cf451f1fc2554eb9ef5f957e55d2656ead"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7559f72b94ae52659086c595dfa017cde03155f7832071d30959049052cb3ece"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e25b7088f9ccbfc0dfcaa52bf969300ca229e10ecf758974ebcbb080a4b37bb"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613fc4ee9eaef26dc5840666214dd6fbcebcf32f46e76f4abc473059f4e13dda"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:85264a90ff4c05c1568dd65f5921c837614b67c60358fb4c17df3b7f2e90690a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe71bca7d547acb17027c7fd1624ff8aae623499c498d3e7011182c4de5c25e0"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05fa4f41f37ec97c9c260441a940450a192f78d774d2b097eee1379f1e1246a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df1d2a1996755b24b9ecee92cb4d36c28f86f464a6a173349c26bab41e94b8c2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8895840ac4809e5f60c88fd07617cd71326e73d6e5a8aa783c5c0f7c24985de2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win32.whl", hash = "sha256:3684a59b158a7683aaeb8e25352e9a9dd2122cec78f2d8530266e4f91b4c7b3f"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:7bd530e6a530bb3ea892f194fafa455f3516ac25ecf7143fd33c09be62b0470a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:0a5ae4dbe43c1076983b72616496919872ae7bbe7a1e21cc48336bc3154d130b"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:edf2765d84e42447f112ad877af8fe1db0089aaec5b28e88d6eab45e7fe99cea"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad3773236e95f7f33991eb125224b7da66f206504d032a253a02da7e134519fb"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a04df86b3f0fade39ec8fd0e0aab089b1da9fbd2b48df778a57ef96f5e7d38df"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6142dbd80c4df62a5d899f0d616d417f84e0bc8d32526c8e5589019d75d028a7"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b35217adefe87f2fe4db7e9766cabe84744bfe9616d9667be18988928c7f2dc"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b95d5e11fc712b752081183a55a244c03cd00570489edd7014d8899f8ceb8162"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141c9498daf2ace9eda35d2b0e376f9ea8b058d84f2aef4f96fccfd449a2f251"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:6f249f8b860a200ad35193af961183ebe9132710484e6f6ce0cf89fd83c63a9a"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4abbf391a70be864920858bf360f4fb380577c9a0f732438a1996726e2c195b"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c74005a7bb87752acf351c93897ec63ad77a07a0da7ecad9c050e32e7286ba34"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:8213afbe8a3a906fb9acb2014423fe3359ee783d0bf90995f70623a3217bfa6c"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049"}, + {file = "rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256"}, +] + [[package]] name = "safetensors" version = "0.8.0rc0" @@ -7358,4 +7497,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "0fa80701f5dcc452033115bc5e53f4752151e971aeaee32a33900a5fae28b9ac" +content-hash = "25e38281b813618127f505b30af1ad9aa7f631945d858797c8870e5b81db432c" diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 32fdc737e81b..3cf7a8766832 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -1223,14 +1223,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.16.4" +version = "1.17.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7"}, - {file = "huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01"}, + {file = "huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c"}, + {file = "huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435"}, ] [package.dependencies] @@ -1260,14 +1260,14 @@ typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types [[package]] name = "idna" -version = "3.16" +version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, - {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, + {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, + {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, ] [package.extras] From 4826f6a4b52a328105e64596f46600f59f9d18fa Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 11:32:08 +0800 Subject: [PATCH 109/308] [https://nvbugs/6084447][fix] Fix MoE DeepGEMM workspace size with attention_dp (#13310) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: kris1025 <34030136+kris1025@users.noreply.github.com> --- .../_torch/modules/fused_moe/configurable_moe.py | 15 +++++++++------ .../_torch/modules/fused_moe/moe_scheduler.py | 8 ++++---- tests/integration/test_lists/waives.txt | 9 --------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index 52c1a62c1249..ea733543424a 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -409,17 +409,20 @@ def _get_quant_config_dict(self, model_config: ModelConfig) -> Optional[Dict]: else False, } + @staticmethod + def _dp_padded_num_rows(all_rank_num_tokens: List[int]) -> int: + """Padded total rows after DP dispatch: num_dp_ranks * max_tokens_per_rank.""" + return len(all_rank_num_tokens) * max(all_rank_num_tokens) + def calculate_num_chunks(self, all_rank_num_tokens: List[int]) -> int: """ - Calculate how many chunks are needed. + Calculate how many chunks are needed based on total tokens after dispatch. - Uses ep_size * max(all_rank_num_tokens) when A2A communication is active, - because the A2A recv buffer is shaped [ep_size, max_tokens_per_rank, hidden] - regardless of how tokens are distributed across ranks. This matches the - actual memory footprint of the MoE GEMM workspace. + When using DP communication, the dispatch (AllGather/AllToAll) collects + tokens from all DP ranks, so total tokens = num_dp_ranks * max_tokens_per_rank. """ if self.use_dp and self.comm is not None: - num_rows = self.mapping.moe_ep_size * max(all_rank_num_tokens) + num_rows = self._dp_padded_num_rows(all_rank_num_tokens) else: num_rows = sum(all_rank_num_tokens) return (num_rows + self.moe_max_num_tokens - 1) // self.moe_max_num_tokens diff --git a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py index f8bd16d10fd5..dd13a8a80905 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py +++ b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py @@ -208,12 +208,12 @@ def _prepare_workspace_deepgemm( num_rows = x.shape[0] if moe.use_dp and moe.comm is not None: # Communication path padding: dispatch outputs are - # ``[ep_size * max_tokens_per_rank, ...]`` (or expert-major for + # ``[num_dp_ranks * max_tokens_per_rank, ...]`` (or expert-major for # DeepEPLowLatency). Workspace must cover that footprint. if isinstance(moe.comm, DeepEPLowLatency): num_rows = moe.num_slots * max(all_rank_num_tokens) else: - num_rows = moe.mapping.moe_ep_size * max(all_rank_num_tokens) + num_rows = moe._dp_padded_num_rows(all_rank_num_tokens) workspaces = moe.backend.get_workspaces([num_rows]) return workspaces[0] @@ -240,7 +240,7 @@ def _prepare_workspaces_for_chunk( # Mirror ``_prepare_workspace_deepgemm``: DeepEPLowLatency dispatches # expert-major outputs sized ``num_slots * max_tokens_per_rank`` per # rank (one shard per slot), while other comms produce - # ``ep_size * max_tokens_per_rank``. Using the wrong formula + # ``num_dp_ranks * max_tokens_per_rank``. Using the wrong formula # under-allocates the workspace for DeepEPLowLatency multi-chunk # runs and is caught by ``DeepGemmFusedMoE.run_moe``. if moe.use_dp and all_rank_num_tokens_list[0] is not None: @@ -248,7 +248,7 @@ def _prepare_workspaces_for_chunk( if isinstance(moe.comm, DeepEPLowLatency): chunk_size_0 = moe.num_slots * max_tokens else: - chunk_size_0 = moe.mapping.moe_ep_size * max_tokens + chunk_size_0 = moe._dp_padded_num_rows(all_rank_num_tokens_list[0]) else: chunk_size_0 = chunk_size_list[0] workspace_chunk_sizes = [chunk_size_0] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 65067488cd38..dbf6089178a2 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -48,15 +48,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpu accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6112497) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162122) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=0] SKIP (https://nvbugs/6084447) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2] SKIP (https://nvbugs/6084447) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=0] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0] SKIP (https://nvbugs/6162122) From 018c432718471c90d8167d19177d9d35cda13762 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 12:24:42 +0800 Subject: [PATCH 110/308] [https://nvbugs/6189416][fix] Add a Blackwell-specific reference entry (extra_acc_spec=sm100_fp8, accuracy=46. (#14484) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tests/integration/defs/accuracy/references/mmmu.yaml | 7 +++++++ .../defs/accuracy/test_llm_api_pytorch_multimodal.py | 11 ++++++++++- tests/integration/test_lists/waives.txt | 1 - 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index fc23b4cbc6eb..883f6b63410a 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -3,6 +3,13 @@ google/gemma-3-27b-it: - quant_algo: FP8 kv_cache_quant_algo: FP8 accuracy: 50.0 + # Blackwell FP8 cubins for Gemma3 multimodal route through a different + # FlashInfer path than Hopper, producing ~5pt lower MMMU. Use a separate + # Blackwell-calibrated reference instead of relaxing the Hopper one. + - quant_algo: FP8 + kv_cache_quant_algo: FP8 + extra_acc_spec: sm100_fp8 + accuracy: 46.0 - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 48.0 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 0e36f38faa57..05e887667df9 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -20,6 +20,7 @@ from tensorrt_llm.quantization import QuantAlgo from ..conftest import ( + get_sm_version, llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell, @@ -313,9 +314,17 @@ def _make_llm(self, model_path: str): ) def test_fp8_prequantized(self): + # Blackwell FP8 numerics differ from Hopper at the cubin level + # (~5pt drop on MMMU). Route to a Blackwell-calibrated reference + # rather than relaxing the Hopper one. + extra_acc_spec = "sm100_fp8" if get_sm_version() >= 100 else None with self._make_llm(self.MODEL_PATH) as llm: task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) + task.evaluate( + llm, + extra_acc_spec=extra_acc_spec, + sampling_params=self.sampling_params, + ) @skip_pre_blackwell def test_nvfp4_prequantized(self): diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index dbf6089178a2..9389ea955044 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -129,7 +129,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_5_9B::test_bf16[mtp_on] SKIP (https: accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response SKIP (https://nvbugs/6200128) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6211189) -accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6189416) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6094070) From 566c226d005d7261dbaff91e920fd957409301ec Mon Sep 17 00:00:00 2001 From: Eran Geva <19514940+MrGeva@users.noreply.github.com> Date: Fri, 29 May 2026 07:41:47 +0300 Subject: [PATCH 111/308] [#14619][perf] AutoDeploy: tune Llama-3.1-8B-Instruct-FP8 TP=2/4 config and handle CG max bs when it is unset in the yaml (#14622) Signed-off-by: egeva <19514940+MrGeva@users.noreply.github.com> --- .../model_registry/configs/llama3_1_8b.yaml | 28 +++++++++++++----- tensorrt_llm/_torch/auto_deploy/llm_args.py | 29 +++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/examples/auto_deploy/model_registry/configs/llama3_1_8b.yaml b/examples/auto_deploy/model_registry/configs/llama3_1_8b.yaml index fab43ff61f46..71cf20ec3518 100644 --- a/examples/auto_deploy/model_registry/configs/llama3_1_8b.yaml +++ b/examples/auto_deploy/model_registry/configs/llama3_1_8b.yaml @@ -1,14 +1,25 @@ -compile_backend: torch-cudagraph enable_chunked_prefill: true -model_factory: AutoModelForCausalLM -runtime: trtllm -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.9 attn_backend: trtllm max_batch_size: 256 max_seq_len: 16384 +cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 192, 256] +kv_cache_config: + dtype: fp8 transforms: + detect_sharding: + allreduce_strategy: SYMM_MEM + sharding_source: ['manual'] + manual_config: + head_dim: 128 + tp_plan: + "q_proj": "colwise" + "k_proj": "colwise" + "v_proj": "colwise" + "o_proj": "rowwise" + "gate_proj": "colwise" + "up_proj": "colwise" + "down_proj": "rowwise" fuse_trtllm_attn_quant_fp8: enabled: true fuse_fp8_gemms: @@ -20,5 +31,8 @@ transforms: fuse_rope_into_trtllm_attention: enabled: true fuse_silu_mul: - enabled: true backend: trtllm + compile_model: + piecewise_enabled: true + mlir_elementwise_fusion: + enabled: true diff --git a/tensorrt_llm/_torch/auto_deploy/llm_args.py b/tensorrt_llm/_torch/auto_deploy/llm_args.py index 4634128e9949..b8b8d5892310 100644 --- a/tensorrt_llm/_torch/auto_deploy/llm_args.py +++ b/tensorrt_llm/_torch/auto_deploy/llm_args.py @@ -331,6 +331,35 @@ def update_transforms_with_shortcuts(self) -> Dict[str, Any]: return self + @model_validator(mode="after") + def extend_default_cuda_graph_config_to_max_batch_size(self): + """Auto-extend the default cuda_graph_config to cover the top-level max_batch_size. + + ``CudaGraphConfig.validate_cuda_graph_config`` falls back to a hard-coded + max of 128 when neither ``cuda_graph_config.batch_sizes`` nor + ``cuda_graph_config.max_batch_size`` is set. This silently leaves any + ``LlmArgs.max_batch_size > 128`` running in eager mode for the larger + batches, which roughly doubles ITL at those batch sizes. + + When the user hasn't explicitly set ``cuda_graph_config`` we rebuild it + with ``max_batch_size`` matching the top-level value so the heuristic + re-generates the batch_sizes list up to the actual max. + """ + # Skip if the user explicitly configured cuda_graph_config (any field set). + if "cuda_graph_config" in self.model_fields_set: + return self + cg = self.cuda_graph_config + if cg is None or cg.max_batch_size >= self.max_batch_size: + return self + # Re-validate a fresh CudaGraphConfig with the correct max_batch_size to + # trigger heuristic regeneration of batch_sizes. + cg_cls = type(cg) + self.cuda_graph_config = cg_cls( + max_batch_size=self.max_batch_size, + enable_padding=cg.enable_padding, + ) + return self + @model_validator(mode="after") def sync_cuda_graph_batch_sizes_to_compile_config(self): """Propagate cuda_graph_config.batch_sizes into compile_model transform config. From 4504fd724fc9e0f7f9539ca64f9b4eef437a13d8 Mon Sep 17 00:00:00 2001 From: Eran Geva <19514940+MrGeva@users.noreply.github.com> Date: Fri, 29 May 2026 08:00:07 +0300 Subject: [PATCH 112/308] [#13561][feat] AutoDeploy: enable MLIR elementwise fusion and trtllm_gen MoE on Nano NVFP4 (#14554) Signed-off-by: egeva <19514940+MrGeva@users.noreply.github.com> --- .../auto_deploy/model_registry/configs/nano_v3.yaml | 12 ++++-------- tensorrt_llm/_torch/auto_deploy/mlir/dialect.py | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/examples/auto_deploy/model_registry/configs/nano_v3.yaml b/examples/auto_deploy/model_registry/configs/nano_v3.yaml index 666df78f163a..f6ffcadf3661 100644 --- a/examples/auto_deploy/model_registry/configs/nano_v3.yaml +++ b/examples/auto_deploy/model_registry/configs/nano_v3.yaml @@ -1,18 +1,11 @@ -runtime: trtllm -compile_backend: torch-cudagraph max_batch_size: 384 max_seq_len: 65536 # tunable enable_chunked_prefill: true attn_backend: trtllm -model_factory: AutoModelForCausalLM -skip_loading_weights: false cuda_graph_config: batch_sizes: [1, 2, 4, 8, 16, 24, 32, 64, 128, 256, 320, 384] kv_cache_config: free_gpu_memory_fraction: 0.88 - # tunable mamba cache dtype - # --> use float32 for accuracy and default (auto) for speed - mamba_ssm_cache_dtype: auto transforms: detect_sharding: allreduce_strategy: SYMM_MEM @@ -42,7 +35,6 @@ transforms: "fc1_latent_proj": "gather" "fc2_latent_proj": "gather" multi_stream_moe: - stage: compile enabled: true gather_logits_before_lm_head: # TODO: fix https://github.com/NVIDIA/TensorRT-LLM/issues/9878 to enable by default @@ -54,3 +46,7 @@ transforms: backend: flashinfer_ssm compile_model: piecewise_enabled: true + fuse_nvfp4_moe: + backend: trtllm_gen + mlir_elementwise_fusion: + enabled: true diff --git a/tensorrt_llm/_torch/auto_deploy/mlir/dialect.py b/tensorrt_llm/_torch/auto_deploy/mlir/dialect.py index 968814cea8bd..fe85cff5a0a6 100644 --- a/tensorrt_llm/_torch/auto_deploy/mlir/dialect.py +++ b/tensorrt_llm/_torch/auto_deploy/mlir/dialect.py @@ -91,6 +91,7 @@ class Float8E5M2Type(ParametrizedAttribute, TypeAttribute): torch.float8_e4m3fn: Float8E4M3FNType, torch.float8_e5m2: Float8E5M2Type, torch.int8: lambda: IntegerType(8), + torch.uint8: lambda: IntegerType(8), # NVFP4 packs FP4 into uint8 buffers torch.int16: lambda: IntegerType(16), torch.int32: lambda: IntegerType(32), torch.int64: lambda: IntegerType(64), From 8c830c90f3fbdc2999aa3d1056dfbc7028f288fc Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 29 May 2026 13:12:15 +0800 Subject: [PATCH 113/308] [https://nvbugs/6221841][fix] Detect via the raw config_dict whether the user actually set a top-level rope_th (#14624) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../_torch/pyexecutor/config_utils.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index ef270040fc26..757b262a0c88 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -496,18 +496,26 @@ def load_pretrained_config(model_name_or_path: str, # needed — model_config.rope_theta (if any) is already canonical. rope_theta = rope_scaling.get("rope_theta") if rope_theta is not None: - existing = getattr(model_config, "rope_theta", None) - if existing is None: - model_config.rope_theta = rope_theta - elif existing != rope_theta: - # Both values are set but disagree. Keep the top-level value - # (canonical in transformers 4.x and 5.x), but warn loudly - # so that any future transformers upgrade that breaks this - # invariant is easy to spot. + # `rope_scaling.rope_theta` mirrors `rope_parameters.rope_theta` + # and is the canonical value in transformers 5.x. Adopt it as + # `model_config.rope_theta` so downstream code can read it. + # Note: `model_config.rope_theta` may be absent entirely + # (transformers 5.x dropped it from some configs) or may be a + # config-class default that does NOT reflect the user's + # config.json (e.g. DeepseekV3Config default 10000 vs. user's + # 1000000 — the original bug). + user_top_level = config_dict.get("rope_theta") + if user_top_level is not None and user_top_level != rope_theta: + # User explicitly set both top-level rope_theta and + # rope_parameters.rope_theta to disagreeing values — keep + # the top-level value but warn loudly. logger.warning( f"rope_scaling.rope_theta ({rope_theta}) differs from " - f"model_config.rope_theta ({existing}); keeping the " + f"config rope_theta ({user_top_level}); keeping the " f"top-level value.") + model_config.rope_theta = user_top_level + else: + model_config.rope_theta = rope_theta model_config.rope_scaling = None return model_config From 7bdd8357c136f98548ffcee79dbb7be31674db2b Mon Sep 17 00:00:00 2001 From: Aswin Visva <31215515+aswinvisva@users.noreply.github.com> Date: Thu, 28 May 2026 22:24:44 -0700 Subject: [PATCH 114/308] [None][perf] Replace Parakeet audio encoder with native trtllm layers (#14474) Signed-off-by: Aswin Visva <31215515+aswinvisva@users.noreply.github.com> --- .../_torch/models/modeling_parakeet.py | 469 ++++++++++++++++-- 1 file changed, 437 insertions(+), 32 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_parakeet.py b/tensorrt_llm/_torch/models/modeling_parakeet.py index d0e53d1dd7db..f687c05d887f 100644 --- a/tensorrt_llm/_torch/models/modeling_parakeet.py +++ b/tensorrt_llm/_torch/models/modeling_parakeet.py @@ -13,17 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math from functools import cache from typing import Dict, NamedTuple, Optional import numpy as np import torch import torch.nn as nn -from transformers import ParakeetEncoder as HFParakeetEncoder +import torch.nn.functional as F from transformers import ParakeetEncoderConfig, PretrainedConfig +from transformers.activations import ACT2FN from transformers.audio_utils import mel_filter_bank +from transformers.models.parakeet.modeling_parakeet import ( + ParakeetEncoderRelPositionalEncoding as HFParakeetEncoderRelPositionalEncoding, +) +from transformers.models.parakeet.modeling_parakeet import ( + ParakeetEncoderSubsamplingConv2D as HFParakeetEncoderSubsamplingConv2D, +) from ...logger import logger +from ..model_config import ModelConfig +from ..modules.attention import Attention +from ..modules.layer_norm import LayerNorm +from ..modules.linear import Linear from ..modules.rms_norm import RMSNorm EPSILON = 1e-5 @@ -169,7 +181,12 @@ def audio_token_count(self, audio_len: int) -> int: num_frames = torch.tensor( [cs // self.config.hop_length for cs in clip_sizes], dtype=torch.float ) - n_tokens = HFParakeetEncoder._get_subsampling_output_length(self, num_frames) + n_tokens = _subsampling_output_length( + num_frames, + self.config.subsampling_factor, + self.config.subsampling_conv_kernel_size, + self.config.subsampling_conv_stride, + ) return max(1, int(n_tokens.sum().item())) def _to_mono_tensor( @@ -256,8 +273,23 @@ def audio_length(self, audio_tokens: int) -> int: return int(audio_tokens * self.config.subsampling_factor * self.config.hop_length) +def _subsampling_output_length( + input_lengths: torch.Tensor, + subsampling_factor: int, + subsampling_conv_kernel_size: int, + subsampling_conv_stride: int, +) -> torch.Tensor: + num_layers = int(math.log2(subsampling_factor)) + add_pad = (subsampling_conv_kernel_size - 1) // 2 * 2 - subsampling_conv_kernel_size + lengths = input_lengths + for _ in range(num_layers): + lengths = torch.div(lengths.to(torch.float) + add_pad, subsampling_conv_stride) + 1.0 + lengths = torch.floor(lengths) + return lengths.to(torch.int) + + # This config is the extractor's single source of truth. It also supplies the fields read by -# `HFParakeetEncoder._get_subsampling_output_length`. +# `_subsampling_output_length`. class _ExtractorConfig(NamedTuple): feature_size: int sampling_rate: int @@ -293,8 +325,8 @@ def _make_parakeet_encoder_config( ) -> ParakeetEncoderConfig: """Build a `ParakeetEncoderConfig` from the HF `sound_config`. - `HFParakeetEncoder` expects fields (`scale_input`, `attention_bias`, `max_position_embeddings`) - that are not present in the raw HF composite config, so we map/default them here. + Several fields required by `ParakeetEncoderConfig` are absent from the raw + HF composite config, so we map/default them here. """ return ParakeetEncoderConfig( hidden_size=sound_config.hidden_size, @@ -308,8 +340,6 @@ def _make_parakeet_encoder_config( subsampling_conv_kernel_size=sound_config.subsampling_conv_kernel_size, subsampling_conv_stride=sound_config.subsampling_conv_stride, num_mel_bins=sound_config.num_mel_bins, - # Fields required by HFParakeetEncoder but absent from the HF - # composite config — use the defaults from ParakeetEncoderConfig. scale_input=getattr(sound_config, "scale_input", False), attention_bias=getattr(sound_config, "attention_bias", False), max_position_embeddings=getattr(sound_config, "max_position_embeddings", 5000), @@ -330,8 +360,8 @@ def __init__( ): super().__init__() self.norm = RMSNorm(hidden_size=hidden_size, eps=eps, dtype=dtype) - self.linear1 = nn.Linear(hidden_size, projection_hidden_size, bias=bias, dtype=dtype) - self.linear2 = nn.Linear(projection_hidden_size, llm_hidden_size, bias=bias, dtype=dtype) + self.linear1 = Linear(hidden_size, projection_hidden_size, bias=bias, dtype=dtype) + self.linear2 = Linear(projection_hidden_size, llm_hidden_size, bias=bias, dtype=dtype) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.norm(hidden_states) @@ -341,6 +371,387 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +# --------------------------------------------------------------------------- +# Conformer encoder +# --------------------------------------------------------------------------- + + +class ParakeetConformerAttention(Attention): + """Conformer self-attention built on trtllm.Attention. + + qkv_proj and o_proj are inherited from trtllm.Attention (TP-sharded, + quantization-ready, fused QKV kernel). The Transformer-XL relative + positional encoding (RPE) is computed here and passed directly to + F.scaled_dot_product_attention as attn_mask. + + relative_k_proj, bias_u, and bias_v implement the content-position term; + the content-content term uses the standard QK product inside SDPA. + """ + + def __init__( + self, + config: ParakeetEncoderConfig, + layer_idx: int, + dtype: Optional[torch.dtype] = None, + ): + head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + num_kv_heads = getattr(config, "num_key_value_heads", config.num_attention_heads) + super().__init__( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + num_key_value_heads=num_kv_heads, + max_position_embeddings=None, + bias=config.attention_bias, + pos_embd_params=None, + rope_fusion=False, + layer_idx=layer_idx, + dtype=dtype, + dense_bias=config.attention_bias, + config=ModelConfig(attn_backend="VANILLA"), + q_scaling=1.0, + head_dim=head_dim, + ) + # Relative-position key projection: position_embeddings → [T_pos, H*d] + self.relative_k_proj = Linear( + config.hidden_size, + config.num_attention_heads * head_dim, + bias=False, + dtype=dtype, + ) + # Transformer-XL global content bias added to q for the content-content term + self.bias_u = nn.Parameter(torch.zeros(self.num_heads, self.head_dim, dtype=dtype)) + # Transformer-XL global positional bias added to q for content-position term + self.bias_v = nn.Parameter(torch.zeros(self.num_heads, self.head_dim, dtype=dtype)) + + @staticmethod + def _rel_shift(x: torch.Tensor) -> torch.Tensor: + """Shaw et al. relative shift. Appendix B of https://arxiv.org/abs/1901.02860.""" + B, H, T, T_pos = x.shape + x = F.pad(x, (1, 0)) + x = x.view(B, H, -1, T) + return x[:, :, 1:].view(B, H, T, T_pos) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_embeddings: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + Args: + hidden_states: [B, T, hidden_size] + attention_mask: Optional [B, 1, T, T] bool mask; True = attend. + position_embeddings: [B, T_pos, hidden_size] relative position encodings. + Returns: + [B, T, hidden_size] + """ + B, T, C = hidden_states.shape + + qkv = self.qkv_proj(hidden_states.reshape(-1, C)) # [B*T, (q+k+v)*d] + q, k, v = self.split_qkv(qkv) # each [B*T, H_rank*d] + + # Reshape to [B, H, T, d] for RPE computation + q = q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(B, T, self.num_key_value_heads, self.head_dim).transpose(1, 2) + v = v.view(B, T, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + # ── Transformer-XL RPE: compute matrix_bd ──────────────────────────── + scale = 1.0 / math.sqrt(self.head_dim) + + # Content-position term: (q + bias_v) @ rel_k^T, then _rel_shift + q_v = q + self.bias_v.view(1, self.num_heads, 1, self.head_dim) + rel_k = self.relative_k_proj(position_embeddings.reshape(-1, C)).view( + B, -1, self.num_heads, self.head_dim + ) # [B, T_pos, H, d] + matrix_bd = q_v @ rel_k.permute(0, 2, 3, 1) # [B, H, T, T_pos] + matrix_bd = self._rel_shift(matrix_bd)[..., :T] * scale # [B, H, T, T] + + # Fold in the padding mask (−∞ for masked/padded positions) + if attention_mask is not None: + matrix_bd = matrix_bd.masked_fill_(~attention_mask, float("-inf")) + + # Content-content term: add bias_u to q; SDPA computes (q+bias_u)@k^T + q = q + self.bias_u.view(1, self.num_heads, 1, self.head_dim) + + out = F.scaled_dot_product_attention(q, k, v, attn_mask=matrix_bd, scale=scale) + attn_output = out.transpose(1, 2).reshape(B * T, self.q_size) + return self.o_proj(attn_output).reshape(B, T, C) + + def load_weights(self, weights: Dict[str, torch.Tensor], prefix: str = "") -> None: + """Load checkpoint weights. + + The checkpoint stores q/k/v as separate projections; qkv_proj is fused. + """ + self.qkv_proj.load_weights( + [ + {"weight": weights[f"{prefix}q_proj.weight"]}, + {"weight": weights[f"{prefix}k_proj.weight"]}, + {"weight": weights[f"{prefix}v_proj.weight"]}, + ] + ) + self.o_proj.load_weights([{"weight": weights[f"{prefix}o_proj.weight"]}]) + self.relative_k_proj.load_weights([{"weight": weights[f"{prefix}relative_k_proj.weight"]}]) + self.bias_u.data.copy_(weights[f"{prefix}bias_u"]) + self.bias_v.data.copy_(weights[f"{prefix}bias_v"]) + + +class ParakeetFeedForward(nn.Module): + """Conformer FFN: Linear(hidden→intermediate) → activation → Linear(intermediate→hidden).""" + + def __init__(self, config: ParakeetEncoderConfig, dtype: Optional[torch.dtype] = None): + super().__init__() + self.linear1 = Linear( + config.hidden_size, + config.intermediate_size, + bias=bool(config.attention_bias), + dtype=dtype, + ) + self.linear2 = Linear( + config.intermediate_size, + config.hidden_size, + bias=bool(config.attention_bias), + dtype=dtype, + ) + self.activation = ACT2FN[getattr(config, "hidden_act", "silu")] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.linear2(self.activation(self.linear1(hidden_states))) + + +class ParakeetConvolutionModule(nn.Module): + """Conformer depthwise convolution module. + + Checkpoint weights for pointwise_conv1/2 have shape [C_out, C_in, 1] and are + squeezed to [C_out, C_in] during load_weights. + """ + + def __init__(self, config: ParakeetEncoderConfig, dtype: Optional[torch.dtype] = None): + super().__init__() + channels = config.hidden_size + kernel_size = config.conv_kernel_size + self.activation = ACT2FN[getattr(config, "hidden_act", "silu")] + self.padding = (kernel_size - 1) // 2 + self.pointwise_conv1 = Linear( + channels, 2 * channels, bias=config.convolution_bias, dtype=dtype + ) + self.depthwise_conv = nn.Conv1d( + channels, + channels, + kernel_size, + stride=1, + padding=self.padding, + groups=channels, + bias=config.convolution_bias, + ) + self.norm = nn.BatchNorm1d(channels) + self.pointwise_conv2 = Linear(channels, channels, bias=config.convolution_bias, dtype=dtype) + if dtype is not None: + self.depthwise_conv.to(dtype=dtype) + self.norm.to(dtype=dtype) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hidden_states = self.pointwise_conv1(hidden_states) # [B, T, 2C] + hidden_states = nn.functional.glu(hidden_states, dim=-1) # [B, T, C] + + # Depthwise conv operates on [B, C, T] + hidden_states = hidden_states.transpose(1, 2) + + if attention_mask is not None: + if attention_mask.dtype == torch.bool: + all_masked_rows = torch.all(~attention_mask, dim=2) + else: + all_masked_rows = torch.all(~(attention_mask == 0.0), dim=2) + hidden_states = hidden_states.masked_fill(all_masked_rows, 0.0) + + hidden_states = self.depthwise_conv(hidden_states) + hidden_states = self.norm(hidden_states) + hidden_states = self.activation(hidden_states) + + hidden_states = hidden_states.transpose(1, 2) # [B, T, C] + return self.pointwise_conv2(hidden_states) + + +class ParakeetConformerBlock(nn.Module): + """Conformer block: FFN → self-attention → convolution → FFN → layer norm.""" + + def __init__( + self, + config: ParakeetEncoderConfig, + layer_idx: int, + dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.feed_forward1 = ParakeetFeedForward(config, dtype=dtype) + self.self_attn = ParakeetConformerAttention(config, layer_idx, dtype=dtype) + self.conv = ParakeetConvolutionModule(config, dtype=dtype) + self.feed_forward2 = ParakeetFeedForward(config, dtype=dtype) + self.norm_feed_forward1 = LayerNorm(hidden_size=config.hidden_size, eps=1e-5, dtype=dtype) + self.norm_self_att = LayerNorm(hidden_size=config.hidden_size, eps=1e-5, dtype=dtype) + self.norm_conv = LayerNorm(hidden_size=config.hidden_size, eps=1e-5, dtype=dtype) + self.norm_feed_forward2 = LayerNorm(hidden_size=config.hidden_size, eps=1e-5, dtype=dtype) + self.norm_out = LayerNorm(hidden_size=config.hidden_size, eps=1e-5, dtype=dtype) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_embeddings: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + # 0.5 residual scaling on both FFN halves is part of the Conformer design. + residual = hidden_states + hidden_states = residual + 0.5 * self.feed_forward1(self.norm_feed_forward1(hidden_states)) + + hidden_states = hidden_states + self.self_attn( + self.norm_self_att(hidden_states), + attention_mask=attention_mask, + position_embeddings=position_embeddings, + ) + + hidden_states = hidden_states + self.conv( + self.norm_conv(hidden_states), attention_mask=attention_mask + ) + + hidden_states = hidden_states + 0.5 * self.feed_forward2( + self.norm_feed_forward2(hidden_states) + ) + + return self.norm_out(hidden_states) + + +class ParakeetEncoder(nn.Module): + """Conformer encoder for Parakeet audio.""" + + def __init__(self, config: ParakeetEncoderConfig, dtype: Optional[torch.dtype] = None): + super().__init__() + self.config = config + self.input_scale = math.sqrt(config.hidden_size) if config.scale_input else 1.0 + self.subsampling = HFParakeetEncoderSubsamplingConv2D(config) + self.encode_positions = HFParakeetEncoderRelPositionalEncoding(config) + self.layers = nn.ModuleList( + [ + ParakeetConformerBlock(config, layer_idx=i, dtype=dtype) + for i in range(config.num_hidden_layers) + ] + ) + + if dtype is not None: + self.subsampling.to(dtype=dtype) + self.encode_positions.to(dtype=dtype) + + def _get_subsampling_output_length(self, input_lengths: torch.Tensor) -> torch.Tensor: + cfg = self.config + return _subsampling_output_length( + input_lengths, + cfg.subsampling_factor, + cfg.subsampling_conv_kernel_size, + cfg.subsampling_conv_stride, + ) + + def _build_attention_mask(self, attention_mask: torch.Tensor, seq_len: int) -> torch.Tensor: + """Convert 2-D input mask to 4-D additive attention mask after subsampling.""" + output_lengths = self._get_subsampling_output_length(attention_mask.sum(-1)) + output_mask = torch.arange(seq_len, device=attention_mask.device) < output_lengths[:, None] + mask_2d = output_mask.unsqueeze(1).expand(-1, seq_len, -1) + mask_2d = mask_2d & mask_2d.transpose(1, 2) + return mask_2d.unsqueeze(1) + + def forward( + self, + input_features: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hidden_states = self.subsampling(input_features, attention_mask) + hidden_states = hidden_states * self.input_scale + position_embeddings = self.encode_positions(hidden_states) + + attn_mask_4d = None + if attention_mask is not None: + attn_mask_4d = self._build_attention_mask(attention_mask, hidden_states.shape[1]) + + for block in self.layers: + hidden_states = block( + hidden_states, + attention_mask=attn_mask_4d, + position_embeddings=position_embeddings, + ) + + return hidden_states + + def load_weights(self, weights: Dict[str, torch.Tensor]) -> None: + """Load weights from a flat dict with keys relative to the encoder root. + + ParakeetConformerAttention modules load their own weights (fused QKV from + separate q/k/v checkpoint keys). All other trtllm.Linear modules are loaded + via their native API. Remaining parameters are loaded via state_dict. + """ + trtllm_param_keys: set[str] = set() + # Track prefixes of attention modules whose sub-modules must be skipped. + attn_prefixes: list[str] = [] + + for mod_name, module in self.named_modules(): + # Skip sub-modules of already-handled attention blocks: named_modules() + # recurses into ParakeetConformerAttention so qkv_proj/o_proj would + # otherwise fall through to the generic Linear branch below. + if any(mod_name.startswith(p) for p in attn_prefixes): + continue + + if isinstance(module, ParakeetConformerAttention): + # Attention owns its own weight loading (fused QKV + relative_k_proj + # + bias_u/bias_v). Prefix maps "layers.i.self_attn" → "layers.i.self_attn." + prefix = f"{mod_name}." if mod_name else "" + attn_prefixes.append(prefix) + module.load_weights(weights, prefix=prefix) + # Checkpoint keys (excluded from remaining dict): + for suffix in ( + "q_proj.weight", + "k_proj.weight", + "v_proj.weight", + "o_proj.weight", + "relative_k_proj.weight", + "bias_u", + "bias_v", + ): + trtllm_param_keys.add(f"{prefix}{suffix}") + if module.qkv_proj.bias is not None: + for sfx in ("q_proj.bias", "k_proj.bias", "v_proj.bias"): + trtllm_param_keys.add(f"{prefix}{sfx}") + if module.o_proj.bias is not None: + trtllm_param_keys.add(f"{prefix}o_proj.bias") + # Model parameter names (allowed as missing in load_state_dict): + # qkv_proj is fused in the model but split in the checkpoint. + trtllm_param_keys.add(f"{prefix}qkv_proj.weight") + if module.qkv_proj.bias is not None: + trtllm_param_keys.add(f"{prefix}qkv_proj.bias") + elif isinstance(module, Linear): + w = weights[f"{mod_name}.weight"] + # Pointwise Conv1d weights are stored as [C_out, C_in, 1]; squeeze for Linear. + entry: Dict[str, torch.Tensor] = {"weight": w.squeeze(-1) if w.dim() == 3 else w} + bias_key = f"{mod_name}.bias" + if bias_key in weights: + entry["bias"] = weights[bias_key] + module.load_weights([entry]) + trtllm_param_keys.add(f"{mod_name}.weight") + trtllm_param_keys.add(bias_key) + + remaining = {k: v for k, v in weights.items() if k not in trtllm_param_keys} + incompatible = self.load_state_dict(remaining, strict=False) + + # Allowed gaps: trtllm weights (loaded above) and optional convolution bias keys. + unexpected_missing = [ + k + for k in incompatible.missing_keys + if k not in trtllm_param_keys and not ("_conv" in k and k.endswith(".bias")) + ] + if unexpected_missing: + raise KeyError(f"Missing encoder weights after loading: {unexpected_missing}") + + class ProjectedParakeet(nn.Module): """Parakeet encoder + MLP projection into the LLM embedding space.""" @@ -352,7 +763,7 @@ def __init__( ): super().__init__() encoder_config = _make_parakeet_encoder_config(sound_config) - self.encoder = HFParakeetEncoder(encoder_config).to(dtype=dtype) + self.encoder = ParakeetEncoder(encoder_config, dtype=dtype) self.projection = ParakeetProjection( hidden_size=sound_config.hidden_size, projection_hidden_size=sound_config.projection_hidden_size, @@ -366,16 +777,11 @@ def forward( input_features: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: - outputs = self.encoder( - input_features=input_features, - attention_mask=attention_mask, - ) - hidden_states = outputs.last_hidden_state - return self.projection(hidden_states) + return self.projection(self.encoder(input_features, attention_mask)) def load_weights(self, weights: Dict[str, torch.Tensor]) -> None: - encoder_weights = {} - projection_weights = {} + encoder_weights: Dict[str, torch.Tensor] = {} + projection_weights: Dict[str, torch.Tensor] = {} for key, value in weights.items(): if key.startswith("sound_encoder.encoder."): sub_key = key[len("sound_encoder.encoder.") :] @@ -387,17 +793,16 @@ def load_weights(self, weights: Dict[str, torch.Tensor]) -> None: sub_key = key[len("sound_projection.") :] projection_weights[sub_key] = value - # There was a bug in `transformers` prior to version 5.0, where the - # `ParakeetEncoderConvolutionModule` would ignore the value of `config.convolution_bias` - # and always use a bias. - incompatible_keys = self.encoder.load_state_dict(encoder_weights, strict=False) - non_conv_bias_keys = [ - key - for key in incompatible_keys.missing_keys - if not ("_conv" in key and key.endswith(".bias")) - ] - if len(non_conv_bias_keys) > 0: - raise KeyError( - f"The following keys were missing from the checkpoint: {non_conv_bias_keys}." - ) - self.projection.load_state_dict(projection_weights, strict=True) + self.encoder.load_weights(encoder_weights) + + # projection.linear1/2 are trtllm.Linear; load them via their API. + # projection.norm (RMSNorm) is loaded via state_dict. + proj = self.projection + proj.linear1.load_weights([{"weight": projection_weights["linear1.weight"]}]) + proj.linear2.load_weights([{"weight": projection_weights["linear2.weight"]}]) + norm_weights = { + k: v + for k, v in projection_weights.items() + if not k.startswith("linear1.") and not k.startswith("linear2.") + } + proj.load_state_dict(norm_weights, strict=False) From 6b126caaf2cbb8c1baa4de1cca9102ec0dfc2542 Mon Sep 17 00:00:00 2001 From: Harris Nover <249353502+hnover-nv@users.noreply.github.com> Date: Thu, 28 May 2026 23:28:09 -0700 Subject: [PATCH 115/308] [https://nvbugs/6194552][fix] stabilize Triton Mamba softplus (#14652) Signed-off-by: Harris Nover <249353502+hnover-nv@users.noreply.github.com> --- tensorrt_llm/_torch/modules/mamba/softplus.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/modules/mamba/softplus.py b/tensorrt_llm/_torch/modules/mamba/softplus.py index 7b64e96432a0..d021af75bc9d 100644 --- a/tensorrt_llm/_torch/modules/mamba/softplus.py +++ b/tensorrt_llm/_torch/modules/mamba/softplus.py @@ -1,7 +1,7 @@ # Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/ops/triton/softplus.py # Copyright (c) 2024, Tri Dao, Albert Gu. # -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,10 +26,13 @@ @triton.jit def softplus(dt): - return tl.math.log(tl.math.exp(dt) + 1) + dt_clamped = tl.minimum(dt, 20.0) + return tl.where(dt <= 20.0, tl.math.log(tl.math.exp(dt_clamped) + 1), + dt) else: @triton.jit def softplus(dt): - return tl.math.log1p(tl.exp(dt)) + dt_clamped = tl.minimum(dt, 20.0) + return tl.where(dt <= 20.0, tl.math.log1p(tl.exp(dt_clamped)), dt) From 8d8a2592293ce65fdf7ed963d1cbcd4d95a26188 Mon Sep 17 00:00:00 2001 From: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> Date: Fri, 29 May 2026 14:43:26 +0800 Subject: [PATCH 116/308] [TRTLLM-13043][chroe] add VisualGen context to AGENTS.md (#14732) Signed-off-by: Zhenhua Wang Signed-off-by: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> --- AGENTS.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index bf34d3a8ba8b..5fe19e95be6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,6 +98,21 @@ HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy/TensorRT) | **Distributed execution** | Tensor/pipeline parallelism via `Mapping` class, multiple backends (MPI, Ray, RPC) | | **Auto-discovery** | Models self-register via `automodel.py`, resolved by HF config `architectures` field | +## VisualGen + +VisualGen is a vertical alongside LLM for Diffusion-Transformer (DiT)-based image/video generation +(text-to-image, text-to-video, image-to-video). It is **not** an LLM backend — it has +its own engine, args, params, and outputs — but shares ops and kernels with the +PyTorch backend where it makes sense (attention, quantization, parallelism). + +Key entry points: +- Public Python API: `from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams`. +- Serving CLI: `trtllm-serve --model --visual_gen_args `. + +Key files: +- `tensorrt_llm/visual_gen/`: VisualGen public Python API. **User-facing surface — before modifying anything here, pause and confirm with the user that a public API change is actually intended; do not infer it from the surrounding task.** +- `tensorrt_llm/_torch/visual_gen/`: VisualGen internal implementation. All non-user-facing code belongs here. + ## Anti-Patterns / Gotchas - **Pre-commit modifies files in-place** — if hooks fail, files are already modified. Re-stage (`git add`) and commit again. From 5421ef9c117776c3755cdbd822ab02f2c5ea6401 Mon Sep 17 00:00:00 2001 From: Joyjit Daw <1127155+tijyojwad@users.noreply.github.com> Date: Fri, 29 May 2026 03:08:56 -0400 Subject: [PATCH 117/308] [None][feature] Add thinking token budget control (#14665) Signed-off-by: tijyojwad <1127155+tijyojwad@users.noreply.github.com> --- tensorrt_llm/llmapi/__init__.py | 4 + tensorrt_llm/llmapi/llm.py | 6 + tensorrt_llm/llmapi/thinking_budget.py | 192 ++++++++++++++++++ tensorrt_llm/sampling_params.py | 16 ++ tensorrt_llm/serve/openai_protocol.py | 24 ++- tensorrt_llm/serve/openai_server.py | 17 ++ tensorrt_llm/serve/responses_utils.py | 7 + .../references/sampling_params.yaml | 4 + tests/unittest/llmapi/test_sampling_params.py | 183 ++++++++++++++++- 9 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 tensorrt_llm/llmapi/thinking_budget.py diff --git a/tensorrt_llm/llmapi/__init__.py b/tensorrt_llm/llmapi/__init__.py index 78b9edaa1c71..74f37df7a8f7 100644 --- a/tensorrt_llm/llmapi/__init__.py +++ b/tensorrt_llm/llmapi/__init__.py @@ -26,6 +26,8 @@ QuantConfig) from .mm_encoder import MultimodalEncoder from .mpi_session import MpiCommSession +from .thinking_budget import (ThinkingBudgetLogitsProcessor, + add_thinking_budget_logits_processor) __all__ = [ 'LLM', @@ -81,4 +83,6 @@ 'SchedulingParams', 'SkipSoftmaxAttentionConfig', 'PrometheusMetricsConfig', + 'ThinkingBudgetLogitsProcessor', + 'add_thinking_budget_logits_processor', ] diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 023bfacf3f85..62f9be618222 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -48,6 +48,7 @@ from .llm_utils import (CachedModelLoader, KvCacheRetentionConfig, LlmBuildStats, ModelLoader, _ModelRuntimeContext) from .mpi_session import MpiPoolSession, external_mpi_comm_available +from .thinking_budget import add_thinking_budget_logits_processor from .tokenizer import TokenizerBase, _xgrammar_tokenizer_info # TODO[chunweiy]: move the following symbols back to utils scope, and remove the following import from .utils import (append_docstring, exception_handler, get_device_count, @@ -1001,6 +1002,11 @@ def _prepare_sampling_params( ) sampling_params._setup(self.tokenizer, self._hf_model_config, self._generation_config) + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser=self.args.reasoning_parser, + tokenizer=self.tokenizer, + ) else: raise TypeError( f"The sampling_params must be type SamplingParams or None, but got {type(sampling_params)}" diff --git a/tensorrt_llm/llmapi/thinking_budget.py b/tensorrt_llm/llmapi/thinking_budget.py new file mode 100644 index 000000000000..d7ca6a5a58a8 --- /dev/null +++ b/tensorrt_llm/llmapi/thinking_budget.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, List, Optional + +import torch + +from tensorrt_llm.sampling_params import ( + LogitsProcessor, + SamplingParams, + validate_thinking_token_budget, +) + + +class ThinkingBudgetLogitsProcessor(LogitsProcessor): + """Force the reasoning end token sequence once a thinking budget is spent.""" + + def __init__( + self, + thinking_token_budget: int, + reasoning_start_token_ids: List[int], + reasoning_end_token_ids: List[int], + ) -> None: + budget = validate_thinking_token_budget(thinking_token_budget) + if budget is None: + raise ValueError( + "thinking_token_budget must be set when creating ThinkingBudgetLogitsProcessor" + ) + if not reasoning_start_token_ids: + raise ValueError("reasoning_start_token_ids must not be empty") + if not reasoning_end_token_ids: + raise ValueError("reasoning_end_token_ids must not be empty") + self.thinking_token_budget = budget + self.reasoning_start_token_ids = list(reasoning_start_token_ids) + self.reasoning_end_token_ids = list(reasoning_end_token_ids) + + def __call__( + self, + req_id: int, + logits: torch.Tensor, + token_ids: List[List[int]], + stream_ptr: Optional[int], + client_id: Optional[int], + ) -> None: + del req_id, client_id + if stream_ptr is None: + self._apply(token_ids, logits) + return + with torch.cuda.stream(torch.cuda.ExternalStream(stream_ptr)): + self._apply(token_ids, logits) + + def _apply(self, token_ids: List[List[int]], logits: torch.Tensor) -> None: + for beam_idx, beam_token_ids in enumerate(token_ids): + forced_token = self._forced_token(beam_token_ids) + if forced_token is not None: + self._force_token(logits, beam_idx, len(token_ids), forced_token) + + def _forced_token(self, token_ids: List[int]) -> Optional[int]: + start_idx = _find_last_sequence_index(token_ids, self.reasoning_start_token_ids) + if start_idx == -1: + return None + + end_idx = _find_last_sequence_index(token_ids, self.reasoning_end_token_ids) + if end_idx > start_idx: + return None + + reasoning_start = start_idx + len(self.reasoning_start_token_ids) + reasoning_token_count = len(token_ids) - reasoning_start + partial_end_len = _longest_suffix_prefix_len(token_ids, self.reasoning_end_token_ids) + if ( + partial_end_len > 0 + and reasoning_token_count - partial_end_len >= self.thinking_token_budget + ): + return self.reasoning_end_token_ids[partial_end_len] + + if reasoning_token_count >= self.thinking_token_budget: + return self.reasoning_end_token_ids[0] + return None + + @staticmethod + def _force_token(logits: torch.Tensor, beam_idx: int, beam_count: int, token_id: int) -> None: + if token_id < 0 or token_id >= logits.shape[-1]: + raise ValueError( + f"Forced reasoning end token id {token_id} is outside the " + f"logits vocabulary dimension {logits.shape[-1]}" + ) + + target = logits + if logits.dim() > 1 and logits.shape[0] == beam_count: + target = logits[beam_idx] + target[:] = float("-inf") + target[..., token_id] = 0 + + +class _ChainedLogitsProcessor(LogitsProcessor): + def __init__(self, processors: List[Any]) -> None: + self.processors = processors + + def __call__( + self, + req_id: int, + logits: torch.Tensor, + token_ids: List[List[int]], + stream_ptr: Optional[int], + client_id: Optional[int], + ) -> None: + for processor in self.processors: + processor(req_id, logits, token_ids, stream_ptr, client_id) + + +def add_thinking_budget_logits_processor( + sampling_params: SamplingParams, + *, + reasoning_parser: Optional[str], + tokenizer: Any, + chat_template_kwargs: Optional[dict[str, Any]] = None, +) -> None: + """Attach a thinking-budget logits processor to ``sampling_params``.""" + if sampling_params.thinking_token_budget is None: + return + existing = sampling_params.logits_processor + if isinstance(existing, ThinkingBudgetLogitsProcessor): + return + if isinstance(existing, _ChainedLogitsProcessor) and any( + isinstance(p, ThinkingBudgetLogitsProcessor) for p in existing.processors + ): + return + if isinstance(existing, list) and any( + isinstance(p, ThinkingBudgetLogitsProcessor) for p in existing + ): + return + if reasoning_parser is None: + raise ValueError("thinking_token_budget requires a configured reasoning_parser") + if tokenizer is None: + raise ValueError("thinking_token_budget requires a tokenizer") + + from .reasoning_parser import ReasoningParserFactory + + parser = ReasoningParserFactory.create_reasoning_parser(reasoning_parser, chat_template_kwargs) + reasoning_start = _get_reasoning_boundary(parser, "start") + reasoning_end = _get_reasoning_boundary(parser, "end") + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=sampling_params.thinking_token_budget, + reasoning_start_token_ids=_encode_token_ids(tokenizer, reasoning_start), + reasoning_end_token_ids=_encode_token_ids(tokenizer, reasoning_end), + ) + + if existing is None: + sampling_params.logits_processor = processor + elif isinstance(existing, list): + existing.append(processor) + else: + sampling_params.logits_processor = _ChainedLogitsProcessor([existing, processor]) + + +def _get_reasoning_boundary(parser: Any, boundary: str) -> str: + if boundary == "start": + candidates = ("reasoning_start", "CHANNEL_OPEN") + else: + candidates = ("reasoning_end", "CHANNEL_CLOSE") + for attr in candidates: + value = getattr(parser, attr, None) + if value: + return value + raise ValueError( + f"Reasoning parser {parser.__class__.__name__} does not define a " + f"reasoning {boundary} string" + ) + + +def _encode_token_ids(tokenizer: Any, text: str) -> List[int]: + try: + return tokenizer.encode(text, add_special_tokens=False) + except TypeError: + return tokenizer.encode(text) + + +def _find_last_sequence_index(token_ids: List[int], sequence: List[int]) -> int: + if not sequence or len(sequence) > len(token_ids): + return -1 + for idx in range(len(token_ids) - len(sequence), -1, -1): + if token_ids[idx : idx + len(sequence)] == sequence: + return idx + return -1 + + +def _longest_suffix_prefix_len(token_ids: List[int], sequence: List[int]) -> int: + max_len = min(len(token_ids), len(sequence) - 1) + for length in range(max_len, 0, -1): + if token_ids[-length:] == sequence[:length]: + return length + return 0 diff --git a/tensorrt_llm/sampling_params.py b/tensorrt_llm/sampling_params.py index 8d38f2fff3ab..910c31445e97 100644 --- a/tensorrt_llm/sampling_params.py +++ b/tensorrt_llm/sampling_params.py @@ -28,6 +28,19 @@ MAX_TOP_LOGPROBS = 20 +def validate_thinking_token_budget(value: Optional[Union[int, float, bool]]) -> Optional[int]: + """Validate ``thinking_token_budget``; return ``None`` if unset.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("thinking_token_budget must be a non-negative integer or -1 for unlimited") + if value == -1: + return None + if value < 0: + raise ValueError("thinking_token_budget must be a non-negative integer or -1 for unlimited") + return value + + def check_logprobs_limit( name: str, value: Optional[int], max_value: int = MAX_TOP_LOGPROBS ) -> None: @@ -226,6 +239,7 @@ class SamplingParams: lookahead_config (tensorrt_llm.bindings.executor.LookaheadDecodingConfig , optional): Lookahead decoding config. Defaults to None. guided_decoding (tensorrt_llm.sampling_params.GuidedDecodingParams, optional): Guided decoding params. Defaults to None. + thinking_token_budget (int, optional): Experimental. Maximum number of tokens allowed inside a reasoning block. Set to -1 or None for unlimited. Defaults to None. ignore_eos (bool): Whether to ignore the EOS token and continue generating tokens after the EOS token is generated. Defaults to False. detokenize (bool): Whether to detokenize the output. Defaults to True. @@ -307,6 +321,7 @@ class SamplingParams: # Guided decoding params guided_decoding: Optional[GuidedDecodingParams] = None + thinking_token_budget: Optional[int] = None # Tokenizer-related configs ignore_eos: bool = False @@ -374,6 +389,7 @@ def _validate(self): if self.guided_decoding is not None: self.guided_decoding._validate() + self.thinking_token_budget = validate_thinking_token_budget(self.thinking_token_budget) # correct types as users might pass in logprob=True for Top-0 logprobs and logprobs=False for no logprobs if self.logprobs is False: diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index e9282795c34f..dbc30bd850e1 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -43,7 +43,8 @@ from tensorrt_llm.llmapi import (DisaggScheduleStyle, GuidedDecodingParams, SamplingParams) from tensorrt_llm.llmapi.reasoning_parser import ReasoningParserFactory -from tensorrt_llm.sampling_params import check_logprobs_limit +from tensorrt_llm.sampling_params import (check_logprobs_limit, + validate_thinking_token_budget) from tensorrt_llm.scheduling_params import AgentHierarchy _LOGIT_BIAS_MIN = -100.0 @@ -394,6 +395,7 @@ class CompletionRequest(OpenAIBaseModel): truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None return_context_logits: bool = False detokenize: bool = True + thinking_token_budget: Optional[int] = None # doc: end-completion-sampling-params # doc: begin-completion-extra-params @@ -467,6 +469,7 @@ def to_sampling_params(self, guided_decoding=_response_format_to_guided_decoding_params( self.response_format), detokenize=self.detokenize, + thinking_token_budget=self.thinking_token_budget, # logits_bias embedding_bias=_logit_bias_to_embedding_bias( @@ -485,6 +488,11 @@ def check_logprobs(cls, data): check_logprobs_limit("logprobs", data.get("logprobs")) return data + @field_validator("thinking_token_budget", mode="before") + @classmethod + def check_thinking_token_budget(cls, value): + return validate_thinking_token_budget(value) + @model_validator(mode="before") @classmethod def validate_stream_options(cls, data): @@ -702,6 +710,7 @@ class ChatCompletionRequest(OpenAIBaseModel): "reasoning is shown in the model's response. Options: " "'low', 'medium', 'high'."), ) + thinking_token_budget: Optional[int] = None prompt_ignore_length: Optional[int] = 0 # doc: begin-chat-completion-sampling-params @@ -848,6 +857,7 @@ def to_sampling_params(self, truncate_prompt_tokens=self.truncate_prompt_tokens, guided_decoding=_response_format_to_guided_decoding_params( self.response_format, reasoning_parser=reasoning_parser), + thinking_token_budget=self.thinking_token_budget, # logits_bias embedding_bias=_logit_bias_to_embedding_bias( @@ -889,6 +899,11 @@ def check_logprobs(cls, data): "logprobs must be true when using top_logprobs") return data + @field_validator("thinking_token_budget", mode="before") + @classmethod + def check_thinking_token_budget(cls, value): + return validate_thinking_token_budget(value) + @model_validator(mode="before") @classmethod def check_suffix(cls, data): @@ -949,6 +964,7 @@ class ResponsesRequest(OpenAIBaseModel): previous_response_id: Optional[str] = None prompt: Optional[ResponsePrompt] = None reasoning: Optional[Reasoning] = None + thinking_token_budget: Optional[int] = None service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto" store: Optional[bool] = True @@ -1006,6 +1022,7 @@ def to_sampling_params( logprobs=self.top_logprobs, stop_token_ids=stop_token_ids, guided_decoding=guided_decoding, + thinking_token_budget=self.thinking_token_budget, ) @model_validator(mode="before") @@ -1024,6 +1041,11 @@ def validate_prompt(cls, data): raise ValueError("prompt template is not supported") return data + @field_validator("thinking_token_budget", mode="before") + @classmethod + def check_thinking_token_budget(cls, value): + return validate_thinking_token_budget(value) + class InputTokensDetails(OpenAIBaseModel): cached_tokens: int diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index a57084b4c225..debee5830dc6 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -40,6 +40,8 @@ from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, MetadataServerConfig, ServerRole) from tensorrt_llm.llmapi.llm import RequestOutput +from tensorrt_llm.llmapi.thinking_budget import \ + add_thinking_budget_logits_processor from tensorrt_llm.logger import logger from tensorrt_llm.media.encoding import image_to_bytes from tensorrt_llm.metrics.collector import MetricsCollector @@ -1148,6 +1150,12 @@ async def chat_stream_generator( gather_generation_logits, reasoning_parser=self.generator.args.reasoning_parser, backend=self.generator.args.backend) + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser=self.generator.args.reasoning_parser, + tokenizer=self.tokenizer, + chat_template_kwargs=request.chat_template_kwargs, + ) if self.tool_parser and request.tools: tool_parser_cls = ToolParserFactory.parsers.get( self.tool_parser.lower()) @@ -1493,6 +1501,11 @@ async def generator_wrapper(generator: AsyncIterator[Any]): gather_generation_logits=self.generator.args. gather_generation_logits, backend=self.generator.args.backend) + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser=self.generator.args.reasoning_parser, + tokenizer=self.tokenizer, + ) # TODO: better way to enable metrics if len(os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH", "")) > 0: sampling_params.return_perf_metrics = True @@ -1625,6 +1638,10 @@ async def create_streaming_generator(promise: RequestOutput, sampling_params = request.to_sampling_params( vocab_size=self._logit_bias_vocab_size(), reasoning_parser="gpt_oss") + if sampling_params.thinking_token_budget is not None: + raise ValueError( + "thinking_token_budget is not supported by the Harmony " + "GPT-OSS serving path; use reasoning_effort instead") sampling_params.detokenize = False # Harmony adapter handles detokenization disaggregated_params = to_llm_disaggregated_params( request.disaggregated_params) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index e75033304f03..499cb6174b13 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -48,6 +48,8 @@ from tensorrt_llm.llmapi.reasoning_parser import (BaseReasoningParser, ReasoningParserFactory, ReasoningParserResult) +from tensorrt_llm.llmapi.thinking_budget import \ + add_thinking_budget_logits_processor from tensorrt_llm.llmapi.tokenizer import TokenizerBase, TransformersTokenizer from tensorrt_llm.logger import logger from tensorrt_llm.serve.chat_utils import (parse_chat_messages_coroutines, @@ -919,6 +921,11 @@ async def request_preprocess( _responses_debug_log("======= Complete Inputs to model =======") _responses_debug_log(_decode_tokens(input_tokens, tokenizer)) _responses_debug_log("========================================") + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser=reasoning_parser, + tokenizer=tokenizer, + ) return input_tokens, sampling_params diff --git a/tests/unittest/api_stability/references/sampling_params.yaml b/tests/unittest/api_stability/references/sampling_params.yaml index edaa4f1469bb..5f4db3e7e7e2 100644 --- a/tests/unittest/api_stability/references/sampling_params.yaml +++ b/tests/unittest/api_stability/references/sampling_params.yaml @@ -15,6 +15,10 @@ methods: prompt_ignore_length: annotation: Optional[int] default: null + thinking_token_budget: + annotation: Optional[int] + default: null + status: prototype logprobs_mode: annotation: LogprobMode default: LogprobMode.RAW diff --git a/tests/unittest/llmapi/test_sampling_params.py b/tests/unittest/llmapi/test_sampling_params.py index 1749fa2897e1..e09e477a1914 100644 --- a/tests/unittest/llmapi/test_sampling_params.py +++ b/tests/unittest/llmapi/test_sampling_params.py @@ -13,7 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest +import torch +from tensorrt_llm.llmapi.thinking_budget import ( + ThinkingBudgetLogitsProcessor, + add_thinking_budget_logits_processor, +) from tensorrt_llm.sampling_params import MAX_TOP_LOGPROBS, SamplingParams, check_logprobs_limit from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest, CompletionRequest @@ -50,7 +55,183 @@ def test_chat_top_logprobs_request_limit(): def test_completion_logprobs_assignment_revalidates(): - request = CompletionRequest(model="test", prompt="hi", logprobs=MAX_TOP_LOGPROBS + 1) + request = CompletionRequest(model="test", prompt="hi", logprobs=MAX_TOP_LOGPROBS) + request.logprobs = MAX_TOP_LOGPROBS + 1 with pytest.raises(ValueError, match=f"less than or equal to {MAX_TOP_LOGPROBS}"): request.to_sampling_params(backend="pytorch") + + +@pytest.mark.parametrize("value", [None, -1]) +def test_thinking_token_budget_unlimited_values(value): + assert SamplingParams(thinking_token_budget=value).thinking_token_budget is None + + +@pytest.mark.parametrize("value", [-2, True, 1.5, "1"]) +def test_thinking_token_budget_rejects_invalid_values(value): + with pytest.raises(ValueError, match="thinking_token_budget"): + SamplingParams(thinking_token_budget=value) + + +def test_chat_thinking_token_budget_request_validation(): + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "hi"}], + thinking_token_budget=8, + ) + + assert request.to_sampling_params().thinking_token_budget == 8 + + with pytest.raises(ValueError, match="thinking_token_budget"): + ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "hi"}], + thinking_token_budget=True, + ) + + +def test_thinking_budget_logits_processor_forces_end_token(): + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=2, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2, 3], + ) + logits = torch.zeros(1, 1, 8) + + processor(0, logits, [[1, 5, 6]], None, None) + + assert logits[0, 0, 2] == 0 + mask = torch.ones(8, dtype=torch.bool) + mask[2] = False + assert torch.isneginf(logits[0, 0, mask]).all() + + +def test_thinking_budget_logits_processor_completes_partial_end_sequence(): + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=2, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2, 3], + ) + logits = torch.zeros(1, 1, 8) + + processor(0, logits, [[1, 5, 6, 2]], None, None) + + assert logits[0, 0, 3] == 0 + mask = torch.ones(8, dtype=torch.bool) + mask[3] = False + assert torch.isneginf(logits[0, 0, mask]).all() + + +def test_thinking_budget_logits_processor_ignores_partial_end_before_budget(): + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=4, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2, 3], + ) + logits = torch.zeros(1, 1, 8) + + processor(0, logits, [[1, 5, 6, 2]], None, None) + + assert torch.equal(logits, torch.zeros(1, 1, 8)) + + +def test_thinking_budget_logits_processor_ignores_closed_reasoning_block(): + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=1, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2], + ) + logits = torch.zeros(1, 1, 8) + + processor(0, logits, [[1, 5, 2, 6]], None, None) + + assert torch.equal(logits, torch.zeros(1, 1, 8)) + + +def test_add_thinking_budget_logits_processor_uses_reasoning_parser_tokens(): + class FakeTokenizer: + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return { + "": [1], + "": [2], + }[text] + + sampling_params = SamplingParams(thinking_token_budget=4) + + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser="qwen3", + tokenizer=FakeTokenizer(), + ) + + assert isinstance(sampling_params.logits_processor, ThinkingBudgetLogitsProcessor) + + +@pytest.mark.parametrize("thinking_token_budget", [0, 2]) +def test_thinking_budget_text_generation_caps_reasoning_tokens(thinking_token_budget): + class FakeTokenizer: + token_to_id = { + "Question:": 0, + "": 1, + "": 2, + "alpha": 3, + "beta": 4, + "gamma": 5, + "delta": 6, + "answer": 7, + } + id_to_token = {token_id: token for token, token_id in token_to_id.items()} + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return [self.token_to_id[token] for token in text.split()] + + def decode(self, token_ids): + return " ".join(self.id_to_token[token_id] for token_id in token_ids) + + tokenizer = FakeTokenizer() + sampling_params = SamplingParams(thinking_token_budget=thinking_token_budget) + add_thinking_budget_logits_processor( + sampling_params, + reasoning_parser="qwen3", + tokenizer=tokenizer, + ) + + prompt_ids = tokenizer.encode("Question:") + generated_ids = [] + reasoning_plan = tokenizer.encode("alpha beta gamma delta") + start_id = tokenizer.encode("")[0] + end_id = tokenizer.encode("")[0] + answer_id = tokenizer.encode("answer")[0] + + def next_model_token(): + if not generated_ids: + return start_id + if end_id in generated_ids: + return answer_id + reasoning_tokens = generated_ids[generated_ids.index(start_id) + 1 :] + if len(reasoning_tokens) < len(reasoning_plan): + return reasoning_plan[len(reasoning_tokens)] + return end_id + + for _ in range(8): + logits = torch.full((1, 1, len(tokenizer.token_to_id)), -100.0) + logits[0, 0, next_model_token()] = 100.0 + sampling_params.logits_processor(0, logits, [prompt_ids + generated_ids], None, None) + generated_ids.append(int(torch.argmax(logits[0, 0]).item())) + if generated_ids[-1] == answer_id: + break + + start_idx = generated_ids.index(start_id) + end_idx = generated_ids.index(end_id) + reasoning_ids = generated_ids[start_idx + 1 : end_idx] + expected_reasoning_ids = reasoning_plan[:thinking_token_budget] + + assert tokenizer.decode(prompt_ids) == "Question:" + assert reasoning_ids == expected_reasoning_ids + assert len(reasoning_ids) <= thinking_token_budget + assert generated_ids[end_idx + 1 :] == [answer_id] + assert tokenizer.decode(generated_ids) == tokenizer.decode( + [start_id, *expected_reasoning_ids, end_id, answer_id] + ) From ecb1b449926eda265c8a31cec5b53f738336995d Mon Sep 17 00:00:00 2001 From: QI JUN <22017000+QiJune@users.noreply.github.com> Date: Fri, 29 May 2026 15:40:34 +0800 Subject: [PATCH 118/308] [TRTLLM-13050][test] Remove two-model eagle3 spec-decoding tests (#14735) Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_dgx_b200.yml | 6 ------ tests/integration/test_lists/test-db/l0_dgx_h100.yml | 4 ---- .../integration/test_lists/test-db/l0_gb200_multi_gpus.yml | 2 -- tests/integration/test_lists/test-db/l0_h100.yml | 5 ----- 4 files changed, 17 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 10adf8fdbd40..586e5986f230 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -295,14 +295,8 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-no_overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-no_overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-two_model-no_overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-two_model-no_overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-no_overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-no_overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-two_model-overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-two_model-no_overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] - unittest/_torch/multi_gpu_modeling -k "deepseek" - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index db6233c82f9c..29e0090d0c0d 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -20,11 +20,9 @@ l0_dgx_h100: - unittest/_torch/multi_gpu -m "not post_merge" TIMEOUT (90) - unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism # ------------- Disaggregated serving tests --------------- - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False] @@ -74,9 +72,7 @@ l0_dgx_h100: orchestrator: mpi tests: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-two_model-overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler] - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index 8811883a6ef9..ae0e24ff2d2b 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -76,8 +76,6 @@ l0_gb200_multi_gpus: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-two_model-overlap_scheduler] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-two_model-overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-overlap_scheduler] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-overlap_scheduler] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index c09068b8a6d3..06b537f64f00 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -98,12 +98,10 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=False-overlap_scheduler=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard[overlap_scheduler=True] @@ -127,10 +125,8 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format - - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False] - - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash @@ -360,7 +356,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance] - accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill] From b1dfd3094c18e4a430993a7fdffae1de01550b94 Mon Sep 17 00:00:00 2001 From: Yiyun Lu <55233584+luyiyun1021@users.noreply.github.com> Date: Fri, 29 May 2026 16:39:09 +0800 Subject: [PATCH 119/308] [TRTLLM-12653][feat] LTX-2 Ulysses cross-attention for v2a with audio padding (#14044) Signed-off-by: Yiyun Lu <55233584+luyiyun1021@users.noreply.github.com> --- .../attention_backend/flash_attn4.py | 31 +- .../visual_gen/attention_backend/vanilla.py | 27 +- .../models/ltx2/ltx2_core/transformer_args.py | 5 + .../models/ltx2/transformer_ltx2.py | 271 ++++++++++++-- .../_torch/visual_gen/modules/attention.py | 24 +- .../visual_gen/multi_gpu/test_ltx2_ulysses.py | 351 ++++++++++++++++++ .../multi_gpu/test_ulysses_attention.py | 76 ++++ .../visual_gen/test_fa4_key_padding_mask.py | 142 +++++++ .../test_vanilla_key_padding_mask.py | 49 +++ 9 files changed, 924 insertions(+), 52 deletions(-) create mode 100644 tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py create mode 100644 tests/unittest/_torch/visual_gen/test_fa4_key_padding_mask.py create mode 100644 tests/unittest/_torch/visual_gen/test_vanilla_key_padding_mask.py diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py b/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py index 0090e43e3005..f75d0b529f1f 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py @@ -71,12 +71,14 @@ def _fwd( k: torch.Tensor, v: torch.Tensor, causal: bool, + seqused_k: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """Calls _flash_attn_fwd with torch.compile disabled. Returns (output, lse).""" output, lse = _flash_attn_fwd( q, k, v, + seqused_k=seqused_k, softmax_scale=self.scale, causal=causal, window_size_left=None, @@ -120,6 +122,7 @@ def forward( v: torch.Tensor, *, attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.FULL, + key_padding_mask: Optional[torch.Tensor] = None, **kwargs, ) -> torch.Tensor: """ @@ -132,11 +135,21 @@ def forward( k: Key tensor [batch_size, seq_len_kv, num_kv_heads, head_dim] v: Value tensor [batch_size, seq_len_kv, num_kv_heads, head_dim] attention_mask: Attention mask type (CAUSAL or FULL) + key_padding_mask: Optional ``[B, S_kv]`` bool tensor; True = valid, + False = pad. Translated to FA4's ``seqused_k = mask.sum(dim=1)`` + (assumes True-prefix layout). Non-causal only. Returns: Output tensor [batch_size, seq_len, num_heads, head_dim] """ - output, _ = self.forward_with_lse(q, k, v, attention_mask=attention_mask, **kwargs) + output, _ = self.forward_with_lse( + q, + k, + v, + attention_mask=attention_mask, + key_padding_mask=key_padding_mask, + **kwargs, + ) return output def forward_with_lse( @@ -145,6 +158,7 @@ def forward_with_lse( k: torch.Tensor, v: torch.Tensor, attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.FULL, + key_padding_mask: Optional[torch.Tensor] = None, **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: """ @@ -157,7 +171,20 @@ def forward_with_lse( partial attention results in Attention2D parallelism. """ q, k, v, is_causal, origin_dtype = self._prepare_inputs(q, k, v, attention_mask) - output, lse = self._fwd(q, k, v, is_causal) + seqused_k = None + if key_padding_mask is not None: + assert not is_causal, "key_padding_mask is not supported with causal attention" + assert key_padding_mask.dim() == 2 and key_padding_mask.shape == ( + q.shape[0], + k.shape[1], + ), ( + f"Invalid key_padding_mask shape: expected [B={q.shape[0]}, " + f"S_kv={k.shape[1]}], got {tuple(key_padding_mask.shape)}" + ) + # FA4 seqused_k assumes a True-prefix layout: positions [0, valid) + # are kept, [valid, S_kv) are masked. mask.sum gives the prefix length. + seqused_k = key_padding_mask.sum(dim=1).to(torch.int32) + output, lse = self._fwd(q, k, v, is_causal, seqused_k=seqused_k) if output.dtype != origin_dtype: output = output.to(origin_dtype) return output, lse diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py b/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py index ea6c3947fc78..043d660bf00f 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/vanilla.py @@ -71,6 +71,7 @@ def forward( v: torch.Tensor, *, attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.FULL, + key_padding_mask: Optional[torch.Tensor] = None, **kwargs, ) -> torch.Tensor: """ @@ -83,6 +84,9 @@ def forward( k: Key tensor [batch_size, num_kv_heads, seq_len_kv, head_dim] v: Value tensor [batch_size, num_kv_heads, seq_len_kv, head_dim] attention_mask: Attention mask type (CAUSAL or FULL) + key_padding_mask: Optional ``[B, S_kv]`` bool tensor; True = valid, + False = pad. Expanded internally to ``[B, 1, 1, S_kv]`` and + passed as ``attn_mask`` to SDPA. Non-causal only. Returns: Output tensor [batch_size, num_heads, seq_len, head_dim] @@ -99,13 +103,24 @@ def forward( f"Invalid v shape: expected [B={q.shape[0]}, H_kv, S_kv, D={self.head_dim}], got {v.shape}" ) + enable_gqa = self.num_heads != self.num_kv_heads + if key_padding_mask is not None: + assert not is_causal, "key_padding_mask is not supported with causal attention" + assert key_padding_mask.dim() == 2 and key_padding_mask.shape == ( + q.shape[0], + k.shape[2], + ), ( + f"Invalid key_padding_mask shape: expected [B={q.shape[0]}, " + f"S_kv={k.shape[2]}], got {tuple(key_padding_mask.shape)}" + ) + # [B, S_kv] -> [B, 1, 1, S_kv] so SDPA broadcasts over H and S_q. + attn_mask = key_padding_mask[:, None, None, :] + return F.scaled_dot_product_attention( + q, k, v, attn_mask=attn_mask, scale=self.scale, enable_gqa=enable_gqa + ) + return F.scaled_dot_product_attention( - q, - k, - v, - is_causal=is_causal, - scale=self.scale, - enable_gqa=self.num_heads != self.num_kv_heads, + q, k, v, is_causal=is_causal, scale=self.scale, enable_gqa=enable_gqa ) @property diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/ltx2_core/transformer_args.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/ltx2_core/transformer_args.py index 558debb9b41d..31b10697d741 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/ltx2_core/transformer_args.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/ltx2_core/transformer_args.py @@ -32,6 +32,11 @@ class TransformerArgs: cross_scale_shift_timestep: torch.Tensor | None cross_gate_timestep: torch.Tensor | None enabled: bool + # Optional [B, S_full_padded] bool mask (True=valid, False=pad) for the + # audio modality when Ulysses padding is engaged (T_a padded to be + # divisible by ulysses_size). Identical across Ulysses ranks (full-seq). + # None when no padding is applied. + audio_padding_mask: torch.Tensor | None = None class TransformerArgsPreprocessor: diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py index 9dbb91059dd1..d16ab44b3b16 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py @@ -103,6 +103,14 @@ def __init__( # Cross-attention: SEPARATE_QKV since K/V come from a different source. qkv_mode = QKVMode.SEPARATE_QKV if self._is_cross_attn else QKVMode.FUSE_QKV + # Decide whether to ask the base class for the Ulysses-wrapped path. + # Caller opts in via use_ulysses. Cross-attn additionally requires no + # other CP to be active (Ring/Attention2D + Ulysses cross-attn is + # unvalidated and falls through to the plain + AG path). + ulysses_size = vgm.ulysses_size if vgm is not None else 1 + cp_size = vgm.cp_size if vgm is not None else 1 + enable_ulysses = use_ulysses and (cp_size == 1 or not self._is_cross_attn) + # Map LTX RoPE type to the fused-kernel INTERLEAVE template parameter: # INTERLEAVED → pair (2i, 2i+1) pattern → kernel INTERLEAVE=true # SPLIT → rotate-half pattern → kernel INTERLEAVE=false @@ -120,23 +128,34 @@ def __init__( fuse_qk_norm_rope=True, config=config, layer_idx=layer_idx, - enable_ulysses=use_ulysses and not self._is_cross_attn, + enable_ulysses=enable_ulysses, ) - # For audio self-attention that may need a runtime Ulysses toggle - # (sequence length not always divisible by ulysses_size), create a - # plain backend as fallback. The base class already set self.attn - # to UlyssesAttention(inner_backend=sharded_backend). + # Build a runtime-toggleable Ulysses ↔ plain pair. + # Self-attn: audio length isn't always divisible by ulysses_size, so + # we need a plain fallback. Cross-attn (v2a): same need — audio Q is + # padded when divisible, plain backend is used otherwise. Plain has + # to be built with the full (unsharded) head count. self._has_dual_attn = False - ulysses_size = vgm.ulysses_size if vgm is not None else 1 - if use_ulysses and not self._is_cross_attn and ulysses_size > 1: + if enable_ulysses and ulysses_size > 1: + U = ulysses_size + H = self.num_attention_heads + H_kv = self.num_key_value_heads + if H % U != 0 or H_kv % U != 0: + raise ValueError( + f"Ulysses requires num_attention_heads ({H}) and " + f"num_key_value_heads ({H_kv}) divisible by ulysses_size ({U})" + ) + # Base class already built `self.attn` as the Ulysses-wrapped path + # (sharded inner backend + UlyssesAttention) for both self-attn and + # cross-attn paths. self._ulysses_attn = self.attn self._plain_attn = create_attention( backend=self.attn_backend, layer_idx=self.layer_idx, - num_heads=self.num_attention_heads, + num_heads=H, head_dim=self.head_dim, - num_kv_heads=self.num_key_value_heads, + num_kv_heads=H_kv, quant_config=self.quant_config, dtype=self.dtype, attention_config=config.attention, @@ -159,14 +178,25 @@ def __init__( self.to_gate_logits = None def set_ulysses_active(self, active: bool): - """Toggle between UlyssesAttention and plain attention at runtime. + """Toggle between Ulysses-wrapped and plain attention at runtime. - Only effective for modules created with ``use_ulysses=True``. + Effective for modules created with ``use_ulysses=True`` (works for + both self-attn and cross-attn). No-op otherwise. """ - if not self._has_dual_attn: - return - self._modules.pop("attn", None) - self.attn = self._ulysses_attn if active else self._plain_attn + if self._has_dual_attn: + self._modules.pop("attn", None) + self.attn = self._ulysses_attn if active else self._plain_attn + + def is_ulysses_active(self) -> bool: + """Whether ``self.attn`` is currently the Ulysses-wrapped path. + + Symmetric with ``set_ulysses_active``. Returns False when no Ulysses + pair was built (e.g. Attention2D mode, ulysses_size==1, or cross-attn + without pure Ulysses), so callers can use it to decide whether to pass + seq-sharded K/V (wrapper handles a2a) or to all-gather K/V into full + sequence first (plain backend). + """ + return self._has_dual_attn and self.attn is self._ulysses_attn def _init_qkv_proj(self): """Override for cross-attention: use _context_dim for K/V input. @@ -246,6 +276,7 @@ def forward( pe: tuple[torch.Tensor, torch.Tensor] | None = None, k_pe: tuple[torch.Tensor, torch.Tensor] | None = None, pre_projected_kv: tuple[torch.Tensor, torch.Tensor] | None = None, + key_padding_mask: torch.Tensor | None = None, ) -> torch.Tensor: """Forward pass. @@ -254,13 +285,22 @@ def forward( - SEPARATE_QKV (cross-attn): cached path requires pre_projected_kv; uncached path requires `context`. pe optional (None = norm-only). k_pe overrides pe for K (e.g. AV cross-attn) when provided. + + Args: + key_padding_mask: Optional ``[B, S_kv]`` bool tensor; True = valid, + False = pad. Forwarded through ``_attn_impl`` to the backend. + Honored by ``VanillaAttention`` and ``FlashAttn4Attention`` + (and ``UlyssesAttention`` wrapping either); the TRTLLM backend + silently ignores it. ``LTX2Attention`` constructs ``audio_attn1`` + with a VANILLA backend whenever Ulysses is active under a + TRTLLM backend config (see ``_init_audio_modules``). """ # Fallback to the naive eager rope path when fusion is disabled or # the kernel doesn't support this head_dim. LTX-2 prod has # fuse_qk_norm_rope=True and head_dim ∈ {64, 128}, so this branch # only fires under mini-config unit tests (head_dim=32). if not self.fuse_qk_norm_rope or self.head_dim not in (64, 128): - return self._forward_unfused(x, context, pe, k_pe, pre_projected_kv) + return self._forward_unfused(x, context, pe, k_pe, pre_projected_kv, key_padding_mask) if self.qkv_mode == QKVMode.FUSE_QKV: # ─── self-attn → packed kernel (norm + rope on QKV in-place) ─── @@ -295,7 +335,10 @@ def forward( k_pe if k_pe is not None else pe, ) - out = self._attn_impl(q, k, v) + attn_kwargs = {} + if key_padding_mask is not None: + attn_kwargs["key_padding_mask"] = key_padding_mask + out = self._attn_impl(q, k, v, **attn_kwargs) if self.to_gate_logits is not None: gate_logits = self.to_gate_logits(x) @@ -314,6 +357,7 @@ def _forward_unfused( pe: tuple[torch.Tensor, torch.Tensor] | None, k_pe: tuple[torch.Tensor, torch.Tensor] | None, pre_projected_kv: tuple[torch.Tensor, torch.Tensor] | None, + key_padding_mask: torch.Tensor | None = None, ) -> torch.Tensor: """Fallback path for unsupported configs (head_dim ∉ {64, 128} or ``fuse_qk_norm_rope=False``). @@ -344,7 +388,10 @@ def _forward_unfused( elif pre_projected_kv is None: k = apply_rotary_emb(k, pe, self.rope_type) - out = self._attn_impl(q, k, v) + attn_kwargs = {} + if key_padding_mask is not None: + attn_kwargs["key_padding_mask"] = key_padding_mask + out = self._attn_impl(q, k, v, **attn_kwargs) if self.to_gate_logits is not None: gate_logits = self.to_gate_logits(x) @@ -448,6 +495,22 @@ def _init_video_modules(self, cfg, rope_type, eps, model_config, idx): self.scale_shift_table = nn.Parameter(torch.empty(6, cfg.dim)) def _init_audio_modules(self, cfg, rope_type, eps, model_config, idx): + # Audio under Ulysses needs key_padding_mask support on audio_attn1 + # (audio is padded to be divisible by ulysses_size; mask zeros pad + # slots). TRTLLM self-attn silently drops key_padding_mask, so downgrade + # to VANILLA whenever Ulysses is active under a TRTLLM backend config. + # Mirrors the existing cross-attn TRTLLM→VANILLA fallback + # (modules/attention.py); audio is small (T_a ~ 126) so the downgrade + # is negligible. + audio_self_config = model_config + vgm = model_config.visual_gen_mapping + ulysses_size = vgm.ulysses_size if vgm is not None else 1 + if ulysses_size > 1 and model_config.attention.backend == "TRTLLM": + audio_self_config = model_config.model_copy( + update={ + "attention": model_config.attention.model_copy(update={"backend": "VANILLA"}) + } + ) self.audio_attn1 = LTX2Attention( query_dim=cfg.dim, heads=cfg.heads, @@ -456,7 +519,7 @@ def _init_audio_modules(self, cfg, rope_type, eps, model_config, idx): rope_type=rope_type, norm_eps=eps, apply_gated_attention=cfg.apply_gated_attention, - config=model_config, + config=audio_self_config, layer_idx=idx, use_ulysses=True, ) @@ -496,6 +559,7 @@ def _init_av_cross_modules(self, v_cfg, a_cfg, rope_type, eps, model_config, idx apply_gated_attention=a_cfg.apply_gated_attention, config=model_config, layer_idx=idx, + use_ulysses=True, ) self.scale_shift_table_a2v_ca_audio = nn.Parameter(torch.empty(5, a_cfg.dim)) self.scale_shift_table_a2v_ca_video = nn.Parameter(torch.empty(5, v_cfg.dim)) @@ -629,7 +693,14 @@ def forward( ) if not skip_a_self: norm_ax = rms_norm(ax, eps=self.norm_eps) * (1 + ascale_msa) + ashift_msa - a_self_out = self.audio_attn1(norm_ax, pe=audio.positional_embeddings) * agate_msa + a_self_out = ( + self.audio_attn1( + norm_ax, + pe=audio.positional_embeddings, + key_padding_mask=audio.audio_padding_mask, + ) + * agate_msa + ) if has_perturbations and perturbations.any_in_batch( PerturbationType.SKIP_AUDIO_SELF_ATTN, self.idx ): @@ -690,7 +761,11 @@ def forward( # so they benefit from Ulysses scaling. RoPE is applied to K # inside project_kv on the sharded shard (RoPE commutes with # seq-dim concat), so the cos/sin all-gather is unneeded and K - # rope work is U× cheaper. + # rope work is U× cheaper. a2v keeps the all-gather path + # (Q=video huge, K/V=audio small — AG of the small audio is + # far cheaper than full-video a2a collectives). + # key_padding_mask zeros attention on the audio pad slots that + # configure_audio_ulysses appended to make T_a divisible by U. k_a2v, v_a2v = self.audio_to_video_attn.project_kv( ax_scaled, pe=audio.cross_positional_embeddings ) @@ -704,6 +779,7 @@ def forward( pre_projected_kv=(k_a2v, v_a2v), pe=video.cross_positional_embeddings, k_pe=None, # K already rotated in project_kv + key_padding_mask=audio.audio_padding_mask, ) * gate_out_a2v ) @@ -719,12 +795,23 @@ def forward( ax_scaled = ax_norm3 * (1 + scale_ca_audio_v2a) + shift_ca_audio_v2a vx_scaled = vx_norm3 * (1 + scale_ca_video_v2a) + shift_ca_video_v2a - # Project-before-gather (video → audio direction). RoPE applied - # to K in project_kv on local shard; see audio→video branch above. + # v2a: when the Ulysses wrapper is active, K/V (video, large) + # stay seq-sharded and the wrapper handles Q + K|V + output + # a2a internally. RoPE is applied to K in project_kv on the + # local shard (commutes with a2a along the seq dim, so + # rotate-before-gather is value-preserving). No + # key_padding_mask — video K/V is unpadded; padded audio Q is + # stripped on exit by LTXModel.forward. When inactive (no + # wrapper built, Stage 2 disable, or audio not sharded), fall + # back to AG so the plain backend sees full K/V. Gate on + # is_ulysses_active() — _audio_is_sharded can be true under + # Attention2D where no wrapper was built. k_v2a, v_v2a = self.video_to_audio_attn.project_kv( vx_scaled, pe=video.cross_positional_embeddings ) - if self._sharder.is_active: + if not self.video_to_audio_attn.is_ulysses_active() and self._sharder.is_active: + # Fallback: wrapper inactive → all-gather sharded video + # K/V to full so plain backend can run. k_v2a = self._sp_all_gather(k_v2a) v_v2a = self._sp_all_gather(v_v2a) @@ -933,6 +1020,7 @@ def __init__( ) self._audio_is_sharded = False + self._audio_pad = 0 # set by configure_audio_ulysses self._cache_dit_video_args: Optional[TransformerArgs] = None self._cache_dit_audio_args: Optional[TransformerArgs] = None # Per-block text cross-attn KV lists, looked up by inner.idx in the @@ -1289,31 +1377,110 @@ def _gather_sequence(self, x: torch.Tensor) -> torch.Tensor: """All-gather hidden states along the sequence dim.""" return self._sharder.gather(x, dim=1) + @staticmethod + def _pad_pe( + pe: tuple[torch.Tensor, torch.Tensor] | None, + pad: int, + seq_dim: int, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + """Repeat-last pad a (cos, sin) PE tuple by ``pad`` slots on ``seq_dim``. + + Used by ``prepare_text_cache`` to extend audio PE when audio padding + is active, so the rope helper sees consistent shapes between padded + audio input and PE. Repeat-last keeps RoPE values at pad slots equal + to the last valid token (no OOB positional index). + + Caller passes ``seq_dim`` explicitly (depends on rope type and PE + layout): SPLIT rope = ``[B, H, T, D]`` so 2; INTERLEAVED = ``[B, T, D]`` + so 1. + """ + if pe is None or pad <= 0: + return pe + cos, sin = pe + if not (0 <= seq_dim < cos.ndim): + raise ValueError(f"_pad_pe: seq_dim={seq_dim} out of range for cos.ndim={cos.ndim}") + + def _ext(t: torch.Tensor) -> torch.Tensor: + idx = [slice(None)] * t.ndim + idx[seq_dim] = slice(t.shape[seq_dim] - 1, t.shape[seq_dim]) + last = t[tuple(idx)] + tail_shape = list(t.shape) + tail_shape[seq_dim] = pad + tail = last.expand(*tail_shape).contiguous() + return torch.cat([t, tail], dim=seq_dim) + + return (_ext(cos), _ext(sin)) + + @staticmethod + def _pad_modality_audio(audio: Modality, pad: int) -> Modality: + """Pad ``audio`` on the token axis by ``pad`` slots. + + Used by ``forward`` when Ulysses is active and ``T_a % ulysses_size + != 0`` to make audio shardable. + + - ``latent``: zero-pad on ``dim=1``. Attention zeros out pad rows via + ``audio_padding_mask``; norm/MLP on zero rows is harmless because + pad rows are stripped before output processing. + - ``positions``: repeat-last on ``dim=2`` (works for both 3D + ``(B, n_dims, T)`` and 4D ``(B, n_dims, T, 2)`` shapes). RoPE + cos/sin at pad slots equals the last valid token's; keeps RoPE + indices inside the model's trained range regardless of + ``positional_embedding_max_pos``. + - ``timesteps``: repeat-last on ``dim=1`` only when per-token + ``(B, T)``; scalar ``(B,)`` timesteps are not padded. + - ``context`` / ``context_mask``: untouched (text-side, audio-token- + count-independent). + """ + if pad <= 0: + return audio + # Latent: zero-pad. F.pad pads last dim first; (0,0,0,pad) pads dim=-2. + latent = F.pad(audio.latent, (0, 0, 0, pad)) + # Positions: repeat-last on dim=2. + pos = audio.positions + last = pos[:, :, -1:, ...] + repeat_shape = list(pos.shape) + repeat_shape[2] = pad + tail = last.expand(*repeat_shape).contiguous() + positions = torch.cat([pos, tail], dim=2) + # Timesteps: repeat-last on dim=1 only when per-token. + if audio.timesteps.ndim >= 2 and audio.timesteps.shape[1] == pos.shape[2]: + ts = audio.timesteps + last_ts = ts[:, -1:, ...].expand(ts.shape[0], pad, *ts.shape[2:]).contiguous() + timesteps = torch.cat([ts, last_ts], dim=1) + else: + timesteps = audio.timesteps + return replace(audio, latent=latent, positions=positions, timesteps=timesteps) + def configure_audio_ulysses(self, audio_seq_len: int) -> None: - """Configure whether audio uses Ulysses based on sequence length. + """Configure audio sharding + padding for Ulysses. Call once before the denoising loop when the audio token count is - known. The decision is cached — ``forward()`` uses it without + known. The decision is cached — ``forward()`` uses it without re-checking. + + When sequence parallelism is active, audio is always padded to + ``(U - T_a % U) % U`` slots so ``T_a`` becomes divisible by ``U`` + and a ``[B, T_a_padded]`` validity mask is attached so attention + zeros out pad positions. ``forward`` strips the pad tail on exit. """ if not self._sharder.is_active: self._audio_is_sharded = False + self._audio_pad = 0 return - is_divisible = (audio_seq_len % self._sharder.size) == 0 - if not is_divisible and self._cp_size > 1 and self._ulysses_size == 1: - raise ValueError( - f"audio_seq_len ({audio_seq_len}) must be divisible by seq_size " - f"({self._sharder.size}) when CP is active without Ulysses " - "(Ring/Attention2D has no plain-attention fallback)." - ) + U = self._sharder.size + self._audio_pad = (U - audio_seq_len % U) % U + self._audio_is_sharded = True - self._audio_is_sharded = is_divisible for block in self.transformer_blocks: target = block.inner if isinstance(block, LTX2CacheDiTPattern0BlockWrapper) else block target._audio_is_sharded = self._audio_is_sharded if hasattr(target, "audio_attn1"): target.audio_attn1.set_ulysses_active(self._audio_is_sharded) + # v2a cross-attn requires sharded audio Q — gate on + # _audio_is_sharded (video K/V is always sharded under Ulysses). + if hasattr(target, "video_to_audio_attn"): + target.video_to_audio_attn.set_ulysses_active(self._audio_is_sharded) def set_ulysses_enabled(self, enabled: bool) -> None: """Enable or disable Ulysses parallelism at runtime. @@ -1344,6 +1511,8 @@ def set_ulysses_enabled(self, enabled: bool) -> None: target.attn1.set_ulysses_active(enabled) if hasattr(target, "audio_attn1") and not enabled: target.audio_attn1.set_ulysses_active(False) + if hasattr(target, "video_to_audio_attn") and not enabled: + target.video_to_audio_attn.set_ulysses_active(False) # -- Output processing --------------------------------------------------- @@ -1397,6 +1566,12 @@ def prepare_text_cache( audio_context, audio_context_mask, audio_positions, dtype ) a_kv = [block.audio_attn2.project_kv(a_ctx) for block in self.transformer_blocks] + # Extend audio PE to padded length once at cache-build time. + # seq_dim per rope type: SPLIT cos [B,H,T,D] → 2, INTERLEAVED [B,T,D] → 1. + if self._audio_pad > 0: + pe_seq_dim = 2 if (a_pe is not None and a_pe[0].ndim == 4) else 1 + a_pe = self._pad_pe(a_pe, self._audio_pad, seq_dim=pe_seq_dim) + a_cross_pe = self._pad_pe(a_cross_pe, self._audio_pad, seq_dim=pe_seq_dim) # Build sharded-local PE in the form the attention consumer expects. # fuse_qk_norm_rope=True (LTX-2 default) -> 2D [T_local, H*D] contiguous, @@ -1454,6 +1629,25 @@ def forward( if not self.model_type.is_audio_enabled() and audio is not None: raise ValueError("Audio is not enabled for this model") + # Audio padding for Ulysses: when self._audio_pad > 0 (set once by + # configure_audio_ulysses to make T_a divisible by ulysses_size), pad + # audio on entry to make it shardable. Build a [B, T_a_padded] bool mask + # (True=valid, False=pad) that travels through TransformerArgs to + # audio_attn1 + a2v. Strip the padded tail on output below. + # text_cache.audio_{pe,cross_pe} are already padded in prepare_text_cache. + audio_padding_mask = None + s_real_audio = None + if audio is not None and self._audio_pad > 0: + s_real_audio = audio.latent.shape[1] + audio = self._pad_modality_audio(audio, self._audio_pad) + s_full_audio = audio.latent.shape[1] + audio_padding_mask = torch.ones( + (audio.latent.shape[0], s_full_audio), + dtype=torch.bool, + device=audio.latent.device, + ) + audio_padding_mask[:, s_real_audio:] = False + video_args = ( self.video_args_preprocessor.prepare( video, @@ -1465,6 +1659,7 @@ def forward( if video is not None else None ) + audio_args = ( self.audio_args_preprocessor.prepare( audio, @@ -1477,6 +1672,11 @@ def forward( else None ) + # Attach the full-seq audio padding mask (identical across Ulysses + # ranks; _shard_transformer_args passes it through unchanged). + if audio_args is not None and audio_padding_mask is not None: + audio_args = replace(audio_args, audio_padding_mask=audio_padding_mask) + # Shard sequences for parallelism (Ulysses head-sharding, ring CP, or Attention2D). # Video is always sharded. Audio sharding is decided once by # configure_audio_ulysses() and cached in self._audio_is_sharded. @@ -1566,6 +1766,9 @@ def forward( if audio_args is not None else None ) + # Strip the padded tail when audio was padded on entry. + if ax is not None and s_real_audio is not None: + ax = ax[:, :s_real_audio] return vx, ax @staticmethod diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index 9cc8f6274bdb..77f9a377a1ae 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -185,16 +185,20 @@ def __init__( col_process_group=vgm.attn2d_col_group, ) else: - # Wrap with parallelism strategies (orthogonal to backend choice) - if (ring_size > 1 or ulysses_size > 1) and self.qkv_mode != QKVMode.SEPARATE_QKV: - if ring_size > 1: - from ..attention_backend.parallel import RingAttention - - self.attn = RingAttention(self.attn, process_group=vgm.ring_group) - if ulysses_size > 1: - from ..attention_backend.parallel import UlyssesAttention - - self.attn = UlyssesAttention(self.attn, process_group=vgm.ulysses_group) + # Ring shards the sequence dim and therefore requires S_q == S_kv, + # so only self-attention is eligible. + if ring_size > 1 and self.qkv_mode != QKVMode.SEPARATE_QKV: + from ..attention_backend.parallel import RingAttention + + self.attn = RingAttention(self.attn, process_group=vgm.ring_group) + # Ulysses shards the head dim and is value-preserving under + # different Q/KV sequence lengths, so it is valid on cross-attn too. + # ``use_ulysses`` already folds in the caller-provided + # ``enable_ulysses`` flag. + if use_ulysses: + from ..attention_backend.parallel import UlyssesAttention + + self.attn = UlyssesAttention(self.attn, process_group=vgm.ulysses_group) def _init_qkv_proj(self) -> None: if self.qkv_mode == QKVMode.FUSE_QKV: diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py new file mode 100644 index 000000000000..750ae3f65d86 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py @@ -0,0 +1,351 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Multi-GPU e2e tests for LTX-2 AudioVideo Ulysses sequence parallelism. + +Runs ``LTXModel.forward`` end-to-end with ``ulysses_size=2`` and compares +to the single-GPU reference (same weights, same inputs). Covers self-attn, +v2a, and a2v cross-attn paths simultaneously. + +Run with: + pytest tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py -v +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" + +from types import SimpleNamespace +from typing import Callable + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +try: + from tensorrt_llm._torch.visual_gen.config import ( + DiffusionModelConfig, + create_attention_metadata_state, + ) + from tensorrt_llm._torch.visual_gen.mapping import VisualGenMapping + from tensorrt_llm._utils import get_free_port + from tensorrt_llm.models.modeling_utils import QuantConfig + from tensorrt_llm.visual_gen.args import AttentionConfig, ParallelConfig, TorchCompileConfig + + MODULES_AVAILABLE = True +except ImportError: + MODULES_AVAILABLE = False + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +# ============================================================================= +# Distributed helpers (same pattern as test_flux_ulysses.py) +# ============================================================================= + + +def init_distributed_worker(rank: int, world_size: int, backend: str = "nccl", port: int = 29500): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + torch.cuda.set_device(rank % torch.cuda.device_count()) + dist.init_process_group(backend=backend, rank=rank, world_size=world_size) + + +def cleanup_distributed(): + if dist.is_initialized(): + dist.destroy_process_group() + + +def _distributed_worker(rank, world_size, backend, test_fn, port, fn_args): + try: + init_distributed_worker(rank, world_size, backend, port) + test_fn(rank, world_size, *fn_args) + except Exception as e: + print(f"Rank {rank} failed with error: {e}") + raise + finally: + cleanup_distributed() + + +def run_test_in_distributed(world_size: int, test_fn: Callable, *fn_args): + if not MODULES_AVAILABLE: + pytest.skip("Required modules not available") + if torch.cuda.device_count() < world_size: + pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") + port = get_free_port() + mp.spawn( + _distributed_worker, + args=(world_size, "nccl", test_fn, port, fn_args), + nprocs=world_size, + join=True, + ) + + +# ============================================================================= +# Model config +# ============================================================================= + +# Small AudioVideo config. ulysses_size=2 needs: +# - num_attention_heads % 2 == 0 (video heads sharded by Ulysses) +# - audio_num_attention_heads % 2 == 0 (audio heads sharded by Ulysses) +# - head_dim ∈ {64, 128} to exercise the fused FA4 / vanilla rope path +# cross_attention_dim must equal inner_dim (=num_heads * head_dim) because +# caption_projection projects caption_channels → inner_dim and the result is +# fed straight into block.attn2.to_k whose input dim = cross_attention_dim. +# Same constraint applies to audio_*. +# head_dim=32 deliberately falls outside {64, 128} so LTX2Attention.forward() +# takes ``_forward_unfused`` and ``project_kv`` skips ``fused_dit_split_norm``. +# That keeps the test runnable in dev images without the LTX-2 C++ extensions +# while still exercising the v2a Ulysses + key_padding_mask code paths under +# both VANILLA and FLASH_ATTN4 backends. +_AV_CONFIG = dict( + num_attention_heads=4, + attention_head_dim=32, + in_channels=16, + out_channels=16, + num_layers=2, + cross_attention_dim=128, # = num_attention_heads * attention_head_dim + caption_channels=64, + norm_eps=1e-6, + positional_embedding_max_pos=[4, 32, 32], + timestep_scale_multiplier=1000, + use_middle_indices_grid=True, + audio_num_attention_heads=4, + audio_attention_head_dim=32, + audio_in_channels=16, + audio_out_channels=16, + audio_cross_attention_dim=128, # = audio_num_attention_heads * audio_attention_head_dim + audio_positional_embedding_max_pos=[64], + av_ca_timestep_scale_multiplier=1, +) + + +def _make_model_config( + ulysses_size: int = 1, + backend: str = "VANILLA", +) -> "DiffusionModelConfig": + """Create DiffusionModelConfig for LTX-2 tests.""" + if ulysses_size > 1 and dist.is_initialized(): + ws = dist.get_world_size() + rk = dist.get_rank() + else: + ws = ulysses_size + rk = 0 + vgm = VisualGenMapping(world_size=ws, rank=rk, ulysses_size=ulysses_size) + + config = DiffusionModelConfig( + pretrained_config=SimpleNamespace(), + quant_config=QuantConfig(), + torch_compile=TorchCompileConfig(enable=False), + attention=AttentionConfig(backend=backend), + visual_gen_mapping=vgm, + cache=None, + attention_metadata_state=( + create_attention_metadata_state() if backend.upper() == "TRTLLM" else None + ), + parallel=ParallelConfig(ulysses_size=ulysses_size), + skip_create_weights_in_init=False, + ) + config.mapping = vgm.to_llm_mapping() + return config + + +def _init_all_weights(model: torch.nn.Module, std: float = 0.02): + """Init weights to small values — TRT-LLM Linear uses empty() (uninit mem).""" + with torch.no_grad(): + for name, p in model.named_parameters(): + if "norm" in name and "weight" in name: + p.fill_(1.0) + elif p.numel() > 0: + torch.nn.init.normal_(p, mean=0.0, std=std) + + +def _make_video_positions( + batch: int, n_patches: int, n_frames: int, grid_h: int, grid_w: int, device: torch.device +) -> torch.Tensor: + positions = torch.zeros(batch, 3, n_patches, 2, device=device) + idx = 0 + for f in range(n_frames): + for h in range(grid_h): + for w in range(grid_w): + positions[:, 0, idx, :] = torch.tensor([f, f + 1], dtype=torch.float32) + positions[:, 1, idx, :] = torch.tensor([h, h + 1], dtype=torch.float32) + positions[:, 2, idx, :] = torch.tensor([w, w + 1], dtype=torch.float32) + idx += 1 + return positions + + +def _make_audio_positions(batch: int, n_patches: int, device: torch.device) -> torch.Tensor: + positions = torch.zeros(batch, 1, n_patches, 2, device=device) + for i in range(n_patches): + positions[:, 0, i, :] = torch.tensor([i, i + 1], dtype=torch.float32) + return positions + + +# ============================================================================= +# Test logic +# ============================================================================= + + +def _build_inputs(batch, v_patches, v_dims, a_patches, dtype, device, seed=456): + """Construct identical inputs across all ranks via shared seed.""" + g = torch.Generator(device=device).manual_seed(seed) + v_frames, v_h, v_w = v_dims + in_channels = _AV_CONFIG["in_channels"] + audio_in_channels = _AV_CONFIG["audio_in_channels"] + caption_channels = _AV_CONFIG["caption_channels"] + text_len = 8 + + v_context = ( + torch.randn(batch, text_len, caption_channels, device=device, dtype=dtype, generator=g) + * 0.02 + ) + a_context = ( + torch.randn(batch, text_len, caption_channels, device=device, dtype=dtype, generator=g) + * 0.02 + ) + v_positions = _make_video_positions(batch, v_patches, v_frames, v_h, v_w, device) + a_positions = _make_audio_positions(batch, a_patches, device) + + from tensorrt_llm._torch.visual_gen.models.ltx2.ltx2_core.modality import Modality + + video = Modality( + latent=torch.randn(batch, v_patches, in_channels, device=device, dtype=dtype, generator=g) + * 0.02, + timesteps=torch.tensor([0.5], device=device), + positions=v_positions, + context=v_context, + ) + audio = Modality( + latent=torch.randn( + batch, a_patches, audio_in_channels, device=device, dtype=dtype, generator=g + ) + * 0.02, + timesteps=torch.tensor([0.5], device=device), + positions=a_positions, + context=a_context, + ) + return video, audio, v_context, a_context, v_positions, a_positions + + +def _logic_ltx2_av_ulysses_vs_single_gpu(rank, world_size, backend, audio_seq_len): + """LTX-2 AudioVideo Ulysses (ws=2) vs single-GPU reference parity. + + Builds the same LTXModel weights on both ranks (shared seed), runs: + 1. Reference: ulysses_size=1, no collectives, computed locally. + 2. Ulysses: ulysses_size=2, collective forward across ranks. + Compares (video_out, audio_out). Drift comes from BF16 accumulation order. + """ + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType + + device = torch.device(f"cuda:{rank}") + dtype = torch.bfloat16 + + batch = 1 + v_dims = (1, 4, 4) # n_frames, grid_h, grid_w + v_patches = v_dims[0] * v_dims[1] * v_dims[2] # 16, divisible by ws=2 + + # ── Reference: single-GPU (no Ulysses) ────────────────────────────────── + torch.manual_seed(123) + ref_cfg = _make_model_config(ulysses_size=1, backend=backend) + ref_model = ( + LTXModel(model_type=LTXModelType.AudioVideo, model_config=ref_cfg, **_AV_CONFIG) + .to(device, dtype=dtype) + .eval() + ) + _init_all_weights(ref_model) + ref_model.configure_audio_ulysses(audio_seq_len) + ref_state = ref_model.state_dict() + + # ── Ulysses model: same weights ───────────────────────────────────────── + torch.manual_seed(123) + u_cfg = _make_model_config(ulysses_size=world_size, backend=backend) + u_model = ( + LTXModel(model_type=LTXModelType.AudioVideo, model_config=u_cfg, **_AV_CONFIG) + .to(device, dtype=dtype) + .eval() + ) + u_model.load_state_dict(ref_state) + u_model.configure_audio_ulysses(audio_seq_len) + + # ── Inputs (identical across ranks) ───────────────────────────────────── + video, audio, v_ctx, a_ctx, v_pos, a_pos = _build_inputs( + batch, v_patches, v_dims, audio_seq_len, dtype, device + ) + + ref_cache = ref_model.prepare_text_cache( + video_context=v_ctx, + video_positions=v_pos, + audio_context=a_ctx, + audio_positions=a_pos, + dtype=dtype, + ) + u_cache = u_model.prepare_text_cache( + video_context=v_ctx, + video_positions=v_pos, + audio_context=a_ctx, + audio_positions=a_pos, + dtype=dtype, + ) + + with torch.no_grad(): + ref_v, ref_a = ref_model(video=video, audio=audio, text_cache=ref_cache) + u_v, u_a = u_model(video=video, audio=audio, text_cache=u_cache) + + # Output shapes match (audio padded tail is stripped inside forward()). + assert ref_v.shape == u_v.shape, f"Rank {rank}: video shape mismatch" + assert ref_a.shape == u_a.shape, f"Rank {rank}: audio shape mismatch" + + # BF16 drift through 2 transformer layers + Ulysses collectives lands in + # the ~1-3% range with these small-scale weights; 5e-2 leaves headroom + # without being so loose that real numerical regressions slip through. + torch.testing.assert_close( + u_v, + ref_v, + rtol=5e-2, + atol=5e-2, + msg=f"Rank {rank}: LTX-2 AV Ulysses video output differs from single-GPU ref", + ) + torch.testing.assert_close( + u_a, + ref_a, + rtol=5e-2, + atol=5e-2, + msg=f"Rank {rank}: LTX-2 AV Ulysses audio output differs from single-GPU ref", + ) + + +# ============================================================================= +# Test classes +# ============================================================================= + + +class TestLTX2AVUlysses: + """End-to-end Ulysses parity for LTX-2 AudioVideo model.""" + + @pytest.mark.parametrize( + "backend", + ["VANILLA", "FA4"], + ) + def test_av_ulysses_no_audio_pad(self, backend): + """ws=2, audio_seq_len % 2 == 0 — pure sharding, no padding mask.""" + run_test_in_distributed(2, _logic_ltx2_av_ulysses_vs_single_gpu, backend, 16) + + @pytest.mark.parametrize( + "backend", + ["VANILLA", "FA4"], + ) + def test_av_ulysses_audio_pad(self, backend): + """ws=2, audio_seq_len % 2 != 0 — audio padding + key_padding_mask.""" + run_test_in_distributed(2, _logic_ltx2_av_ulysses_vs_single_gpu, backend, 15) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py index 0e8f4e830c87..c9940b5d614b 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py @@ -310,6 +310,67 @@ def _logic_ulysses_vs_standard_multi_gpu(rank, world_size): ) +def _logic_ulysses_with_key_padding_mask_parity(rank, world_size): + """UlyssesAttention forwards ``key_padding_mask`` through a2a + sharded SDPA; + valid Q rows match plain SDPA on the unpadded prefix. + + Mirrors LTX-2 audio_attn1 under Ulysses padding: audio is padded so the + seq dim divides ulysses_size, each rank holds + ``[B, S_padded/U, H, D]``, and ``key_padding_mask=[B, S_padded]`` zeroes + the padded K/V columns inside the wrapped backend. Catches regressions + where the wrapper would consume / shadow / misalign the mask across the + a2a + reverse-a2a pipeline. + """ + batch = 2 + s_real = 30 # deliberately not divisible by world_size + pad = (world_size - s_real % world_size) % world_size + s_padded = s_real + pad + seq_per_rank = s_padded // world_size + num_heads = world_size * 4 + head_dim = 64 + # CPU + gloo (use_cuda=False): the test validates wrapper math, not + # kernel perf. Forcing CPU avoids ``cuda:rank`` ordinal errors when fewer + # GPUs are visible than spawned ranks. + device = torch.device("cpu") + + # Identical full tensors on every rank (same seed); the [s_real, s_padded) + # tail is junk but mask must zero its contribution. + torch.manual_seed(42) + x_full = torch.randn(batch, s_padded, num_heads, head_dim, device=device) + mask = torch.zeros(batch, s_padded, dtype=torch.bool, device=device) + mask[:, :s_real] = True + + # Reference: plain SDPA on the unpadded prefix. + ref = F.scaled_dot_product_attention( + x_full[:, :s_real].transpose(1, 2), + x_full[:, :s_real].transpose(1, 2), + x_full[:, :s_real].transpose(1, 2), + scale=1.0 / math.sqrt(head_dim), + dropout_p=0.0, + ).transpose(1, 2) # [B, s_real, H, D] + + # Wrapped path: this rank holds the [rank*seq_per_rank, (rank+1)*seq_per_rank) shard. + inner = VanillaAttention(num_heads=num_heads // world_size, head_dim=head_dim) + attention = UlyssesAttention(inner_backend=inner, process_group=None) + x_shard = x_full[:, rank * seq_per_rank : (rank + 1) * seq_per_rank].contiguous() + out_shard = attention.forward(q=x_shard, k=x_shard, v=x_shard, key_padding_mask=mask) + + # Compare this rank's valid Q rows (those that fall within s_real) to ref. + start, end = rank * seq_per_rank, (rank + 1) * seq_per_rank + valid_in_shard = max(0, min(end, s_real) - start) + if valid_in_shard > 0: + torch.testing.assert_close( + out_shard[:, :valid_in_shard], + ref[:, start : start + valid_in_shard], + rtol=1e-4, + atol=1e-4, + msg=( + f"Rank {rank}: UlyssesAttention with key_padding_mask diverges " + f"from unpadded SDPA on valid Q rows" + ), + ) + + def _logic_ulysses_invalid_heads(rank, world_size): """Invalid head count (not divisible by world_size) cannot be sharded.""" assert rank >= 0 and rank < world_size @@ -626,6 +687,21 @@ def test_ulysses_vs_standard_attention_multi_gpu(self): world_size=2, test_fn=_logic_ulysses_vs_standard_multi_gpu, use_cuda=True ) + def test_ulysses_with_key_padding_mask_parity(self): + """Padded Q=K=V + key_padding_mask: valid Q rows match unpadded SDPA. + + Verifies that ``UlyssesAttention`` correctly forwards + ``key_padding_mask`` through its a2a + sharded SDPA + reverse-a2a + pipeline (the audio_attn1 path under Ulysses with audio padding). + ws=2 exercises a single sharding boundary; CPU + gloo so the test + runs without GPUs. + """ + run_test_in_distributed( + world_size=2, + test_fn=_logic_ulysses_with_key_padding_mask_parity, + use_cuda=False, + ) + def test_ulysses_attention_invalid_heads(self): """Test that invalid head count raises error.""" run_test_in_distributed(world_size=2, test_fn=_logic_ulysses_invalid_heads, use_cuda=False) diff --git a/tests/unittest/_torch/visual_gen/test_fa4_key_padding_mask.py b/tests/unittest/_torch/visual_gen/test_fa4_key_padding_mask.py new file mode 100644 index 000000000000..f0923cfb5042 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_fa4_key_padding_mask.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Numerical parity for FlashAttn4Attention's key_padding_mask. + +FA4 is used for LTX-2 audio self-attn and a2v cross-attn when the user picks +``attention.backend: FA4``. Under Ulysses, audio is padded so T_a is divisible +by ulysses_size, and padded K columns must produce zero attention +contribution. FA4's cute interface accepts +``seqused_k`` per-batch valid lengths; this test verifies that translating a +True-prefix bool mask to ``seqused_k`` yields output identical to running FA4 +on the unpadded K/V (within bf16 tolerance). + +Requires CUDA (FA4 is GPU-only). +""" + +import pytest +import torch + +try: + from tensorrt_llm._torch.visual_gen.attention_backend.flash_attn4 import ( + FlashAttn4Attention, + _flash_attn_fwd, + ) + + FA4_AVAILABLE = _flash_attn_fwd is not None +except ImportError: + FA4_AVAILABLE = False + +pytestmark = [ + pytest.mark.skipif(not torch.cuda.is_available(), reason="FA4 requires CUDA"), + pytest.mark.skipif(not FA4_AVAILABLE, reason="FA4 kernel not available"), +] + + +def _run_self_attn(B, S_real, S_pad, H, d_h, dtype=torch.bfloat16): + """Helper: self-attention case (Q=K=V, padded only on the seq dim).""" + device = "cuda" + torch.manual_seed(0) + S_full = S_real + S_pad + + # Build a padded sequence with random valid prefix + arbitrary junk suffix. + x_valid = torch.randn(B, S_real, H, d_h, dtype=dtype, device=device) + x_pad = torch.randn(B, S_pad, H, d_h, dtype=dtype, device=device) + x_full = torch.cat([x_valid, x_pad], dim=1) + mask = torch.zeros(B, S_full, dtype=torch.bool, device=device) + mask[:, :S_real] = True + + fa4 = FlashAttn4Attention(num_heads=H, head_dim=d_h, num_kv_heads=H) + + # FA4 path with seqused_k (q_full = k_full = v_full = x_full padded; only valid prefix attends). + out_padded = fa4.forward(q=x_full, k=x_full, v=x_full, key_padding_mask=mask) + + # Reference: FA4 on unpadded inputs only (Q seq = K seq = S_real). + out_ref = fa4.forward(q=x_valid, k=x_valid, v=x_valid) + + # Only the valid Q rows must match; pad Q rows are stripped downstream by + # the caller and are not part of the contract. + return out_padded[:, :S_real], out_ref + + +def _run_cross_attn(B, S_q, S_real_kv, S_pad_kv, H, d_h, dtype=torch.bfloat16): + """Helper: cross-attention case (Q full, K/V padded on seq dim).""" + device = "cuda" + torch.manual_seed(1) + S_full_kv = S_real_kv + S_pad_kv + + q = torch.randn(B, S_q, H, d_h, dtype=dtype, device=device) + k_valid = torch.randn(B, S_real_kv, H, d_h, dtype=dtype, device=device) + v_valid = torch.randn(B, S_real_kv, H, d_h, dtype=dtype, device=device) + k_pad = torch.randn(B, S_pad_kv, H, d_h, dtype=dtype, device=device) + v_pad = torch.randn(B, S_pad_kv, H, d_h, dtype=dtype, device=device) + k_full = torch.cat([k_valid, k_pad], dim=1) + v_full = torch.cat([v_valid, v_pad], dim=1) + mask = torch.zeros(B, S_full_kv, dtype=torch.bool, device=device) + mask[:, :S_real_kv] = True + + fa4 = FlashAttn4Attention(num_heads=H, head_dim=d_h, num_kv_heads=H) + + out_padded = fa4.forward(q=q, k=k_full, v=v_full, key_padding_mask=mask) + out_ref = fa4.forward(q=q, k=k_valid, v=v_valid) + return out_padded, out_ref + + +def test_self_attn_padded_kv_with_mask_matches_unpadded(): + """FA4 self-attn: masked padded K/V matches unpadded K/V on valid Q rows.""" + out, ref = _run_self_attn(B=2, S_real=126, S_pad=2, H=8, d_h=64) + torch.testing.assert_close( + out, + ref, + rtol=2e-3, + atol=2e-3, + msg="FA4 self-attn key_padding_mask diverges from unpadded SDPA", + ) + + +def test_cross_attn_padded_kv_with_mask_matches_unpadded(): + """FA4 cross-attn: padded K/V + mask matches unpadded K/V (matches a2v use).""" + out, ref = _run_cross_attn(B=2, S_q=320, S_real_kv=126, S_pad_kv=2, H=8, d_h=64) + torch.testing.assert_close( + out, + ref, + rtol=2e-3, + atol=2e-3, + msg="FA4 cross-attn key_padding_mask diverges from unpadded SDPA", + ) + + +def test_self_attn_pad_junk_values_dont_affect_valid_output(): + """Two different junk pad fills with same mask produce same valid-row output.""" + B, S_real, S_pad, H, d_h = 1, 64, 4, 4, 64 + device = "cuda" + S_full = S_real + S_pad + dtype = torch.bfloat16 + + torch.manual_seed(7) + x_valid = torch.randn(B, S_real, H, d_h, dtype=dtype, device=device) + mask = torch.zeros(B, S_full, dtype=torch.bool, device=device) + mask[:, :S_real] = True + fa4 = FlashAttn4Attention(num_heads=H, head_dim=d_h, num_kv_heads=H) + + # Pad fill A: zeros. Pad fill B: random. + pad_a = torch.zeros(B, S_pad, H, d_h, dtype=dtype, device=device) + pad_b = torch.randn(B, S_pad, H, d_h, dtype=dtype, device=device) + x_a = torch.cat([x_valid, pad_a], dim=1) + x_b = torch.cat([x_valid, pad_b], dim=1) + + out_a = fa4.forward(q=x_a, k=x_a, v=x_a, key_padding_mask=mask) + out_b = fa4.forward(q=x_b, k=x_b, v=x_b, key_padding_mask=mask) + + # Q[:S_real] sees the same K/V[:S_real]; mask zeros K/V[S_real:] contribution. + # Only the valid Q rows are part of the contract. + torch.testing.assert_close( + out_a[:, :S_real], + out_b[:, :S_real], + rtol=1e-5, + atol=1e-5, + msg="FA4 pad fill leaked into valid-row output — mask is not fully suppressing pads", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/test_vanilla_key_padding_mask.py b/tests/unittest/_torch/visual_gen/test_vanilla_key_padding_mask.py new file mode 100644 index 000000000000..723126d97475 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_vanilla_key_padding_mask.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Numerical parity for VanillaAttention's key_padding_mask. + +v2a Ulysses pads audio K/V so T_a is divisible by U. The mathematical +guarantee the padding relies on: masking the padded K/V columns via +key_padding_mask must produce identical output to running SDPA on the +unpadded K/V. Single-rank, CPU. +""" + +import pytest +import torch + +from tensorrt_llm._torch.visual_gen.attention_backend import VanillaAttention + + +def test_padded_kv_with_mask_matches_unpadded(): + torch.manual_seed(42) + B, S_q, S_kv_valid, S_pad, H, H_kv, d_h = 2, 12, 7, 5, 8, 8, 64 + S_kv = S_kv_valid + S_pad + + attn = VanillaAttention(num_heads=H, head_dim=d_h, num_kv_heads=H_kv) + + q = torch.randn(B, H, S_q, d_h) + k_valid = torch.randn(B, H_kv, S_kv_valid, d_h) + v_valid = torch.randn(B, H_kv, S_kv_valid, d_h) + + ref = attn.forward(q=q, k=k_valid, v=v_valid) + + # Pad K/V with realistic magnitudes (O(1)). Real audio pad goes through + # K/V Linear from a zero-latent, so it is bounded by the layer's bias norm. + k_padded = torch.cat([k_valid, torch.randn(B, H_kv, S_pad, d_h)], dim=2) + v_padded = torch.cat([v_valid, torch.randn(B, H_kv, S_pad, d_h)], dim=2) + mask = torch.zeros(B, S_kv, dtype=torch.bool) + mask[:, :S_kv_valid] = True + + out = attn.forward(q=q, k=k_padded, v=v_padded, key_padding_mask=mask) + + torch.testing.assert_close( + out, + ref, + rtol=1e-4, + atol=1e-4, + msg="key_padding_mask did not produce numerical parity with unpadded SDPA", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 0a192053d9377944605f6464ffffbcd33c0ca4c4 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Fri, 29 May 2026 16:48:52 +0800 Subject: [PATCH 120/308] [None][refactor] Flatten thop.attention sequence kwargs + rename rotary_embedding_* to rope_* (#14569) Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- cpp/tensorrt_llm/nanobind/thop/bindings.cpp | 16 +- cpp/tensorrt_llm/thop/attentionOp.cpp | 179 ++++++++---------- cpp/tensorrt_llm/thop/attentionOp.h | 20 +- .../_torch/attention_backend/trtllm.py | 102 ++++------ .../custom_ops/attention/trtllm_attention.py | 98 ++++------ .../auto_deploy/custom_ops/mla/trtllm_mla.py | 140 ++++++++------ .../fuse_rope_into_trtllm_attention.py | 6 +- .../test_attention_op_sync.py | 32 ++++ 8 files changed, 300 insertions(+), 293 deletions(-) diff --git a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp index 76c95037e945..3c2e59945aaa 100644 --- a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp @@ -144,16 +144,20 @@ void initBindings(nb::module_& m) nb::arg("num_heads"), nb::arg("num_kv_heads"), nb::arg("head_size"), nb::arg("tokens_per_block").none(), nb::arg("max_num_requests"), nb::arg("max_context_length"), nb::arg("attention_window_size"), nb::arg("beam_width"), nb::arg("mask_type"), nb::arg("quant_mode"), nb::arg("q_scaling"), - nb::arg("position_embedding_type"), nb::arg("rotary_embedding_dim"), nb::arg("rotary_embedding_base"), - nb::arg("rotary_embedding_scale_type"), nb::arg("rotary_embedding_scales"), - nb::arg("rotary_embedding_max_position_info"), nb::arg("use_paged_context_fmha"), + nb::arg("position_embedding_type"), nb::arg("rope_dim"), nb::arg("rope_base"), nb::arg("rope_scale_type"), + nb::arg("rope_scale"), nb::arg("rope_short_m_scale"), nb::arg("rope_long_m_scale"), + nb::arg("rope_max_positions"), nb::arg("rope_original_max_positions"), nb::arg("use_paged_context_fmha"), nb::arg("attention_input_type").none(), nb::arg("is_mla_enable"), nb::arg("chunked_prefill_buffer_batch_size").none(), nb::arg("q_lora_rank").none(), nb::arg("kv_lora_rank").none(), nb::arg("qk_nope_head_dim").none(), nb::arg("qk_rope_head_dim").none(), nb::arg("v_head_dim").none(), nb::arg("rope_append").none(), nb::arg("mrope_rotary_cos_sin").none(), - nb::arg("mrope_position_deltas").none(), nb::arg("helix_tensor_params"), nb::arg("attention_chunk_size").none(), - nb::arg("softmax_stats_tensor").none(), nb::arg("spec_decoding_bool_params"), - nb::arg("spec_decoding_tensor_params"), nb::arg("sparse_kv_indices").none(), + nb::arg("mrope_position_deltas").none(), nb::arg("helix_position_offsets").none(), + nb::arg("helix_is_inactive_rank").none(), nb::arg("attention_chunk_size").none(), + nb::arg("softmax_stats_tensor").none(), nb::arg("is_spec_decoding_enabled"), nb::arg("use_spec_decoding"), + nb::arg("is_spec_dec_tree"), nb::arg("spec_decoding_generation_lengths").none(), + nb::arg("spec_decoding_position_offsets_for_cpp").none(), nb::arg("spec_decoding_packed_mask").none(), + nb::arg("spec_decoding_bl_tree_mask_offset").none(), nb::arg("spec_decoding_bl_tree_mask").none(), + nb::arg("spec_bl_tree_first_sparse_mask_offset_kv").none(), nb::arg("sparse_kv_indices").none(), nb::arg("sparse_kv_offsets").none(), nb::arg("sparse_attn_indices").none(), nb::arg("sparse_attn_offsets").none(), nb::arg("sparse_attn_indices_block_size"), nb::arg("num_sparse_topk") = std::nullopt, nb::arg("sparse_mla_topk_lens") = std::nullopt, diff --git a/cpp/tensorrt_llm/thop/attentionOp.cpp b/cpp/tensorrt_llm/thop/attentionOp.cpp index 79447cc93172..2f6f986daf4e 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.cpp +++ b/cpp/tensorrt_llm/thop/attentionOp.cpp @@ -358,9 +358,14 @@ class RunnerBase torch::optional rotary_cos_sin, torch::optional latent_cache, torch::optional q_pe, torch::optional block_ids_per_seq, torch::optional mrope_rotary_cos_sin, torch::optional mrope_position_deltas, - std::vector> helix_tensor_params, + std::optional helix_position_offsets, std::optional helix_is_inactive_rank, torch::optional softmax_stats_tensor, - c10::ArrayRef> spec_decoding_tensor_params, + std::optional spec_decoding_generation_lengths, + std::optional spec_decoding_position_offsets_for_cpp, + std::optional spec_decoding_packed_mask, + std::optional spec_decoding_bl_tree_mask_offset, + std::optional spec_decoding_bl_tree_mask, + std::optional spec_bl_tree_first_sparse_mask_offset_kv, torch::optional attention_sinks, torch::optional sparse_kv_indices, torch::optional sparse_kv_offsets, torch::optional sparse_attn_indices, torch::optional sparse_attn_offsets, int64_t const sparse_attn_indices_block_size, @@ -420,9 +425,14 @@ class Runner : public RunnerBase torch::optional rotary_cos_sin, torch::optional latent_cache, torch::optional q_pe, torch::optional block_ids_per_seq, torch::optional mrope_rotary_cos_sin, torch::optional mrope_position_deltas, - std::vector> helix_tensor_params, + std::optional helix_position_offsets, std::optional helix_is_inactive_rank, torch::optional softmax_stats_tensor, - c10::ArrayRef> spec_decoding_tensor_params, + std::optional spec_decoding_generation_lengths, + std::optional spec_decoding_position_offsets_for_cpp, + std::optional spec_decoding_packed_mask, + std::optional spec_decoding_bl_tree_mask_offset, + std::optional spec_decoding_bl_tree_mask, + std::optional spec_bl_tree_first_sparse_mask_offset_kv, torch::optional attention_sinks, torch::optional sparse_kv_indices, torch::optional sparse_kv_offsets, torch::optional sparse_attn_indices, torch::optional sparse_attn_offsets, int64_t const sparse_attn_indices_block_size, @@ -462,8 +472,6 @@ class Runner : public RunnerBase [[maybe_unused]] MlaParams mla_params; if (op.isMLAEnabled()) { - TORCH_CHECK(helix_tensor_params.size() == 2, - "Expecting 2 tensors for helix_tensor_params: helix_position_offsets and helix_is_inactive_rank."); if (is_context && op.mUseSparseAttention) { if (latent_cache.has_value()) @@ -507,8 +515,6 @@ class Runner : public RunnerBase mla_params.v_buf = v_ptr; // For generation, helix position is in ropeOp - auto const& helix_position_offsets = helix_tensor_params[0]; - auto const& helix_is_inactive_rank = helix_tensor_params[1]; if (helix_position_offsets.has_value()) { mla_params.helix_position_offsets = helix_position_offsets->data_ptr(); @@ -731,23 +737,18 @@ class Runner : public RunnerBase common_enqueue_params.softmax_stats = static_cast(softmax_stats_tensor.value().data_ptr()); } - // Shared helper to extract helix params from helix_tensor_params into enqueue params. + // Shared helper to wire helix params into the enqueue params. // Works for both EnqueueContextParams and EnqueueGenerationParams since both have // helix_position_offsets and helix_is_inactive_rank fields. - auto const extractHelixParams = [&helix_tensor_params](auto& params) + auto const extractHelixParams = [&helix_position_offsets, &helix_is_inactive_rank](auto& params) { - if (helix_tensor_params.size() == 2) + if (helix_position_offsets.has_value()) { - auto const& helix_position_offsets = helix_tensor_params[0]; - auto const& helix_is_inactive_rank = helix_tensor_params[1]; - if (helix_position_offsets.has_value()) - { - params.helix_position_offsets = helix_position_offsets->data_ptr(); - } - if (helix_is_inactive_rank.has_value()) - { - params.helix_is_inactive_rank = helix_is_inactive_rank->data_ptr(); - } + params.helix_position_offsets = helix_position_offsets->data_ptr(); + } + if (helix_is_inactive_rank.has_value()) + { + params.helix_is_inactive_rank = helix_is_inactive_rank->data_ptr(); } }; @@ -809,52 +810,33 @@ class Runner : public RunnerBase if (op.mIsSpecDecodingEnabled && op.mUseSpecDecoding) { bool useTllmGen = tensorrt_llm::common::isSM100Family(); + TORCH_CHECK(spec_decoding_generation_lengths.has_value(), + "Expecting spec_decoding_generation_lengths in spec-dec mode."); + TORCH_CHECK(spec_decoding_position_offsets_for_cpp.has_value(), + "Expecting spec_decoding_position_offsets_for_cpp in spec-dec mode."); + TORCH_CHECK( + spec_decoding_packed_mask.has_value(), "Expecting spec_decoding_packed_mask in spec-dec mode."); if (useTllmGen) { - TORCH_CHECK(spec_decoding_tensor_params.size() == 6, - "Expecting 6 tensors for spec-dec mode, spec_decoding_generation_lengths, " - "spec_decoding_position_offsets, spec_decoding_packed_mask, " - "spec_decoding_bl_tree_mask_offset, " - "spec_decoding_bl_tree_mask and spec_bl_tree_first_sparse_mask_offset_kv."); - TORCH_CHECK(spec_decoding_tensor_params[0].has_value(), - "Expecting spec_decoding_generation_lengths spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[1].has_value(), - "Expecting spec_decoding_position_offsets spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[2].has_value(), - "Expecting spec_decoding_packed_mask spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[3].has_value(), - "Expecting spec_decoding_bl_tree_mask_offset spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[4].has_value(), - "Expecting spec_decoding_bl_tree_mask spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[5].has_value(), - "Expecting spec_bl_tree_first_sparse_mask_offset_kv spec-dec mode."); + TORCH_CHECK(spec_decoding_bl_tree_mask_offset.has_value(), + "Expecting spec_decoding_bl_tree_mask_offset in trtllm-gen spec-dec mode."); + TORCH_CHECK(spec_decoding_bl_tree_mask.has_value(), + "Expecting spec_decoding_bl_tree_mask in trtllm-gen spec-dec mode."); + TORCH_CHECK(spec_bl_tree_first_sparse_mask_offset_kv.has_value(), + "Expecting spec_bl_tree_first_sparse_mask_offset_kv in trtllm-gen spec-dec mode."); enqueue_params.spec_decoding_bl_tree_mask_offset - = spec_decoding_tensor_params[3].value().data_ptr(); - enqueue_params.spec_decoding_bl_tree_mask - = spec_decoding_tensor_params[4].value().data_ptr(); + = spec_decoding_bl_tree_mask_offset->data_ptr(); + enqueue_params.spec_decoding_bl_tree_mask = spec_decoding_bl_tree_mask->data_ptr(); enqueue_params.spec_bl_tree_first_sparse_mask_offset_kv - = spec_decoding_tensor_params[5].value().data_ptr(); - } - else - { - TORCH_CHECK(spec_decoding_tensor_params.size() == 3, - "Expecting 3 tensors for spec-dec mode, spec_decoding_generation_lengths, " - "spec_decoding_position_offsets and spec_decoding_packed_mask."); - TORCH_CHECK(spec_decoding_tensor_params[0].has_value(), - "Expecting spec_decoding_generation_lengths spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[1].has_value(), - "Expecting spec_decoding_position_offsets spec-dec mode."); - TORCH_CHECK(spec_decoding_tensor_params[2].has_value(), - "Expecting spec_decoding_packed_mask spec-dec mode."); + = spec_bl_tree_first_sparse_mask_offset_kv->data_ptr(); } - enqueue_params.spec_decoding_generation_lengths - = spec_decoding_tensor_params[0].value().data_ptr(); + enqueue_params.spec_decoding_generation_lengths = spec_decoding_generation_lengths->data_ptr(); enqueue_params.spec_decoding_position_offsets - = spec_decoding_tensor_params[1].value().data_ptr(); - enqueue_params.spec_decoding_packed_mask = spec_decoding_tensor_params[2].value().data_ptr(); + = spec_decoding_position_offsets_for_cpp->data_ptr(); + enqueue_params.spec_decoding_packed_mask = spec_decoding_packed_mask->data_ptr(); enqueue_params.spec_decoding_is_generation_length_variable = true; - TLLM_CHECK(spec_decoding_tensor_params[1].value().dim() == 2); // [batch_size, max_draft_len + 1] - enqueue_params.spec_decoding_max_generation_length = spec_decoding_tensor_params[1].value().sizes()[1]; + TLLM_CHECK(spec_decoding_position_offsets_for_cpp->dim() == 2); // [batch_size, max_draft_len + 1] + enqueue_params.spec_decoding_max_generation_length = spec_decoding_position_offsets_for_cpp->sizes()[1]; } // Current mlaGeneration will using fmha to do attention, so we don't go into enqueueGeneration @@ -931,17 +913,23 @@ void attention(torch::Tensor q, std::optional k, std::optional const tokens_per_block, int64_t const max_num_requests, int64_t const max_context_length, int64_t const attention_window_size, int64_t const beam_width, int64_t const mask_type, int64_t const quant_mode, double const q_scaling, - int64_t const position_embedding_type, int64_t const rotary_embedding_dim, double const rotary_embedding_base, - int64_t const rotary_embedding_scale_type, std::vector rotary_embedding_scales, - std::vector rotary_embedding_max_position_info, bool const use_paged_context_fmha, - std::optional attention_input_type, bool is_mla_enable, + int64_t const position_embedding_type, int64_t const rope_dim, double const rope_base, + int64_t const rope_scale_type, double const rope_scale, double const rope_short_m_scale, + double const rope_long_m_scale, int64_t const rope_max_positions, int64_t const rope_original_max_positions, + bool const use_paged_context_fmha, std::optional attention_input_type, bool is_mla_enable, std::optional chunked_prefill_buffer_batch_size, std::optional q_lora_rank, std::optional kv_lora_rank, std::optional qk_nope_head_dim, std::optional qk_rope_head_dim, std::optional v_head_dim, std::optional rope_append, std::optional mrope_rotary_cos_sin, std::optional mrope_position_deltas, - std::vector> helix_tensor_params, std::optional attention_chunk_size, - std::optional softmax_stats_tensor, std::vector spec_decoding_bool_params, - std::vector> spec_decoding_tensor_params, + std::optional helix_position_offsets, std::optional helix_is_inactive_rank, + std::optional attention_chunk_size, std::optional softmax_stats_tensor, + bool const is_spec_decoding_enabled, bool const use_spec_decoding, bool const is_spec_dec_tree, + std::optional spec_decoding_generation_lengths, + std::optional spec_decoding_position_offsets_for_cpp, + std::optional spec_decoding_packed_mask, + std::optional spec_decoding_bl_tree_mask_offset, + std::optional spec_decoding_bl_tree_mask, + std::optional spec_bl_tree_first_sparse_mask_offset_kv, std::optional sparse_kv_indices, std::optional sparse_kv_offsets, std::optional sparse_attn_indices, std::optional sparse_attn_offsets, int64_t const sparse_attn_indices_block_size, std::optional num_sparse_topk, @@ -1035,12 +1023,6 @@ void attention(torch::Tensor q, std::optional k, std::optionalmax_num_requests = max_num_requests; runner->attention_window_size = attention_window_size; - double const rotary_embedding_scale = rotary_embedding_scales[0]; - double const rotary_embedding_short_m_scale = rotary_embedding_scales[1]; - double const rotary_embedding_long_m_scale = rotary_embedding_scales[2]; - int64_t const rotary_embedding_max_positions = rotary_embedding_max_position_info[0]; - int64_t const rotary_embedding_original_max_positions = rotary_embedding_max_position_info[1]; - auto op = std::make_shared(); op->mType = dtype; op->mFMHAForceFP32Acc = dtype == nvinfer1::DataType::kBF16; @@ -1059,15 +1041,14 @@ void attention(torch::Tensor q, std::optional k, std::optionalmQScaling = q_scaling; op->mPositionEmbeddingType = static_cast(int8_t(position_embedding_type)); - op->mRotaryEmbeddingDim = rotary_embedding_dim; - op->mRotaryEmbeddingBase = rotary_embedding_base; - op->mRotaryEmbeddingScaleType - = static_cast(int8_t(rotary_embedding_scale_type)); - op->mRotaryEmbeddingScale = rotary_embedding_scale; - op->mRotaryEmbeddingShortMscale = rotary_embedding_short_m_scale; - op->mRotaryEmbeddingLongMscale = rotary_embedding_long_m_scale; - op->mRotaryEmbeddingMaxPositions = rotary_embedding_max_positions; - op->mRotaryEmbeddingOriginalMaxPositions = rotary_embedding_original_max_positions; + op->mRotaryEmbeddingDim = rope_dim; + op->mRotaryEmbeddingBase = rope_base; + op->mRotaryEmbeddingScaleType = static_cast(int8_t(rope_scale_type)); + op->mRotaryEmbeddingScale = rope_scale; + op->mRotaryEmbeddingShortMscale = rope_short_m_scale; + op->mRotaryEmbeddingLongMscale = rope_long_m_scale; + op->mRotaryEmbeddingMaxPositions = rope_max_positions; + op->mRotaryEmbeddingOriginalMaxPositions = rope_original_max_positions; op->mFP8ContextFMHA = is_fp8_out || is_fp4_out || (op->mKVCacheQuantMode.hasFp8KvCache() && use_paged_context_fmha) || use_sage_attn; // SageAttention block sizes and quantization mode. @@ -1087,11 +1068,9 @@ void attention(torch::Tensor q, std::optional k, std::optionalmSkipSoftmaxTotalBlocks = reinterpret_cast(skip_softmax_stat.value().data_ptr()); op->mSkipSoftmaxSkippedBlocks = op->mSkipSoftmaxTotalBlocks + 1; #endif - TORCH_CHECK(spec_decoding_bool_params.size() == 3, - "Expecting 3 bools for spec-dec mode, is_spec_decoding_enabled, use_spec_decoding, and is_spec_dec_tree."); - op->mIsSpecDecodingEnabled = spec_decoding_bool_params[0]; // is_spec_decoding_enabled - op->mUseSpecDecoding = spec_decoding_bool_params[1]; // use_spec_decoding - op->mIsSpecDecTree = spec_decoding_bool_params[2]; // is_spec_dec_tree + op->mIsSpecDecodingEnabled = is_spec_decoding_enabled; + op->mUseSpecDecoding = use_spec_decoding; + op->mIsSpecDecTree = is_spec_dec_tree; op->mUseSparseAttention = false; op->mUseTllmGenSparseAttentionPaged = false; @@ -1231,11 +1210,14 @@ void attention(torch::Tensor q, std::optional k, std::optional 0) && (attn_input_type != AttentionInputType::ContextOnly)) @@ -1250,11 +1232,14 @@ void attention(torch::Tensor q, std::optional k, std::optional k, std::optional const tokens_per_block, int64_t const max_num_requests, int64_t const max_context_length, int64_t const attention_window_size, int64_t const beam_width, int64_t const mask_type, int64_t const quant_mode, double const q_scaling, - int64_t const position_embedding_type, int64_t const rotary_embedding_dim, double const rotary_embedding_base, - int64_t const rotary_embedding_scale_type, std::vector rotary_embedding_scales, - std::vector rotary_embedding_max_position_info, bool const use_paged_context_fmha, - std::optional attention_input_type, bool is_mla_enable, + int64_t const position_embedding_type, int64_t const rope_dim, double const rope_base, + int64_t const rope_scale_type, double const rope_scale, double const rope_short_m_scale, + double const rope_long_m_scale, int64_t const rope_max_positions, int64_t const rope_original_max_positions, + bool const use_paged_context_fmha, std::optional attention_input_type, bool is_mla_enable, std::optional chunked_prefill_buffer_batch_size, std::optional q_lora_rank, std::optional kv_lora_rank, std::optional qk_nope_head_dim, std::optional qk_rope_head_dim, std::optional v_head_dim, std::optional rope_append, std::optional mrope_rotary_cos_sin, std::optional mrope_position_deltas, - std::vector> helix_tensor_params, std::optional attention_chunk_size, - std::optional softmax_stats_tensor, std::vector spec_decoding_bool_params, - std::vector> spec_decoding_tensor_params, + std::optional helix_position_offsets, std::optional helix_is_inactive_rank, + std::optional attention_chunk_size, std::optional softmax_stats_tensor, + bool const is_spec_decoding_enabled, bool const use_spec_decoding, bool const is_spec_dec_tree, + std::optional spec_decoding_generation_lengths, + std::optional spec_decoding_position_offsets_for_cpp, + std::optional spec_decoding_packed_mask, + std::optional spec_decoding_bl_tree_mask_offset, + std::optional spec_decoding_bl_tree_mask, + std::optional spec_bl_tree_first_sparse_mask_offset_kv, std::optional sparse_kv_indices, std::optional sparse_kv_offsets, std::optional sparse_attn_indices, std::optional sparse_attn_offsets, int64_t const sparse_attn_indices_block_size, std::optional num_sparse_topk, diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index aedd4c839d03..7b819106eb0b 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -157,22 +157,6 @@ def effective_workspace(self) -> Optional[torch.Tensor]: """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace - @property - def helix_tensor_params(self) -> List[Optional[torch.Tensor]]: - """``[helix_position_offsets, helix_is_inactive_rank]`` — the positional - helix tensor list expected by the C++ attention op.""" - return [self.helix_position_offsets, self.helix_is_inactive_rank] - - @property - def spec_decoding_bool_params(self) -> List[bool]: - """``[is_spec_decoding_enabled, use_spec_decoding, is_spec_dec_tree]`` — - the positional bool list expected by the C++ attention op.""" - return [ - self.is_spec_decoding_enabled, - self.use_spec_decoding, - self.is_spec_dec_tree, - ] - @property def spec_decoding_position_offsets_for_cpp(self) -> Optional[torch.Tensor]: """``spec_decoding_position_offsets`` reshaped to the 2D layout the C++ @@ -1051,22 +1035,6 @@ def generate_spec_decoding_generation_length(self, runtime_draft_len): def is_sm_version_trtllm_gen_kernel(self, sm): return not (sm < 100 or sm in [120, 121]) - @property - def spec_decoding_tensor_params(self) -> List[Optional[torch.Tensor]]: - """Positional spec-decoding tensor list for the C++ attention op. - Includes three Blackwell-tree mask tensors on SM versions that take - the trtllm-gen kernel.""" - params = [ - self.spec_decoding_generation_lengths, - self.spec_decoding_position_offsets_for_cpp, - self.spec_decoding_packed_mask, - ] - if self.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()): - params.append(self.spec_decoding_bl_tree_mask_offset) - params.append(self.spec_decoding_bl_tree_mask) - params.append(self.spec_bl_tree_first_sparse_mask_offset_kv) - return params - class TrtllmAttention(AttentionBackend[TrtllmAttentionMetadata]): @@ -1332,35 +1300,36 @@ def create_output(self, q, *, is_quantize_output: bool, ] @property - def rotary_embedding_dim(self) -> int: + def rope_dim(self) -> int: return self.rope_params.dim @property - def rotary_embedding_base(self) -> float: + def rope_base(self) -> float: return self.rope_params.theta @property - def rotary_embedding_scale_type(self) -> int: + def rope_scale_type(self) -> int: return int(self.rope_params.scale_type) @property - def rotary_embedding_scales(self) -> List[float]: - """``[scale, short_m_scale, long_m_scale]`` — the positional RoPE-scale - list expected by the C++ attention op.""" - return [ - self.rope_params.scale, - self.rope_params.short_m_scale, - self.rope_params.long_m_scale, - ] + def rope_scale(self) -> float: + return self.rope_params.scale @property - def rotary_embedding_max_position_info(self) -> List[int]: - """``[max_positions, original_max_positions]`` — the positional - RoPE-positions list expected by the C++ attention op.""" - return [ - self.rope_params.max_positions, - self.rope_params.original_max_positions, - ] + def rope_short_m_scale(self) -> float: + return self.rope_params.short_m_scale + + @property + def rope_long_m_scale(self) -> float: + return self.rope_params.long_m_scale + + @property + def rope_max_positions(self) -> int: + return self.rope_params.max_positions + + @property + def rope_original_max_positions(self) -> int: + return self.rope_params.original_max_positions @property def skip_softmax_threshold_scale_factor_prefill(self) -> Optional[float]: @@ -1530,10 +1499,21 @@ def _run( max_num_requests=metadata.max_num_requests, beam_width=metadata.beam_width, use_paged_context_fmha=metadata.use_paged_context_fmha, - helix_tensor_params=metadata.helix_tensor_params, - spec_decoding_bool_params=metadata.spec_decoding_bool_params, - spec_decoding_tensor_params=metadata. - spec_decoding_tensor_params, + helix_position_offsets=metadata.helix_position_offsets, + helix_is_inactive_rank=metadata.helix_is_inactive_rank, + is_spec_decoding_enabled=metadata.is_spec_decoding_enabled, + use_spec_decoding=metadata.use_spec_decoding, + is_spec_dec_tree=metadata.is_spec_dec_tree, + spec_decoding_generation_lengths=metadata. + spec_decoding_generation_lengths, + spec_decoding_position_offsets_for_cpp=metadata. + spec_decoding_position_offsets_for_cpp, + spec_decoding_packed_mask=metadata.spec_decoding_packed_mask, + spec_decoding_bl_tree_mask_offset=metadata. + spec_decoding_bl_tree_mask_offset, + spec_decoding_bl_tree_mask=metadata.spec_decoding_bl_tree_mask, + spec_bl_tree_first_sparse_mask_offset_kv=metadata. + spec_bl_tree_first_sparse_mask_offset_kv, num_sparse_topk=metadata.num_sparse_topk, flash_mla_tile_scheduler_metadata=metadata. flash_mla_tile_scheduler_metadata, @@ -1584,12 +1564,14 @@ def _run( quant_mode=self.quant_mode, q_scaling=self.q_scaling, position_embedding_type=self.position_embedding_type, - rotary_embedding_dim=self.rotary_embedding_dim, - rotary_embedding_base=self.rotary_embedding_base, - rotary_embedding_scale_type=self.rotary_embedding_scale_type, - rotary_embedding_scales=self.rotary_embedding_scales, - rotary_embedding_max_position_info=self. - rotary_embedding_max_position_info, + rope_dim=self.rope_dim, + rope_base=self.rope_base, + rope_scale_type=self.rope_scale_type, + rope_scale=self.rope_scale, + rope_short_m_scale=self.rope_short_m_scale, + rope_long_m_scale=self.rope_long_m_scale, + rope_max_positions=self.rope_max_positions, + rope_original_max_positions=self.rope_original_max_positions, is_mla_enable=self.is_mla_enable, q_lora_rank=self.q_lora_rank, kv_lora_rank=self.kv_lora_rank, diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py index dfd04e4caefa..f80f78e39357 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/trtllm_attention.py @@ -26,7 +26,7 @@ """ import math -from typing import Any, List, Optional, Tuple +from typing import List, Optional, Tuple import torch from torch._ops import OpOverloadPacket @@ -107,10 +107,10 @@ def __init__(self): # - ``is_spec_dec_active``: spec-dec has been initialized for this planner. # Used as the idempotency guard for ``init_spec_decoding`` and gates # planner-level per-graph work (see use sites for why). - # - ``is_spec_decoding_enabled``: the attention kernel uses spec-dec mode (i.e. - # ``spec_decoding_bool_params[0]`` is True). Mirrors the PyTorch backend's - # ``is_spec_decoding_enabled`` (see ``tensorrt_llm/_torch/attention_backend/ - # trtllm.py:1567-1570``). Forced False on Blackwell (SM100+, except 120/121) + # - ``is_spec_decoding_enabled``: the attention kernel uses spec-dec mode + # (passed as the ``is_spec_decoding_enabled`` kwarg to thop.attention). + # Mirrors the PyTorch backend's ``is_spec_decoding_enabled``. + # Forced False on Blackwell (SM100+, except 120/121) # because thop.attention routes through a trtllm-Gen FMHA spec-dec branch # (``cpp/tensorrt_llm/thop/attentionOp.cpp:499-526``) that requires extra # bl_tree mask tensors AutoDeploy does not produce for Eagle linear chains. @@ -119,22 +119,13 @@ def __init__(self): self.spec_decoding_generation_lengths: Optional[torch.Tensor] = None self.spec_decoding_position_offsets: Optional[torch.Tensor] = None self.spec_decoding_packed_mask: Optional[torch.Tensor] = None - # Pre-packed arg lists for thop.attention's spec-dec parameters. Sized in ``reset()`` - # based on SM version and populated in ``init_spec_decoding`` / per-graph in - # ``prepare_trtllm_metadata``. The attention op reads these directly to avoid the - # per-layer list-building overhead. - # - # spec_decoding_bool_params: - # [0] is_spec_decoding_enabled: kernel enters spec-dec branch (False on Blackwell) - # [1] use_spec_decoding: use spec-dec THIS forward - # [2] is_spec_dec_tree: draft is a tree structure (False for linear Eagle chain) - # - # spec_decoding_tensor_params (first 3 slots; Blackwell+ adds 3 trailing None tree-mask slots): - # [0] generation_lengths: [max_requests] tokens per request this step - # [1] position_offsets: [max_requests, draft_len+1] position offsets per token - # [2] packed_mask: [max_requests, draft_len+1, ceil((draft_len+1)/32)] causal mask - self.spec_decoding_bool_params: List[Any] = [False, False, False] - self.spec_decoding_tensor_params: List[Optional[torch.Tensor]] = [] + # Spec-dec scalars passed directly as thop.attention kwargs. + # - use_spec_decoding gates use of spec-dec THIS forward; set per-batch + # in ``prepare_trtllm_metadata``. + # - The trtllm-Gen FMHA bl_tree mask tensors (last 3 spec_decoding_* + # slots in thop.attention) are always None for AutoDeploy linear + # Eagle chains. + self.use_spec_decoding: bool = False def reset(self, device: torch.device, max_batch: int, max_blocks_per_seq: int) -> None: """One-time allocation of ALL persistent buffers. @@ -168,13 +159,6 @@ def reset(self, device: torch.device, max_batch: int, max_blocks_per_seq: int) - ) self.context_lengths_gpu = torch.zeros(max_batch, dtype=torch.int32, device=device) - # Blackwell+ SMs (excluding 120/121 consumer variants) expect an extended spec-dec tensor - # list with three trailing tree-mask Nones. Size the list accordingly. Contents stay as - # Nones until init_spec_decoding populates the first three entries. - self.spec_decoding_tensor_params = [None] * ( - 6 if _is_blackwell_trtllm_gen_kernel(get_sm_version()) else 3 - ) - def init_spec_decoding(self, max_batch: int, max_draft_len: int) -> None: """Initialize persistent spec-decoding tensors once when configured. @@ -204,19 +188,12 @@ def init_spec_decoding(self, max_batch: int, max_draft_len: int) -> None: ) self.spec_decoding_packed_mask = _generate_spec_decoding_packed_mask(max_batch, draft_len) - # Populate the pre-packed spec-dec params (static references). Only do this when - # the kernel actually enters the spec-dec branch; on Blackwell we leave the bools - # False and the tensor slots None so attentionOp.cpp's spec-dec block is skipped. + # On Blackwell, ``is_spec_decoding_enabled`` stays False so attentionOp.cpp's + # spec-dec block is skipped (the trtllm-Gen FMHA path requires bl_tree mask + # tensors AutoDeploy does not produce). ``use_spec_decoding`` is flipped to + # True per-batch during target model forward to match the PyTorch backend. if self.is_spec_decoding_enabled: - # spec_decoding_bool_params[0] gates the C++ spec-dec branch. - # spec_decoding_bool_params[1] is set to True during target model forward to - # match the PyTorch backend. - self.spec_decoding_bool_params[0] = True - self.spec_decoding_bool_params[1] = True - - self.spec_decoding_tensor_params[0] = self.spec_decoding_generation_lengths - self.spec_decoding_tensor_params[1] = self.spec_decoding_position_offsets - self.spec_decoding_tensor_params[2] = self.spec_decoding_packed_mask + self.use_spec_decoding = True def get_layer_tensors( self, @@ -499,9 +476,7 @@ def prepare_trtllm_metadata( if _GlobalTrtllmPlanner.is_spec_dec_active: _GlobalTrtllmPlanner.refresh_batch_state(batch_info) if _GlobalTrtllmPlanner.is_spec_decoding_enabled: - _GlobalTrtllmPlanner.spec_decoding_bool_params[1] = ( - batch_info.get_num_sequences()[2] == 0 - ) + _GlobalTrtllmPlanner.use_spec_decoding = batch_info.get_num_sequences()[2] == 0 block_offset_multiplier = batch_info.get_block_offset_multiplier() _GlobalTrtllmPlanner.plan_device( @@ -659,12 +634,6 @@ def trtllm_mha_with_cache( # Pool mapping (shared, always zeros since layer offset is in pool_pointers) host_kv_cache_pool_mapping = _GlobalTrtllmPlanner.host_pool_mapping - # Pack parameters for thop.attention - rotary_embedding_scales = [1.0, 1.0, 1.0] - rotary_embedding_max_position_info = [max_context_length, max_context_length] - - mla_tensor_params = [None, None] - # For Llama + Eagle3 + torch-cudagraph, we observe a crash without the following # use of _scratch_block_offsets. See: # https://github.com/NVIDIA/TensorRT-LLM/issues/13100 @@ -702,7 +671,7 @@ def trtllm_mha_with_cache( True, # is_fused_qkv True, # update_kv_cache 1, # predicted_tokens_per_seq (always 1 except for MLA kernel) - 0, # layer_idx (always 0; pool_pointers already encodes the layer offset) + 0, # local_layer_idx (always 0; pool_pointers already encodes the layer offset) num_heads, # num_heads num_kv_heads, # num_kv_heads head_dim, # head_size @@ -715,11 +684,14 @@ def trtllm_mha_with_cache( quant_mode, # quant_mode scale * math.sqrt(head_dim) if scale is not None else 1.0, # q_scaling position_embedding_type, # position_embedding_type - rotary_embedding_dim, # rotary_embedding_dim - 10000.0, # rotary_embedding_base - 0, # rotary_embedding_scale_type - rotary_embedding_scales, # rotary_embedding_scales - rotary_embedding_max_position_info, # rotary_embedding_max_position_info + rotary_embedding_dim, # rope_dim + 10000.0, # rope_base + 0, # rope_scale_type + 1.0, # rope_scale + 1.0, # rope_short_m_scale + 1.0, # rope_long_m_scale + max_context_length, # rope_max_positions + max_context_length, # rope_original_max_positions True, # use_paged_context_fmha 0, # attention_input_type False, # is_mla_enable @@ -732,11 +704,19 @@ def trtllm_mha_with_cache( None, # rope_append None, # mrope_rotary_cos_sin None, # mrope_position_deltas - mla_tensor_params, # mla_tensor_params + None, # helix_position_offsets + None, # helix_is_inactive_rank None, # attention_chunk_size None, # softmax_stats_tensor - _GlobalTrtllmPlanner.spec_decoding_bool_params, # spec_decoding_bool_params - _GlobalTrtllmPlanner.spec_decoding_tensor_params, # spec_decoding_tensor_params + _GlobalTrtllmPlanner.is_spec_decoding_enabled, # is_spec_decoding_enabled + _GlobalTrtllmPlanner.use_spec_decoding, # use_spec_decoding + False, # is_spec_dec_tree (always False for AutoDeploy linear Eagle chains) + _GlobalTrtllmPlanner.spec_decoding_generation_lengths, + _GlobalTrtllmPlanner.spec_decoding_position_offsets, + _GlobalTrtllmPlanner.spec_decoding_packed_mask, + None, # spec_decoding_bl_tree_mask_offset (None for AutoDeploy linear Eagle chains) + None, # spec_decoding_bl_tree_mask + None, # spec_bl_tree_first_sparse_mask_offset_kv None, # sparse_kv_indices None, # sparse_kv_offsets None, # sparse_attn_indices @@ -994,7 +974,7 @@ def get_constants(cls, source_attn_node: Node) -> List[Constant]: if rope_info is not None: rope_cos_sin = rope_info["cos_sin_node"] pos_emb_type = rope_info["position_embedding_type"] - rot_emb_dim = rope_info["rotary_embedding_dim"] + rot_emb_dim = rope_info["rope_dim"] else: rope_cos_sin = None pos_emb_type = 0 diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py index d36d030cb402..a7f3a33386b2 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/mla/trtllm_mla.py @@ -69,7 +69,7 @@ from tensorrt_llm.functional import AttentionMaskType from tensorrt_llm.quantization import QuantMode -from ..._compat import KvCacheConfig, get_sm_version, prefer_pinned +from ..._compat import KvCacheConfig, prefer_pinned from ...utils.cuda_graph import cuda_graph_state from ...utils.node_utils import extract_op_args from ..attention_interface import ( @@ -990,15 +990,6 @@ def _handle_prefill_thop( # replays: this Python body would otherwise only execute at capture # time. - sm_version = get_sm_version() - rotary_embedding_scales = [1.0, 1.0, 1.0] - rotary_embedding_max_position_info = [max_context_length, max_context_length] - spec_decoding_bool_params = [False, False, False] - spec_decoding_tensor_params = [None, None, None] - if sm_version >= 89: - spec_decoding_tensor_params.extend([None, None, None]) - mla_tensor_params = [None, None] - # NOTE: Do NOT clone V here. The V tensor must be a non-contiguous view # from kv.split() with token stride = numHeads * (qk_nope + v_head_dim). # The C++ FP8 quantize kernel and FMHA kernel read V using this stride. @@ -1056,11 +1047,14 @@ def _handle_prefill_thop( quant_mode, # quant_mode q_scaling, # q_scaling int(PositionEmbeddingType.yarn), # position_embedding_type - qk_rope_head_dim, # rotary_embedding_dim - 10000.0, # rotary_embedding_base - 5, # rotary_embedding_scale_type (YaRN) - rotary_embedding_scales, # rotary_embedding_scales - rotary_embedding_max_position_info, # rotary_embedding_max_position_info + qk_rope_head_dim, # rope_dim + 10000.0, # rope_base + 5, # rope_scale_type (YaRN) + 1.0, # rope_scale + 1.0, # rope_short_m_scale + 1.0, # rope_long_m_scale + max_context_length, # rope_max_positions + max_context_length, # rope_original_max_positions True, # use_paged_context_fmha int(AttentionInputType.context_only), # attention_input_type True, # is_mla_enable @@ -1073,11 +1067,19 @@ def _handle_prefill_thop( True, # rope_append None, # mrope_rotary_cos_sin None, # mrope_position_deltas - mla_tensor_params, # helix_tensor_params + None, # helix_position_offsets + None, # helix_is_inactive_rank None, # attention_chunk_size None, # softmax_stats_tensor - spec_decoding_bool_params, # spec_decoding_bool_params - spec_decoding_tensor_params, # spec_decoding_tensor_params + False, # is_spec_decoding_enabled + False, # use_spec_decoding + False, # is_spec_dec_tree + None, # spec_decoding_generation_lengths + None, # spec_decoding_position_offsets_for_cpp + None, # spec_decoding_packed_mask + None, # spec_decoding_bl_tree_mask_offset + None, # spec_decoding_bl_tree_mask + None, # spec_bl_tree_first_sparse_mask_offset_kv None, # sparse_kv_indices None, # sparse_kv_offsets None, # sparse_attn_indices @@ -1244,14 +1246,6 @@ def _handle_prefill_thop_cached_kv( planner._kv_b_proj_grouped_cache[perm_cache_key] = w_grouped # Constants reused across all thop.attention calls below. - sm_version = get_sm_version() - rotary_embedding_scales = [1.0, 1.0, 1.0] - rotary_embedding_max_position_info = [max_context_length, max_context_length] - spec_decoding_bool_params = [False, False, False] - spec_decoding_tensor_params = [None, None, None] - if sm_version >= 89: - spec_decoding_tensor_params.extend([None, None, None]) - mla_tensor_params = [None, None] workspace = planner._select_workspace() chunked_loop_num = planner.chunked_loop_num @@ -1343,11 +1337,14 @@ def _handle_prefill_thop_cached_kv( quant_mode, q_scaling, int(PositionEmbeddingType.yarn), - qk_rope_head_dim, - 10000.0, - 5, - rotary_embedding_scales, - rotary_embedding_max_position_info, + qk_rope_head_dim, # rope_dim + 10000.0, # rope_base + 5, # rope_scale_type (YaRN) + 1.0, # rope_scale + 1.0, # rope_short_m_scale + 1.0, # rope_long_m_scale + max_context_length, # rope_max_positions + max_context_length, # rope_original_max_positions True, # use_paged_context_fmha int(AttentionInputType.context_only), True, # is_mla_enable @@ -1360,11 +1357,19 @@ def _handle_prefill_thop_cached_kv( True, # rope_append None, # mrope_rotary_cos_sin None, # mrope_position_deltas - mla_tensor_params, + None, # helix_position_offsets + None, # helix_is_inactive_rank None, # attention_chunk_size temp_softmax_stats, # softmax_stats_tensor (per-iteration output) - spec_decoding_bool_params, - spec_decoding_tensor_params, + False, # is_spec_decoding_enabled + False, # use_spec_decoding + False, # is_spec_dec_tree + None, # spec_decoding_generation_lengths + None, # spec_decoding_position_offsets_for_cpp + None, # spec_decoding_packed_mask + None, # spec_decoding_bl_tree_mask_offset + None, # spec_decoding_bl_tree_mask + None, # spec_bl_tree_first_sparse_mask_offset_kv None, # sparse_kv_indices None, # sparse_kv_offsets None, # sparse_attn_indices @@ -1460,14 +1465,17 @@ def _handle_prefill_thop_cached_kv( quant_mode, q_scaling, int(PositionEmbeddingType.yarn), - qk_rope_head_dim, - 10000.0, - 5, - rotary_embedding_scales, - rotary_embedding_max_position_info, - True, + qk_rope_head_dim, # rope_dim + 10000.0, # rope_base + 5, # rope_scale_type (YaRN) + 1.0, # rope_scale + 1.0, # rope_short_m_scale + 1.0, # rope_long_m_scale + max_context_length, # rope_max_positions + max_context_length, # rope_original_max_positions + True, # use_paged_context_fmha int(AttentionInputType.context_only), - True, + True, # is_mla_enable _TRTLLM_MLA_CHUNK_BATCH_SIZE, 0, kv_lora_rank, @@ -1477,11 +1485,19 @@ def _handle_prefill_thop_cached_kv( True, # rope_append None, # mrope_rotary_cos_sin None, # mrope_position_deltas - mla_tensor_params, + None, # helix_position_offsets + None, # helix_is_inactive_rank None, # attention_chunk_size temp_softmax_stats, # softmax_stats_tensor - spec_decoding_bool_params, - spec_decoding_tensor_params, + False, # is_spec_decoding_enabled + False, # use_spec_decoding + False, # is_spec_dec_tree + None, # spec_decoding_generation_lengths + None, # spec_decoding_position_offsets_for_cpp + None, # spec_decoding_packed_mask + None, # spec_decoding_bl_tree_mask_offset + None, # spec_decoding_bl_tree_mask + None, # spec_bl_tree_first_sparse_mask_offset_kv None, # sparse_kv_indices None, # sparse_kv_offsets None, # sparse_attn_indices @@ -1665,15 +1681,6 @@ def _handle_decode_impl( output_latent = planner.output_latent[:num_tokens] - sm_version = get_sm_version() - rotary_embedding_scales = [1.0, 1.0, 1.0] - rotary_embedding_max_position_info = [max_context_length, max_context_length] - spec_decoding_bool_params = [False, False, False] - spec_decoding_tensor_params = [None, None, None] - if sm_version >= 89: - spec_decoding_tensor_params.extend([None, None, None]) - mla_tensor_params = [None, None] - thop.attention( fused_q_flat, # q (fused Q for decode) None, # k @@ -1716,11 +1723,14 @@ def _handle_decode_impl( quant_mode, # quant_mode q_scaling, # q_scaling int(PositionEmbeddingType.yarn), # position_embedding_type - qk_rope_head_dim, # rotary_embedding_dim - 10000.0, # rotary_embedding_base - 5, # rotary_embedding_scale_type (YaRN) - rotary_embedding_scales, # rotary_embedding_scales - rotary_embedding_max_position_info, # rotary_embedding_max_position_info + qk_rope_head_dim, # rope_dim + 10000.0, # rope_base + 5, # rope_scale_type (YaRN) + 1.0, # rope_scale + 1.0, # rope_short_m_scale + 1.0, # rope_long_m_scale + max_context_length, # rope_max_positions + max_context_length, # rope_original_max_positions False, # use_paged_context_fmha int(AttentionInputType.generation_only), # attention_input_type True, # is_mla_enable @@ -1733,11 +1743,19 @@ def _handle_decode_impl( True, # rope_append None, # mrope_rotary_cos_sin None, # mrope_position_deltas - mla_tensor_params, # helix_tensor_params + None, # helix_position_offsets + None, # helix_is_inactive_rank None, # attention_chunk_size None, # softmax_stats_tensor - spec_decoding_bool_params, # spec_decoding_bool_params - spec_decoding_tensor_params, # spec_decoding_tensor_params + False, # is_spec_decoding_enabled + False, # use_spec_decoding + False, # is_spec_dec_tree + None, # spec_decoding_generation_lengths + None, # spec_decoding_position_offsets_for_cpp + None, # spec_decoding_packed_mask + None, # spec_decoding_bl_tree_mask_offset + None, # spec_decoding_bl_tree_mask + None, # spec_bl_tree_first_sparse_mask_offset_kv None, # sparse_kv_indices None, # sparse_kv_offsets None, # sparse_attn_indices diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_into_trtllm_attention.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_into_trtllm_attention.py index 0e3d0b0f33a6..bfa4ac66a436 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_into_trtllm_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_into_trtllm_attention.py @@ -254,13 +254,13 @@ def _extract_complex_rope_info(gm: GraphModule, rope_node: Node) -> Optional[Dic return _build_rope_info(fi_tensor, is_neox=False, head_dim=head_dim) -def _convert_to_thop_cos_sin(fi_cache: torch.Tensor, rotary_embedding_dim: int) -> torch.Tensor: +def _convert_to_thop_cos_sin(fi_cache: torch.Tensor, rope_dim: int) -> torch.Tensor: """Convert FlashInfer-format cos_sin_cache to thop.attention format. FlashInfer: ``[max_pos, head_dim]`` — ``[cos_0..cos_{d/2-1}, sin_0..sin_{d/2-1}]`` thop: ``[1, max_pos * dim]`` — interleaved ``[cos_0, sin_0, cos_1, sin_1, ...]`` """ - half = rotary_embedding_dim // 2 + half = rope_dim // 2 cos_part = fi_cache[:, :half] sin_part = fi_cache[:, half:] thop_cache = torch.stack([cos_part, sin_part], dim=-1) @@ -272,7 +272,7 @@ def _build_rope_info(fi_cache: torch.Tensor, is_neox: bool, head_dim: int) -> Di return { "cos_sin_tensor": _convert_to_thop_cos_sin(fi_cache, fi_cache.shape[-1]), "position_embedding_type": 2 if is_neox else 1, - "rotary_embedding_dim": head_dim, + "rope_dim": head_dim, } diff --git a/tests/unittest/_torch/attention_backend/test_attention_op_sync.py b/tests/unittest/_torch/attention_backend/test_attention_op_sync.py index b664eda11ab5..c2f218d2a189 100644 --- a/tests/unittest/_torch/attention_backend/test_attention_op_sync.py +++ b/tests/unittest/_torch/attention_backend/test_attention_op_sync.py @@ -546,3 +546,35 @@ def test_excluded_fields_match_real_fields(): f"AttentionForwardArgs field: {sorted(stale)}. Drop them or " f"restore the field." ) + + +def test_no_sequence_kwargs_at_thop_attention_boundary(): + """``thop.attention`` must not accept ``std::vector<...>`` / + ``c10::ArrayRef<...>`` / ``std::array<...>`` parameters. + + Sequence params couple list position to semantic meaning, which the + other sync tests in this file cannot verify element-by-element. Flat + named params let every slot be checked individually (name, type, + source). If a new sequence param creeps back in, flatten it the same + way ``rotary_embedding_scales`` / ``helix_tensor_params`` / + ``spec_decoding_*_params`` were flattened. + """ + sequence_params = [] + for name, cpp_type in _parse_attention_decl(): + bare = cpp_type.strip() + # _strip_inner_type only strips trailing const/&/*; we want to + # detect outer container types regardless of qualifiers, so look + # at the raw type string with leading qualifiers stripped. + outer = re.sub(r"^(const\s+|volatile\s+)+", "", bare) + outer = re.sub(r"\s*(const|&|\*)\s*$", "", outer) + if ( + outer.startswith("std::vector<") + or outer.startswith("c10::ArrayRef<") + or outer.startswith("std::array<") + ): + sequence_params.append((name, cpp_type)) + assert not sequence_params, ( + "thop.attention must not accept Sequence-typed kwargs. Flatten " + "the following params into their named scalar/tensor components:\n" + + "\n".join(f" - {name}: {t}" for name, t in sequence_params) + ) From 3f21a4828b339dd3dadbf700b09c5094447fcdc3 Mon Sep 17 00:00:00 2001 From: taianz-nv Date: Fri, 29 May 2026 17:53:22 +0800 Subject: [PATCH 121/308] [https://nvbugs/6162857][fix] Use generation metrics for VisualGen perf sanity (#14176) Signed-off-by: Taian Zhang --- tensorrt_llm/serve/openai_server.py | 17 ++++-- tensorrt_llm/serve/openai_video_routes.py | 19 +++++-- .../serve/scripts/benchmark_visual_gen.py | 56 ++++++++++++++++++- tensorrt_llm/serve/visual_gen_metrics.py | 35 ++++++++++++ .../README_test_visual_gen_perf_sanity.md | 10 +++- .../defs/perf/test_visual_gen_perf_sanity.py | 38 +++++++++---- .../defs/perf/visual_gen_perf_utils.py | 26 ++++++--- tests/integration/test_lists/waives.txt | 6 -- .../visual_gen/test_trtllm_serve_endpoints.py | 24 +++++++- 9 files changed, 189 insertions(+), 42 deletions(-) create mode 100644 tensorrt_llm/serve/visual_gen_metrics.py diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index debee5830dc6..c53abe885713 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -86,6 +86,8 @@ from tensorrt_llm.serve.responses_utils import \ request_preprocess as responses_api_request_preprocess from tensorrt_llm.serve.tool_parser.tool_parser_factory import ToolParserFactory +from tensorrt_llm.serve.visual_gen_metrics import \ + build_visual_gen_timing_headers from tensorrt_llm.serve.visual_gen_utils import parse_visual_gen_params from tensorrt_llm.version import __version__ as VERSION from tensorrt_llm.visual_gen import VisualGen @@ -1945,12 +1947,15 @@ async def openai_image_generation(self, request: ImageGenerationRequest, "URL mode is not supported for image generation") latency = time.perf_counter() - image_gen_start # seconds - logger.info( - f"Image {image_id} generated and encoded: " - f"latency={latency:.3f}s generation={getattr(output.metrics, 'generation', 0.0):.3f}s " - f"denoise={getattr(output.metrics, 'denoise', 0.0):.3f}s") - - return JSONResponse(content=response.model_dump()) + metrics = output.metrics + generation = metrics.generation if metrics is not None else 0.0 + denoise = metrics.denoise if metrics is not None else 0.0 + logger.info(f"Image {image_id} generated and encoded: " + f"latency={latency:.3f}s generation={generation:.3f}s " + f"denoise={denoise:.3f}s") + headers = build_visual_gen_timing_headers(metrics) + + return JSONResponse(content=response.model_dump(), headers=headers) except Exception as e: logger.error(traceback.format_exc()) diff --git a/tensorrt_llm/serve/openai_video_routes.py b/tensorrt_llm/serve/openai_video_routes.py index ca64a96138b6..4f739201d276 100644 --- a/tensorrt_llm/serve/openai_video_routes.py +++ b/tensorrt_llm/serve/openai_video_routes.py @@ -24,6 +24,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.media.encoding import resolve_video_format from tensorrt_llm.serve.openai_protocol import VideoGenerationRequest, VideoJob, VideoJobList +from tensorrt_llm.serve.visual_gen_metrics import build_visual_gen_timing_headers from tensorrt_llm.serve.visual_gen_utils import VIDEO_STORE, parse_visual_gen_params from tensorrt_llm.visual_gen.params import VisualGenParams @@ -79,11 +80,15 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response: frame_rate=output.frame_rate or request.fps or params.frame_rate, ) latency = time.perf_counter() - sync_video_start # seconds + metrics = output.metrics + generation = metrics.generation if metrics is not None else 0.0 + denoise = metrics.denoise if metrics is not None else 0.0 logger.info( f"Video {video_id} generated and encoded: " - f"latency={latency:.3f}s generation={getattr(output.metrics, 'generation', 0.0):.3f}s " - f"denoise={getattr(output.metrics, 'denoise', 0.0):.3f}s" + f"latency={latency:.3f}s generation={generation:.3f}s " + f"denoise={denoise:.3f}s" ) + headers = build_visual_gen_timing_headers(metrics) # TODO(TRTLLM-11579): the OpenAI Videos API does not yet define a # multi-file response, so we return only the first video as a file @@ -96,6 +101,7 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response: actual_output_path, media_type=media_type, filename=actual_path.name, + headers=headers, ) except ValueError as e: @@ -271,15 +277,20 @@ async def _generate_video_background( frame_rate=output.frame_rate or request.fps or params.frame_rate, ) latency = time.perf_counter() - background_start # seconds + metrics = output.metrics + generation = metrics.generation if metrics is not None else 0.0 + denoise = metrics.denoise if metrics is not None else 0.0 logger.info( f"Video {video_id} async-generated and encoded: " - f"latency={latency:.3f}s generation={getattr(output.metrics, 'generation', 0.0):.3f}s " - f"denoise={getattr(output.metrics, 'denoise', 0.0):.3f}s" + f"latency={latency:.3f}s generation={generation:.3f}s " + f"denoise={denoise:.3f}s" ) job = await VIDEO_STORE.get(video_id) if job: job.status = "completed" job.completed_at = int(time.time()) + # TODO: Expose VisualGen timing metrics for async jobs once the + # OpenAI video job metadata contract includes server timings. # Store the first path on output_path for single-video # compatibility, and the full list on output_paths. job.output_path = str(saved_paths[0]) diff --git a/tensorrt_llm/serve/scripts/benchmark_visual_gen.py b/tensorrt_llm/serve/scripts/benchmark_visual_gen.py index d63eb6b33d57..9beaf6543b00 100644 --- a/tensorrt_llm/serve/scripts/benchmark_visual_gen.py +++ b/tensorrt_llm/serve/scripts/benchmark_visual_gen.py @@ -35,6 +35,7 @@ import asyncio import gc import json +import math import os import random import sys @@ -59,6 +60,11 @@ load_visual_gen_prompts, print_visual_gen_results, ) +from tensorrt_llm.serve.visual_gen_metrics import ( + SERVER_TIMING_HEADER, + VISUAL_GEN_DENOISE_TIMING, + VISUAL_GEN_GENERATION_TIMING, +) AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60) @@ -107,6 +113,41 @@ def _get_headers() -> dict[str, str]: } +def _parse_server_timing_header(headers: Any) -> dict[str, float]: + """Parse required VisualGen Server-Timing metrics into seconds. + + Online VisualGen perf sanity gates on engine-side generation time, so a + successful response without valid ``Server-Timing`` metadata is treated as + a failed benchmark request instead of silently contributing a zero sample. + """ + value = headers.get(SERVER_TIMING_HEADER) + if value is None: + raise ValueError(f"Missing VisualGen timing response header: {SERVER_TIMING_HEADER}") + + timings = {} + for entry in value.split(","): + parts = [part.strip() for part in entry.split(";")] + name = parts[0] + for parameter in parts[1:]: + key, _, parameter_value = parameter.partition("=") + if key.strip() == "dur": + timings[name] = float(parameter_value) / 1000.0 + break + return timings + + +def _get_server_timing_metric( + timings: dict[str, float], name: str, *, require_positive: bool +) -> float: + """Return a required Server-Timing metric, in seconds.""" + if name not in timings: + raise ValueError(f"Missing VisualGen Server-Timing metric: {name}") + timing = timings[name] + if not math.isfinite(timing) or timing < 0 or (require_positive and timing <= 0): + raise ValueError(f"Invalid VisualGen Server-Timing metric {name}: {timing}") + return timing + + async def _do_post( request_input: VisualGenRequestInput, payload: dict[str, Any], @@ -130,6 +171,17 @@ async def _do_post( await response.read() output.success = True output.latency = time.perf_counter() - st + server_timings = _parse_server_timing_header(response.headers) + output.generation = _get_server_timing_metric( + server_timings, + VISUAL_GEN_GENERATION_TIMING, + require_positive=True, + ) + output.denoise = _get_server_timing_metric( + server_timings, + VISUAL_GEN_DENOISE_TIMING, + require_positive=False, + ) else: body = await response.text() output.error = f"HTTP {response.status}: {body}" @@ -314,7 +366,7 @@ def load_prompts(args: argparse.Namespace) -> list[VisualGenSampleRequest]: def _resolve_num_gpus(args: argparse.Namespace) -> int: """Determine the number of GPUs from explicit arg or server config YAML. - Priority: --num-gpus (explicit) > --extra-visual-gen-options YAML > default 1. + Priority: --num-gpus (explicit) > --visual-gen-args YAML > default 1. """ if args.num_gpus is not None: return args.num_gpus @@ -575,7 +627,7 @@ def main(args: argparse.Namespace): type=int, default=None, help="Number of GPUs used by the server. Overrides the value inferred " - "from --extra-visual-gen-options. Defaults to 1 if neither is given.", + "from --visual-gen-args. Defaults to 1 if neither is given.", ) output_group = parser.add_argument_group("Output") diff --git a/tensorrt_llm/serve/visual_gen_metrics.py b/tensorrt_llm/serve/visual_gen_metrics.py new file mode 100644 index 000000000000..3e05dbb3e323 --- /dev/null +++ b/tensorrt_llm/serve/visual_gen_metrics.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared metadata names for VisualGen serving metrics.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from tensorrt_llm.visual_gen.output import VisualGenMetrics + +SERVER_TIMING_HEADER = "Server-Timing" +VISUAL_GEN_DENOISE_TIMING = "denoise" +VISUAL_GEN_GENERATION_TIMING = "generation" + + +def _server_timing_metric(name: str, duration_seconds: float) -> str: + # Server-Timing ``dur`` is in milliseconds; VisualGenMetrics stores seconds. + return f"{name};dur={duration_seconds * 1000:.6f}" + + +def build_visual_gen_timing_headers( + metrics: Optional["VisualGenMetrics"], +) -> dict[str, str]: + """Build standard Server-Timing headers for VisualGen engine timings.""" + if metrics is None: + return {} + return { + SERVER_TIMING_HEADER: ", ".join( + [ + _server_timing_metric(VISUAL_GEN_GENERATION_TIMING, metrics.generation), + _server_timing_metric(VISUAL_GEN_DENOISE_TIMING, metrics.denoise), + ] + ) + } diff --git a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md index a74a247d67a8..1c0808c41ea3 100644 --- a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md @@ -29,8 +29,14 @@ shared perf regression pipeline. It is responsible for: | List | Contents | |------|----------| | `MAXIMIZE_METRICS` | `d_request_throughput`, `d_per_gpu_throughput` | -| `MINIMIZE_METRICS` | `d_mean_e2e_latency`, `d_median_e2e_latency`, `d_p90_e2e_latency`, `d_p99_e2e_latency` | -| `REGRESSION_METRICS` | `d_mean_e2e_latency` | +| `MINIMIZE_METRICS` | `d_mean_latency`, `d_median_latency`, `d_p90_latency`, `d_p99_latency`, `d_mean_generation`, `d_median_generation`, `d_p90_generation`, `d_p99_generation` | +| `REGRESSION_METRICS` | `d_median_generation` | + +Latency and generation metrics are read from the VisualGen benchmark JSON in +seconds. Latency includes media encoding and persistence overhead, while median +generation tracks the typical engine-side generation time used for regression +checks. Online benchmarks receive generation timing from the VisualGen serving +`Server-Timing` response header and store it in the result JSON. ## Match Keys diff --git a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py index c59216a69bc9..c3e9e08f2d80 100644 --- a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py +++ b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py @@ -18,6 +18,7 @@ import copy import json +import math import os import subprocess import sys @@ -178,6 +179,9 @@ def __init__(self, test_case_name: str, output_dir: str): self.config_file, self.select_pattern, self.upload_to_db = parse_test_case_name( test_case_name ) + self.upload_to_db = self.upload_to_db and bool( + os.environ.get("OPEN_SEARCH_DB_BASE_URL", "") + ) self.raw_gpu_type, self.gpu_type = get_gpu_types() self.config_dir = get_config_dir() @@ -436,7 +440,7 @@ def _build_client_command( host, "--port", str(port), - "--extra-visual-gen-options", + "--visual-gen-args", server_config_path, "--save-result", "--result-dir", @@ -513,9 +517,12 @@ def _validate_benchmark_result( "num_gpus", "request_throughput", "per_gpu_throughput", - "mean_e2e_latency_ms", - "median_e2e_latency_ms", - "percentiles_e2e_latency_ms", + "mean_latency", + "median_latency", + "percentiles_latency", + "mean_generation", + "median_generation", + "percentiles_generation", ] missing_keys = [key for key in required_keys if key not in result_data] if missing_keys: @@ -547,12 +554,23 @@ def _validate_benchmark_result( f"completed={completed_requests}, total={total_requests}" ) - percentiles = result_data.get("percentiles_e2e_latency_ms", {}) - for percentile in ("p90", "p99"): - if percentile not in percentiles: - raise ValueError( - f"Missing {percentile} E2E latency in benchmark result {result_path}" - ) + for metric_name in ("latency", "generation"): + percentiles = result_data.get(f"percentiles_{metric_name}", {}) + for percentile in ("p90", "p99"): + if percentile not in percentiles: + raise ValueError( + f"Missing {percentile} {metric_name} in benchmark result {result_path}" + ) + + for latency_metric in ("mean_latency", "median_latency"): + latency_value = float(result_data[latency_metric]) + if not math.isfinite(latency_value) or latency_value <= 0: + raise ValueError(f"Invalid {latency_metric} in benchmark result {result_path}") + + for generation_metric in ("mean_generation", "median_generation"): + generation_value = float(result_data[generation_metric]) + if not math.isfinite(generation_value) or generation_value <= 0: + raise ValueError(f"Invalid {generation_metric} in benchmark result {result_path}") def _load_benchmark_result( self, diff --git a/tests/integration/defs/perf/visual_gen_perf_utils.py b/tests/integration/defs/perf/visual_gen_perf_utils.py index 44b41c331363..08c7960e15e8 100644 --- a/tests/integration/defs/perf/visual_gen_perf_utils.py +++ b/tests/integration/defs/perf/visual_gen_perf_utils.py @@ -27,14 +27,18 @@ ] MINIMIZE_METRICS = [ - "d_mean_e2e_latency", - "d_median_e2e_latency", - "d_p90_e2e_latency", - "d_p99_e2e_latency", + "d_mean_latency", + "d_median_latency", + "d_p90_latency", + "d_p99_latency", + "d_mean_generation", + "d_median_generation", + "d_p90_generation", + "d_p99_generation", ] REGRESSION_METRICS = [ - "d_mean_e2e_latency", + "d_median_generation", ] MATCH_KEYS = [ @@ -63,10 +67,14 @@ RESULT_METRIC_PATHS = { "d_request_throughput": "request_throughput", "d_per_gpu_throughput": "per_gpu_throughput", - "d_mean_e2e_latency": "mean_e2e_latency_ms", - "d_median_e2e_latency": "median_e2e_latency_ms", - "d_p90_e2e_latency": "percentiles_e2e_latency_ms.p90", - "d_p99_e2e_latency": "percentiles_e2e_latency_ms.p99", + "d_mean_latency": "mean_latency", + "d_median_latency": "median_latency", + "d_p90_latency": "percentiles_latency.p90", + "d_p99_latency": "percentiles_latency.p99", + "d_mean_generation": "mean_generation", + "d_median_generation": "median_generation", + "d_p90_generation": "percentiles_generation.p90", + "d_p99_generation": "percentiles_generation.p99", } diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 9389ea955044..ab89142f9798 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -291,12 +291,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6200257) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6221022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6162857) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] SKIP (https://nvbugs/6162857) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] SKIP (https://nvbugs/6162857) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) diff --git a/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py b/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py index 16eb0c57e493..2b720da90185 100644 --- a/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py +++ b/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py @@ -27,6 +27,7 @@ from tensorrt_llm.serve.openai_protocol import VideoJob from tensorrt_llm.serve.openai_server import _normalize_image_output +from tensorrt_llm.serve.visual_gen_metrics import SERVER_TIMING_HEADER from tensorrt_llm.serve.visual_gen_utils import VIDEO_STORE from tensorrt_llm.visual_gen.output import VisualGenMetrics, VisualGenOutput @@ -68,6 +69,21 @@ def _run_async(coro): loop.close() +def _make_dummy_metrics() -> VisualGenMetrics: + return VisualGenMetrics( + generation=1.25, + pre_denoise=0.125, + denoise=0.75, + post_denoise=0.375, + ) + + +def _assert_visual_gen_server_timing(headers) -> None: + server_timing = headers[SERVER_TIMING_HEADER] + assert "generation;dur=1250.000000" in server_timing + assert "denoise;dur=750.000000" in server_timing + + # --------------------------------------------------------------------------- # Mock VisualGen # --------------------------------------------------------------------------- @@ -121,7 +137,7 @@ def generate(self, inputs=None, params=None) -> VisualGenOutput: image=self._maybe_batch(self._image, n), video=self._maybe_batch(self._video, n), audio=self._audio, - metrics=VisualGenMetrics(), + metrics=_make_dummy_metrics(), ) def generate_async(self, inputs=None, params=None) -> "MockVisualGenResult": @@ -192,7 +208,7 @@ async def aresult(self, timeout=None): image=self._image, video=self._video, audio=self._audio, - metrics=VisualGenMetrics(), + metrics=_make_dummy_metrics(), ) def result(self, timeout=None): @@ -203,7 +219,7 @@ def result(self, timeout=None): image=self._image, video=self._video, audio=self._audio, - metrics=VisualGenMetrics(), + metrics=_make_dummy_metrics(), ) @@ -329,6 +345,7 @@ def test_basic_image_generation_b64(self, image_client): }, ) assert resp.status_code == 200 + _assert_visual_gen_server_timing(resp.headers) data = resp.json() assert "data" in data assert len(data["data"]) >= 1 @@ -614,6 +631,7 @@ def test_basic_sync_video_generation(self, video_client): ) assert resp.status_code == 200 assert resp.headers["content-type"] == "video/mp4" + _assert_visual_gen_server_timing(resp.headers) assert len(resp.content) > 0 def test_sync_video_generation_with_params(self, video_client): From 3e80e336556540540f9f4164f40a3b6a109bc417 Mon Sep 17 00:00:00 2001 From: mpikulski <206748156+ixlmar@users.noreply.github.com> Date: Fri, 29 May 2026 12:57:52 +0200 Subject: [PATCH 122/308] [https://nvbugs/6045177][fix] resolve mypy error (#14689) Signed-off-by: ixlmar <206748156+ixlmar@users.noreply.github.com> --- pyproject.toml | 1 - tensorrt_llm/_torch/pyexecutor/sampler.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2cfd18c49ad5..fd5f14508851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1467,7 +1467,6 @@ follow_imports = "silent" explicit_package_bases = true namespace_packages = true strict = true -warn_unused_ignores = false show_error_codes = true # NB: # . -> 'tensorrt_llm' diff --git a/tensorrt_llm/_torch/pyexecutor/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler.py index ba08c2e89b18..55a1c081cd2a 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler.py @@ -301,7 +301,7 @@ class SampleStateWithMMResult(SampleState[SampleStateTensors, SampleStateTensors @dataclass(kw_only=True, frozen=True, slots=True) -class RequestGroupKey(Generic[GenericStrategyKeyType]): # type: ignore[misc] +class RequestGroupKey(Generic[GenericStrategyKeyType]): strategy_key: GenericStrategyKeyType needs_probs: bool From c7683f2f0ac975e8398cd552b62069a6cf569450 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Fri, 29 May 2026 13:52:26 +0100 Subject: [PATCH 123/308] [None][feat] add Poolside Laguna tool parser (#14638) Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 10 +- .../serve/tool_parser/poolside_v1_parser.py | 318 ++++++++++++++++++ .../serve/tool_parser/tool_parser_factory.py | 3 + .../unittest/llmapi/apps/test_tool_parsers.py | 301 +++++++++++++++++ 4 files changed, 627 insertions(+), 5 deletions(-) create mode 100644 tensorrt_llm/serve/tool_parser/poolside_v1_parser.py diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index a41664bcc424..ef99deacb0c0 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -44,8 +44,8 @@ from tensorrt_llm.mapping import CpType from tensorrt_llm.serve import OpenAIDisaggServer, OpenAIServer from tensorrt_llm.serve.tool_parser import ToolParserFactory -from tensorrt_llm.serve.tool_parser.tool_parser_factory import \ - resolve_auto_tool_parser +from tensorrt_llm.serve.tool_parser.tool_parser_factory import ( + MODEL_TYPE_TO_TOOL_PARSER, resolve_auto_tool_parser) from tensorrt_llm.tools.importlib_utils import import_custom_module_from_dir from tensorrt_llm.usage import config as _telemetry_config from tensorrt_llm.visual_gen import VisualGen @@ -898,11 +898,11 @@ def serve( if tool_parser == "auto": resolved = resolve_auto_tool_parser(model) if resolved is None: + supported_model_types = ", ".join( + sorted(MODEL_TYPE_TO_TOOL_PARSER.keys())) raise click.BadParameter( f"Cannot auto-detect tool parser for model '{model}'. " - f"Supported model types for auto-detection: qwen2, qwen3, " - f"qwen3_moe, qwen3_5, qwen3_5_moe, qwen3_next, deepseek_v3, " - f"deepseek_v32, kimi_k2, kimi_k25, glm4. " + f"Supported model types for auto-detection: {supported_model_types}. " f"Please specify a parser explicitly: " f"{list(ToolParserFactory.parsers.keys())}", param_hint="--tool_parser") diff --git a/tensorrt_llm/serve/tool_parser/poolside_v1_parser.py b/tensorrt_llm/serve/tool_parser/poolside_v1_parser.py new file mode 100644 index 000000000000..9e5d4878d353 --- /dev/null +++ b/tensorrt_llm/serve/tool_parser/poolside_v1_parser.py @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import re +from typing import Any + +from tensorrt_llm.logger import logger +from tensorrt_llm.serve.openai_protocol import ChatCompletionToolsParam as Tool +from tensorrt_llm.serve.tool_parser.base_tool_parser import BaseToolParser +from tensorrt_llm.serve.tool_parser.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from tensorrt_llm.serve.tool_parser.utils import infer_type_from_json_schema + +TOOL_CALL_START = "" # nosec B105 +TOOL_CALL_END = "" # nosec B105 +ARG_KEY_START = "" # nosec B105 +ARG_KEY_END = "" # nosec B105 +ARG_VALUE_START = "" # nosec B105 +ARG_VALUE_END = "" # nosec B105 + + +def _get_argument_type(func_name: str, arg_key: str, tools: list[Tool]) -> str | None: + """Return the JSON-schema type for a tool argument, if declared.""" + name_to_tool = {tool.function.name: tool for tool in tools} + tool = name_to_tool.get(func_name) + if tool is None: + return None + + properties = (tool.function.parameters or {}).get("properties", {}) + if not isinstance(properties, dict): + return None + + schema = properties.get(arg_key) + if not isinstance(schema, dict): + return None + + return infer_type_from_json_schema(schema) + + +def _convert_number(value: str) -> Any: + """Convert a numeric string to int or float.""" + if "." in value or "e" in value.lower(): + return float(value) + return int(value) + + +def _parse_argument_value(value: str, arg_type: str | None) -> Any: + """Parse a raw argument value according to its schema type.""" + value = value.strip() + + if arg_type == "string": + return value + + try: + parsed = json.loads(value) + except (json.JSONDecodeError, ValueError): + parsed = None + else: + if arg_type == "integer" and not isinstance(parsed, bool): + try: + return int(parsed) + except (TypeError, ValueError): + return parsed + if arg_type == "number" and not isinstance(parsed, bool): + try: + return float(parsed) + except (TypeError, ValueError): + return parsed + return parsed + + if arg_type == "integer": + try: + return int(value) + except (TypeError, ValueError): + return value + + if arg_type == "number": + try: + return _convert_number(value) + except (TypeError, ValueError): + return value + + if arg_type == "boolean": + if value.lower() == "true": + return True + if value.lower() == "false": + return False + + return value + + +class PoolsideV1ToolParser(BaseToolParser): + """Tool parser for Poolside Laguna's function call format. + + The Laguna chat template asks the model to emit tool calls as: + + function-name + argument-key + value-of-argument-key + + + Multiple arguments are encoded as repeated ``arg_key`` / ``arg_value`` pairs, + and multiple tool calls are encoded as repeated ``tool_call`` blocks. + """ + + def __init__(self): + super().__init__() + self.bot_token = TOOL_CALL_START + self.eot_token = TOOL_CALL_END + # Match each ... / ... pair, + # allowing either real whitespace or a literal "\n" between the tags. + self.func_arg_regex = re.compile( + rf"{ARG_KEY_START}(.*?){ARG_KEY_END}(?:\\n|\s)*{ARG_VALUE_START}(.*?){ARG_VALUE_END}", + re.DOTALL, + ) + + def has_tool_call(self, text: str) -> bool: + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: list[Tool]) -> StreamingParseResult: + """Parse complete generated text. + + Returns a StreamingParseResult containing normal text with tool blocks + removed and one ToolCallItem per complete Poolside v1 tool call. + """ + if self.bot_token not in text: + return StreamingParseResult(normal_text=text, calls=[]) + + normal_text_parts = [] + match_texts = [] + cursor = 0 + + while cursor < len(text): + bot_idx = text.find(self.bot_token, cursor) + if bot_idx == -1: + normal_text_parts.append(text[cursor:]) + break + + normal_text_parts.append(text[cursor:bot_idx]) + block_end = self._find_complete_tool_call_end(text, bot_idx) + if block_end is None: + # Non-streaming output cannot receive more chunks, so preserve + # the malformed/incomplete tool block as normal assistant text. + normal_text_parts.append(text[bot_idx:]) + break + + match_texts.append(text[bot_idx:block_end]) + cursor = block_end + + calls = [] + for match_text in match_texts: + call = self._parse_tool_call_block(match_text, tools) + if call is not None: + calls.append(call) + + return StreamingParseResult(normal_text="".join(normal_text_parts).strip(), calls=calls) + + def parse_streaming_increment(self, new_text: str, tools: list[Tool]) -> StreamingParseResult: + """Parse one streaming text increment. + + Returns normal text that can be emitted immediately plus tool-call + deltas for any complete tool-call blocks. Incomplete tool calls remain + buffered and produce no tool-call delta. + """ + self._buffer += new_text + normal_text_parts: list[str] = [] + calls: list[ToolCallItem] = [] + + while self._buffer: + bot_idx = self._buffer.find(self.bot_token) + if bot_idx == -1: + if self._ends_with_partial_token(self._buffer, self.bot_token): + break + normal_text_parts.append(self._buffer.replace(self.eot_token, "")) + self._buffer = "" + break + + if bot_idx > 0: + normal_text_parts.append(self._buffer[:bot_idx]) + self._buffer = self._buffer[bot_idx:] + + block_end = self._find_complete_tool_call_end(self._buffer, 0) + if block_end is None: + break + + block = self._buffer[:block_end] + call = self._parse_tool_call_block(block, tools) + if call is not None: + if not self.current_tool_name_sent: + calls.append(self._start_streaming_tool_call(call.name or "")) + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=call.parameters, + ) + ) + self.current_tool_id += 1 + self.current_tool_name_sent = False + + self._buffer = self._buffer[block_end:] + + return StreamingParseResult(normal_text="".join(normal_text_parts), calls=calls) + + def supports_structural_tag(self) -> bool: + """Return whether this parser supports strict structural tags.""" + return False + + def structure_info(self) -> _GetInfoFunc: + """Return structural tag info for guided decoding.""" + raise NotImplementedError() + + def _parse_tool_call_block(self, text: str, tools: list[Tool]) -> ToolCallItem | None: + """Parse one complete ... block. + + Returns a ToolCallItem with JSON-encoded arguments, or None when the + block is malformed or has no function name. + """ + if not text.startswith(self.bot_token) or not text.endswith(self.eot_token): + return None + + inner_text = text[len(self.bot_token) : -len(self.eot_token)] + arg_start_idx = inner_text.find(ARG_KEY_START) + if arg_start_idx == -1: + func_name = inner_text.strip() + func_args = "" + else: + func_name = inner_text[:arg_start_idx].strip() + func_args = inner_text[arg_start_idx:] + + if not func_name: + logger.warning("Empty Poolside v1 tool call name detected, skipping tool call") + return None + + arguments = self._parse_argument_pairs( + self.func_arg_regex.findall(func_args), func_name, tools + ) + tool_indices = self._get_tool_indices(tools) + tool_index = tool_indices.get(func_name, -1) + if tool_index == -1: + logger.warning(f"Model attempted to call undefined function: {func_name}") + + return ToolCallItem( + tool_index=tool_index, + name=func_name, + parameters=json.dumps(arguments, ensure_ascii=False), + ) + + def _parse_argument_pairs(self, pairs, func_name: str, tools: list[Tool]) -> dict[str, Any]: + """Convert regex argument key/value matches into a Python dict. + + Returns argument names mapped to schema-coerced Python values. + """ + arguments = {} + for arg_key, arg_value in pairs: + arg_key = arg_key.strip() + arg_type = _get_argument_type(func_name, arg_key, tools) + arguments[arg_key] = _parse_argument_value(arg_value, arg_type) + return arguments + + def _find_complete_tool_call_end(self, text: str, start_idx: int) -> int | None: + """Find the end of a complete tool-call block. + + Returns the exclusive end index of the matching , or None + if the block is incomplete. text inside is + treated as argument content. + """ + pos = start_idx + len(self.bot_token) + in_arg_value = False + while pos < len(text): + if in_arg_value: + arg_value_end_idx = text.find(ARG_VALUE_END, pos) + if arg_value_end_idx == -1: + return None + pos = arg_value_end_idx + len(ARG_VALUE_END) + in_arg_value = False + continue + + eot_idx = text.find(self.eot_token, pos) + arg_value_start_idx = text.find(ARG_VALUE_START, pos) + # Only treat as the block end when it appears before + # the next ; otherwise it is part of the argument text. + if eot_idx != -1 and (arg_value_start_idx == -1 or eot_idx < arg_value_start_idx): + return eot_idx + len(self.eot_token) + if arg_value_start_idx == -1: + return None + + pos = arg_value_start_idx + len(ARG_VALUE_START) + in_arg_value = True + + return None + + def _start_streaming_tool_call(self, func_name: str) -> ToolCallItem: + """Create the streaming delta that announces a new tool name.""" + if self.current_tool_id == -1: + self.current_tool_id = 0 + + self.current_tool_name_sent = True + return ToolCallItem( + tool_index=self.current_tool_id, + name=func_name, + parameters="", + ) diff --git a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py index fc5affe8bfa2..a0f54fdb2558 100644 --- a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py +++ b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py @@ -11,6 +11,7 @@ from .glm47_parser import Glm47ToolParser from .kimi_k2_tool_parser import KimiK2ToolParser from .minimax_m2_parser import MiniMaxM2ToolParser +from .poolside_v1_parser import PoolsideV1ToolParser from .qwen3_coder_parser import Qwen3CoderToolParser from .qwen3_tool_parser import Qwen3ToolParser @@ -31,6 +32,7 @@ "glm_moe_dsa": "glm47", "gemma4": "gemma4", "gemma4_text": "gemma4", + "laguna": "poolside_v1", } @@ -59,6 +61,7 @@ class ToolParserFactory: "glm4": Glm4ToolParser, "glm47": Glm47ToolParser, "minimax_m2": MiniMaxM2ToolParser, + "poolside_v1": PoolsideV1ToolParser, } @staticmethod diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index b6768334260d..59d0e8b2814a 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -32,6 +32,8 @@ from tensorrt_llm.serve.tool_parser.glm47_parser import Glm47ToolParser from tensorrt_llm.serve.tool_parser.kimi_k2_tool_parser import KimiK2ToolParser from tensorrt_llm.serve.tool_parser.minimax_m2_parser import MiniMaxM2ToolParser +from tensorrt_llm.serve.tool_parser.poolside_v1_parser import \ + PoolsideV1ToolParser from tensorrt_llm.serve.tool_parser.qwen3_coder_parser import \ Qwen3CoderToolParser from tensorrt_llm.serve.tool_parser.qwen3_tool_parser import Qwen3ToolParser @@ -2327,6 +2329,305 @@ def test_create_glm47_parser(self): assert isinstance(parser, Glm47ToolParser) +# ============================================================================ +# PoolsideV1ToolParser Tests +# ============================================================================ + + +class TestPoolsideV1ToolParser(BaseToolParserTestClass): + """Test suite for Poolside Laguna v1 tool calls.""" + + def make_parser(self): + return PoolsideV1ToolParser() + + def make_tool_parser_test_cases(self): + return ToolParserTestCases( + has_tool_call_true=("Some text get_weather\n" + "location\n" + "NYC\n" + ""), + detect_and_parse_single_tool=( + ("Normal text\n" + "get_weather\n" + "location\n" + "NYC\n" + ""), + "Normal text", + "get_weather", + { + "location": "NYC" + }, + ), + detect_and_parse_multiple_tools=( + ("get_weather\n" + "location\n" + "LA\n" + "\n" + "search_web\n" + "query\n" + "AI\n" + ""), + ("get_weather", "search_web"), + ), + detect_and_parse_malformed_tool=("get_weather\n" + "location\n" + "NYC"), + detect_and_parse_with_parameters_key=( + ("search_web\n" + "query\n" + "test\n" + ""), + "search_web", + { + "query": "test" + }, + ), + parse_streaming_increment_partial_bot_token="undefined_func\n" + "arg\n" + "value\n" + ""), + ) + + def test_initialization(self, parser): + assert parser.bot_token == "" + assert parser.eot_token == "" + assert parser.supports_structural_tag() is False + + def test_no_newline_format(self, sample_tools, parser): + text = ("get_weather" + "location" + "Tokyo" + "") + + result = parser.detect_and_parse(text, sample_tools) + + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert json.loads(result.calls[0].parameters) == {"location": "Tokyo"} + + def test_zero_arg_tool_call(self, parser): + tools = [ + ChatCompletionToolsParam( + type="function", + function=FunctionDefinition( + name="get_time", + description="Get current time", + parameters={ + "type": "object", + "properties": {}, + }, + ), + ) + ] + text = "get_time" + + result = parser.detect_and_parse(text, tools) + + assert len(result.calls) == 1 + assert result.calls[0].name == "get_time" + assert json.loads(result.calls[0].parameters) == {} + + def test_detect_and_parse_preserves_suffix(self, sample_tools, parser): + text = ("prefix " + "get_weather\n" + "location\n" + "NYC\n" + "" + " suffix") + + result = parser.detect_and_parse(text, sample_tools) + + assert "prefix" in result.normal_text + assert "suffix" in result.normal_text + assert len(result.calls) == 1 + + def test_detect_and_parse_allows_end_tag_text_in_string_arg( + self, sample_tools, parser): + text = ("search_web\n" + "query\n" + "literal marker\n" + "") + + result = parser.detect_and_parse(text, sample_tools) + + assert result.normal_text == "" + assert len(result.calls) == 1 + assert result.calls[0].name == "search_web" + assert json.loads(result.calls[0].parameters) == { + "query": "literal marker" + } + + def test_schema_aware_argument_coercion(self, parser): + tools = [ + ChatCompletionToolsParam( + type="function", + function=FunctionDefinition( + name="set_values", + description="Set typed values", + parameters={ + "type": "object", + "properties": { + "string_value": { + "type": "string" + }, + "integer_value": { + "type": "integer" + }, + "number_value": { + "type": "number" + }, + "boolean_value": { + "type": "boolean" + }, + "array_value": { + "type": "array" + }, + "object_value": { + "type": "object" + }, + }, + }, + ), + ) + ] + text = ("set_values\n" + "string_value\n" + "true\n" + "integer_value\n" + "42\n" + "number_value\n" + "3.5\n" + "boolean_value\n" + "true\n" + "array_value\n" + "[1, 2]\n" + "object_value\n" + '{"k": 1}\n' + "") + + result = parser.detect_and_parse(text, tools) + params = json.loads(result.calls[0].parameters) + + assert params == { + "string_value": "true", + "integer_value": 42, + "number_value": 3.5, + "boolean_value": True, + "array_value": [1, 2], + "object_value": { + "k": 1 + }, + } + + def test_parse_streaming_increment_split_tags(self, sample_tools, parser): + result = parser.parse_streaming_increment("get_weather\nlocation" + "SF" + "", sample_tools) + assert len(result.calls) == 2 + assert result.calls[0].tool_index == 0 + assert result.calls[0].name == "get_weather" + assert result.calls[0].parameters == "" + assert result.calls[1].tool_index == 0 + assert json.loads(result.calls[1].parameters) == {"location": "SF"} + + def test_parse_streaming_increment_buffers_truncated_tool_call( + self, sample_tools, parser): + result = parser.parse_streaming_increment( + "get_weather\n" + "location\n" + "SF", + sample_tools, + ) + + assert result.normal_text == "" + assert len(result.calls) == 0 + + def test_parse_streaming_increment_allows_end_tag_text_in_string_arg( + self, sample_tools, parser): + result = parser.parse_streaming_increment( + "search_web\n" + "query\n" + "literal marker\n" + "", + sample_tools, + ) + + assert len(result.calls) == 2 + assert result.calls[0].tool_index == 0 + assert result.calls[0].name == "search_web" + assert result.calls[0].parameters == "" + assert result.calls[1].tool_index == 0 + assert json.loads(result.calls[1].parameters) == { + "query": "literal marker" + } + + def test_parse_streaming_increment_multiple_tools(self, sample_tools, + parser): + result = parser.parse_streaming_increment( + "get_weather\n" + "location\n" + "NYC\n" + "" + "search_web\n" + "query\n" + "AI\n" + "", + sample_tools, + ) + + assert [call.name for call in result.calls if call.name] == [ + "get_weather", + "search_web", + ] + assert [call.tool_index for call in result.calls if call.name] == [0, 1] + params = [ + json.loads(call.parameters) for call in result.calls + if call.parameters + ] + assert params == [{"location": "NYC"}, {"query": "AI"}] + + +class TestPoolsideV1ToolParserFactory: + """Test that Poolside v1 parser is registered in the factory.""" + + def test_poolside_v1_registered(self): + from tensorrt_llm.serve.tool_parser.tool_parser_factory import \ + ToolParserFactory + assert "poolside_v1" in ToolParserFactory.parsers + + def test_create_poolside_v1_parser(self): + from tensorrt_llm.serve.tool_parser.tool_parser_factory import \ + ToolParserFactory + parser = ToolParserFactory.create_tool_parser("poolside_v1") + assert isinstance(parser, PoolsideV1ToolParser) + + def test_laguna_model_type_mapping(self): + from tensorrt_llm.serve.tool_parser.tool_parser_factory import \ + MODEL_TYPE_TO_TOOL_PARSER + assert MODEL_TYPE_TO_TOOL_PARSER["laguna"] == "poolside_v1" + + def test_auto_detect_laguna(self, tmp_path): + from tensorrt_llm.serve.tool_parser.tool_parser_factory import \ + resolve_auto_tool_parser + model_dir = tmp_path / "Laguna" + model_dir.mkdir() + (model_dir / "config.json").write_text( + json.dumps({"model_type": "laguna"})) + + assert resolve_auto_tool_parser(str(model_dir)) == "poolside_v1" + + # ============================================================================ # Integration Tests # ============================================================================ From 3d56a4e3996c429e243b319a921f49d8058298c2 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Fri, 29 May 2026 10:20:59 -0500 Subject: [PATCH 124/308] [TRTLLM-10004][chore] Enable NCCL symmetric zero-copy by default (#14472) --- tensorrt_llm/_torch/distributed/ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/distributed/ops.py b/tensorrt_llm/_torch/distributed/ops.py index 8ab87db8e1e2..cb98eaf8d777 100644 --- a/tensorrt_llm/_torch/distributed/ops.py +++ b/tensorrt_llm/_torch/distributed/ops.py @@ -22,12 +22,12 @@ from tensorrt_llm.plugin.plugin import CustomAllReduceHelper # Feature flag: GEMM→NCCL-window zero-copy (writes GEMM output directly into -# the window buffer so the allreduce needs no extra copy). Off by default -# (0); set TLLM_NCCL_SYMMETRIC_ZERO_COPY=1 to enable. +# the window buffer so the allreduce needs no extra copy). On by default +# (1); set TLLM_NCCL_SYMMETRIC_ZERO_COPY=0 to disable. # Evaluated once at import — O(1) module-global lookup on every call, # equivalent to a C++ static-bool cached env var. _NCCL_SYMMETRIC_ZERO_COPY: bool = (os.environ.get( - "TLLM_NCCL_SYMMETRIC_ZERO_COPY", "0") == "1") + "TLLM_NCCL_SYMMETRIC_ZERO_COPY", "1") == "1") _thread_local = threading.local() From cf3c1d96a925e52e90a0245b8d08783ac2454111 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 29 May 2026 23:25:13 +0800 Subject: [PATCH 125/308] [None][infra] Fix cbts tokenmacro b64 (#14718) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 23 ++++++++++++++++------- jenkins/L0_Test.groovy | 26 ++++++++++++++++++++------ jenkins/scripts/cbts/README.md | 22 ++++++++++++++-------- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index e9d45f49daa9..dda93fbfc076 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1,6 +1,7 @@ @Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException +import java.nio.charset.StandardCharsets import groovy.transform.Field import groovy.json.JsonOutput import groovy.json.JsonSlurper @@ -770,15 +771,23 @@ def getCbtsResult(pipeline, testFilter, globalVars) return null } // Piggyback input JSON on testFilter so each L0_Test stage agent can - // re-run main.py and regenerate cbts_test_db/ locally. Capped at - // 256 KB; oversize → drop piggyback, Layer 3 falls back to source. + // re-run main.py and regenerate cbts_test_db/ locally. The payload is + // base64-encoded because the raw JSON contains PR diffs and may include + // ${...} or {...} sequences that the Jenkins tokenmacro plugin would + // try to evaluate when the parent serializes globalVars for the + // Parameterized-Remote-Trigger plugin, raising MacroEvaluationException + // and blocking test dispatch. Capped at 256 KB (post-encoding, since + // that is what travels on the wire); oversize → drop piggyback, + // Layer 3 falls back to source. final int CBTS_INPUT_PIGGYBACK_MAX_BYTES = 256000 - def inputJsonSize = inputJson.length() - if (inputJsonSize <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { - result.cbts_input_json = inputJson - pipeline.echo("CBTS Layer 3: cbts_input_json piggyback enabled (${inputJsonSize} bytes)") + def inputJsonB64 = inputJson.getBytes(StandardCharsets.UTF_8).encodeBase64().toString() + def inputJsonB64Size = inputJsonB64.length() + if (inputJsonB64Size <= CBTS_INPUT_PIGGYBACK_MAX_BYTES) { + result.cbts_input_json_b64 = inputJsonB64 + pipeline.echo("CBTS Layer 3: cbts_input_json_b64 piggyback enabled " + + "(${inputJsonB64Size} bytes encoded, ${inputJson.length()} bytes raw)") } else { - pipeline.echo("CBTS Layer 3: cbts_input_json is ${inputJsonSize} bytes, " + + pipeline.echo("CBTS Layer 3: cbts_input_json_b64 is ${inputJsonB64Size} bytes, " + "exceeds ${CBTS_INPUT_PIGGYBACK_MAX_BYTES}-byte piggyback limit; " + "downstream stages will fall back to source test-db " + "(Layer 2 stage filtering still applies)") diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0751daffcb99..4c8297debbf6 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1,6 +1,7 @@ @Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException +import java.nio.charset.StandardCharsets import groovy.transform.Field import groovy.json.JsonOutput import com.nvidia.bloom.KubernetesManager @@ -2456,16 +2457,29 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" // CBTS Layer 3: regenerate cbts_test_db/ on this stage agent from the - // piggybacked input JSON if not already present. + // piggybacked input JSON if not already present. The piggyback payload is + // base64-encoded on the orchestrator (see getCbtsResult in + // L0_MergeRequest.groovy) to keep tokenmacro from interpreting ${...} or + // {...} fragments inside the PR diff when globalVars is serialized. If + // decoding or regeneration throws (truncated/malformed payload), we + // swallow the error: the override directory will be absent below, the + // overrideYaml check will fail, and renderTestDB falls back to the + // source test-db. def cbts = testFilter[(CBTS_RESULT)] - if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json) { + if (cbts != null && cbts.test_db_dir_override && cbts.cbts_input_json_b64) { def overrideDir = "${llmSrc}/${cbts.test_db_dir_override}" def dirExists = sh(returnStdout: true, script: "test -d ${overrideDir} && echo yes || echo no").trim() if (dirExists != "yes") { - def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") - pipeline.writeFile(file: cbtsInputLocal, text: cbts.cbts_input_json) - sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" - sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py ${cbtsInputLocal} > /dev/null 2>&1 || true" + try { + def cbtsInputJson = new String(cbts.cbts_input_json_b64.decodeBase64(), StandardCharsets.UTF_8) + def cbtsInputLocal = Utils.createTempLocation(pipeline, "./cbts_input.json") + pipeline.writeFile(file: cbtsInputLocal, text: cbtsInputJson) + sh "apt-get update -qq && apt-get install -y -qq python3-yaml || true" + sh "cd ${llmSrc} && python3 jenkins/scripts/cbts/main.py ${cbtsInputLocal} > /dev/null 2>&1 || true" + } catch (Exception e) { + echo "CBTS Layer 3: failed to materialize piggyback payload " + + "(${e.class.simpleName}: ${e.message}); falling back to source test-db" + } } } def testDBPath = "${llmSrc}/tests/integration/test_lists/test-db" diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index aa445265f9d1..9dbb1cdc5f36 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -197,13 +197,19 @@ Decision JSON: `cbts_test_db/` is written on the L0_MergeRequest agent and is not available to downstream `L0_Test-*` pods. To regenerate it per stage: -1. `getCbtsResult` stores the input JSON in `result.cbts_input_json`, - which rides along inside `testFilter`. -2. `renderTestDB` on the stage agent writes it to a temp file and re-runs - `main.py`. Output is deterministic, so each agent gets the same - `cbts_test_db/` as L0_MergeRequest produced. - -If `cbts_input_json` exceeds 256 KB the piggyback is dropped; Layer 3 falls +1. `getCbtsResult` base64-encodes the input JSON and stores it in + `result.cbts_input_json_b64`, which rides along inside `testFilter`. + Encoding is mandatory: the raw payload contains PR diff text which can + include `${...}` or `{...}` fragments (Python f-strings, shell vars, etc.) + that the Jenkins tokenmacro plugin tries to evaluate when the parent + serializes `globalVars` for `Parameterized-Remote-Trigger`, raising + `MacroEvaluationException` and blocking test dispatch. +2. `renderTestDB` on the stage agent decodes `cbts_input_json_b64`, writes + it to a temp file, and re-runs `main.py`. Output is deterministic, so + each agent gets the same `cbts_test_db/` as L0_MergeRequest produced. + +If `cbts_input_json_b64` exceeds 256 KB (post-encoding, the size that +actually travels over the wire) the piggyback is dropped; Layer 3 falls back to the source test-db on each stage agent. Layer 2 still applies. ## Split-collapse heuristic (Layer 2.5) @@ -279,7 +285,7 @@ CBTS defers to the existing filter chain when: YAML edit) - Combined scope is `None` (incompatible mix) - Layer 3 narrowing would empty a block — block keeps original tests -- `cbts_input_json` exceeds 256 KB — Layer 3 falls back per stage +- `cbts_input_json_b64` (post-encoding) exceeds 256 KB — Layer 3 falls back per stage - Narrowed YAML missing/empty on a stage agent — renderTestDB falls back Every fallback emits an `echo` log line. From 208ed9b5df69142e8a347189c74760097d6d5285 Mon Sep 17 00:00:00 2001 From: mpikulski <206748156+ixlmar@users.noreply.github.com> Date: Fri, 29 May 2026 18:57:14 +0200 Subject: [PATCH 126/308] [TRTLLM-12982][perf] remove sync after FlashInfer attention plan() (#14634) Signed-off-by: ixlmar <206748156+ixlmar@users.noreply.github.com> --- tensorrt_llm/_torch/attention_backend/flashinfer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorrt_llm/_torch/attention_backend/flashinfer.py b/tensorrt_llm/_torch/attention_backend/flashinfer.py index 49964319a18a..3322422075dc 100644 --- a/tensorrt_llm/_torch/attention_backend/flashinfer.py +++ b/tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -1006,7 +1006,6 @@ def _plan_with_params(self, self.kv_layout, backend="cudnn", )) - torch.cuda.current_stream().synchronize() if self.num_contexts <= 0: raise ValueError( "FlashInfer ragged prefill without KV cache requires " From 27b4e565db5912b74ba741fa7abc3a5c58fd60d6 Mon Sep 17 00:00:00 2001 From: dongfengy <99041270+dongfengy@users.noreply.github.com> Date: Fri, 29 May 2026 10:03:09 -0700 Subject: [PATCH 127/308] [https://nvbugs/6156233][test] unwaive GPT-OSS dflash test since bug has been closed as fix unknown (#14713) Signed-off-by: Dongfeng Yu --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index ab89142f9798..f08fb36003d2 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -63,7 +63,6 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughpu accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput_trtllm] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5772993) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[two_model] SKIP (https://nvbugs/5596343) From 29971b4c1dec40c31594768e497bbf34a89f3066 Mon Sep 17 00:00:00 2001 From: "Wang, Xiao" <24860335+xwang233@users.noreply.github.com> Date: Fri, 29 May 2026 10:08:17 -0700 Subject: [PATCH 128/308] [TRTLLM-12901][fix] cap per-rank max_num_active_requests by max_num_tokens under attention DP (#14481) Signed-off-by: Xiao Wang <24860335+xwang233@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 23 ++++++ .../_torch/pyexecutor/request_utils.py | 27 +++++++ .../_torch/executor/test_request_utils.py | 81 +++++++++++++++++++ 3 files changed, 131 insertions(+) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index fedb14e0aa79..a893264d1ef1 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -66,6 +66,7 @@ from .model_engine import ModelEngine from .perf_metrics_manager import PerfMetricsManager from .request_utils import (RequestBroadcaster, attach_py_objects_to_requests, + derive_attention_dp_per_rank_request_cap, get_from_waiting_queue, merge_requests) from .resource_manager import (KVCacheManagerV2, ResourceManager, ResourceManagerType, request_context) @@ -417,6 +418,28 @@ def __init__( self.max_input_len = max_input_len # _executor_loop private data self.max_num_active_requests = model_engine.get_max_num_sequences() + # nvbug-6133201: under attention DP, tighten the per-rank request + # cap so per-rank gen-phase step-token load cannot exceed + # max_num_tokens. No-op for correctly-sized configs. + if self.enable_attention_dp: + derived_cap = derive_attention_dp_per_rank_request_cap( + base_cap=self.max_num_active_requests, + max_num_tokens=self.max_num_tokens, + max_total_draft_tokens=self.max_total_draft_tokens, + ) + if derived_cap < self.max_num_active_requests: + step_tokens_per_req = 1 + max(self.max_total_draft_tokens, 0) + required_max_num_tokens = (self.max_num_active_requests * + step_tokens_per_req) + logger.warning( + f"[PyExecutor] enable_attention_dp: max_num_tokens=" + f"{self.max_num_tokens} cannot fit max_batch_size=" + f"{self.max_num_active_requests} at {step_tokens_per_req} " + f"step-tokens each; capping per-rank max_num_active_requests " + f"to {derived_cap}. Raise max_num_tokens to " + f"{required_max_num_tokens} to run at the declared " + f"max_batch_size.") + self.max_num_active_requests = derived_cap self.active_requests: List[LlmRequest] = [] self.expected_num_active_requests = 0 # TODO: Remove PP size == 1 gate for disagg + block reuse with PP > 1. diff --git a/tensorrt_llm/_torch/pyexecutor/request_utils.py b/tensorrt_llm/_torch/pyexecutor/request_utils.py index d613a6cf2257..0a8866fdfc69 100644 --- a/tensorrt_llm/_torch/pyexecutor/request_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/request_utils.py @@ -70,6 +70,33 @@ def attach_py_objects_to_requests(requests: List, py_request_objects: Tuple) -> setattr(item.request, attr_name, py_obj) +def derive_attention_dp_per_rank_request_cap( + base_cap: int, + max_num_tokens: Optional[int], + max_total_draft_tokens: int, +) -> int: + """Cap per-rank requests at ``max_num_tokens // (1 + max_total_draft_tokens)`` + so gen-phase per-step token load cannot exceed ``max_num_tokens`` under + attention DP, where no component otherwise enforces a per-rank token cap + (nvbug-6133201). Each gen request occupies ``1 + max_total_draft_tokens`` + token slots per step. Mirrors the CUDA graph batch-size cap at + ``model_engine._filter_cuda_graph_batch_sizes``. + + Args: + base_cap: Per-rank request cap from ``get_max_num_sequences()``. + max_num_tokens: ``LlmArgs.max_num_tokens``; ``None`` disables tightening. + max_total_draft_tokens: Draft tokens per gen request (0 without spec + decoding); negative values are clamped to 0. + + Returns: + The tighter of ``base_cap`` and ``max_num_tokens // step_tokens``. + """ + if max_num_tokens is None: + return base_cap + step_tokens_per_req = 1 + max(max_total_draft_tokens, 0) + return min(base_cap, max_num_tokens // step_tokens_per_req) + + def can_process_attention_dp_request( req_item, all_ranks_num_active_requests: List[int], max_num_active_requests: int ) -> bool: diff --git a/tests/unittest/_torch/executor/test_request_utils.py b/tests/unittest/_torch/executor/test_request_utils.py index c53cfe940d3b..899b1e8851ee 100644 --- a/tests/unittest/_torch/executor/test_request_utils.py +++ b/tests/unittest/_torch/executor/test_request_utils.py @@ -13,6 +13,7 @@ from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem from tensorrt_llm._torch.pyexecutor.request_utils import ( can_process_attention_dp_request, + derive_attention_dp_per_rank_request_cap, get_from_waiting_queue, merge_helix_requests, merge_requests, @@ -379,3 +380,83 @@ def test_can_process_attention_dp_request(attention_dp_config): assert not can_process_attention_dp_request( req_no_capacity, all_ranks_full, max_num_active_requests ) + + +# -------------------------------------------------------------------------- +# nvbug-6133201: per-rank gen-phase step-token cap via tightened +# per-rank request cap. +# -------------------------------------------------------------------------- +# Under enable_attention_dp the global Python scheduler caps tokens +# cluster-wide and the ADP router caps per-rank requests, but no +# component caps per-rank gen-phase step-tokens. PyExecutor tightens +# the per-rank request cap to +# ``max_num_tokens // (1 + max_total_draft_tokens)`` so per-rank step- +# token load cannot exceed max_num_tokens by construction. + + +class TestDeriveAttentionDpPerRankRequestCap: + """Unit tests for ``derive_attention_dp_per_rank_request_cap``. + + The fix for nvbug-6133201 is the cap arithmetic implemented in this + helper; PyExecutor calls it once at ``__init__`` and the result + flows through the existing ``max_num_active_requests`` plumbing. + """ + + def test_no_tightening_when_max_num_tokens_is_none(self): + # LlmArgs.max_num_tokens == None -> helper is a no-op. + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=None, max_total_draft_tokens=3 + ) + == 128 + ) + + def test_nvbug_6133201_failing_config(self): + # nvbug-6133201 numbers: max_batch_size=128, + # max_total_draft_tokens=3 (MTP3), max_num_tokens=256. + # Per-rank step-token cost per req = 1 + 3 = 4. + # Effective cap = 256 // 4 = 64; per-rank load at saturation + # 64 * 4 = 256 = max_num_tokens, so the per-rank assert in + # model_engine.py cannot trip on gen-phase accumulation. + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=256, max_total_draft_tokens=3 + ) + == 64 + ) + + def test_no_tightening_when_arithmetic_already_fits(self): + # Correctly-sized LlmArgs (max_batch_size * (1+max_total_draft_tokens) + # <= max_num_tokens): cap == base_cap, no behavioral change. + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=512, max_total_draft_tokens=3 + ) + == 128 + ) + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=4096, max_total_draft_tokens=3 + ) + == 128 + ) + + def test_no_spec_decoding(self): + # max_total_draft_tokens == 0: step-token cost per req == 1, + # effective cap == max_num_tokens. + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=64, max_total_draft_tokens=0 + ) + == 64 + ) + + def test_negative_max_total_draft_tokens_clamped(self): + # Defensive: a stray negative value must not yield div-by-zero + # or a negative cap. + assert ( + derive_attention_dp_per_rank_request_cap( + base_cap=128, max_num_tokens=256, max_total_draft_tokens=-5 + ) + == 128 + ) From 59de15d21a791013b7e7c04c934352da6f218d22 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Sat, 30 May 2026 01:59:01 +0800 Subject: [PATCH 129/308] [None][feat] Enable NVFP4 KV cache support in trtllm-gen attention (#12544) Signed-off-by: Yihan Wang --- cpp/tensorrt_llm/nanobind/thop/bindings.cpp | 29 +++--- cpp/tensorrt_llm/thop/attentionOp.cpp | 20 +++- cpp/tensorrt_llm/thop/attentionOp.h | 8 +- cpp/tensorrt_llm/thop/trtllmGenFusedOps.h | 8 +- .../thop/trtllmGenQKVProcessOp.cpp | 42 ++++++-- .../_torch/attention_backend/trtllm.py | 3 +- .../_torch/attention_backend/trtllm_gen.py | 95 ++++++++++++++----- 7 files changed, 149 insertions(+), 56 deletions(-) diff --git a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp index 3c2e59945aaa..ca1206344ac6 100644 --- a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp @@ -73,8 +73,9 @@ nb::tuple trtllmGenContextPreprocessBinding(torch::Tensor qkv_input, torch::Tens }(); return nb::make_tuple(std::get<0>(result), optionalTensorToObject(std::get<1>(result)), - optionalTensorToObject(std::get<2>(result)), std::get<3>(result), std::get<4>(result), std::get<5>(result), - std::get<6>(result), std::get<7>(result), std::get<8>(result)); + optionalTensorToObject(std::get<2>(result)), optionalTensorToObject(std::get<3>(result)), + optionalTensorToObject(std::get<4>(result)), optionalTensorToObject(std::get<5>(result)), std::get<6>(result), + std::get<7>(result), std::get<8>(result), std::get<9>(result), std::get<10>(result), std::get<11>(result)); } nb::tuple trtllmGenGenerationPreprocessBinding(torch::Tensor qkv_input, torch::Tensor workspace, @@ -108,8 +109,9 @@ nb::tuple trtllmGenGenerationPreprocessBinding(torch::Tensor qkv_input, torch::T }(); return nb::make_tuple(std::get<0>(result), optionalTensorToObject(std::get<1>(result)), - optionalTensorToObject(std::get<2>(result)), std::get<3>(result), optionalTensorToObject(std::get<4>(result)), - std::get<5>(result), std::get<6>(result), std::get<7>(result), std::get<8>(result)); + optionalTensorToObject(std::get<2>(result)), optionalTensorToObject(std::get<3>(result)), std::get<4>(result), + std::get<5>(result), std::get<6>(result), optionalTensorToObject(std::get<7>(result)), std::get<8>(result), + std::get<9>(result), std::get<10>(result), std::get<11>(result)); } } // namespace @@ -292,13 +294,18 @@ void initBindings(nb::module_& m) int64_t head_dim, int64_t kv_factor, int64_t total_num_blocks, int64_t kv_cache_quant_mode, int64_t batch_start, int64_t batch_size, at::ScalarType dtype) -> nb::tuple { - nb::gil_scoped_release release; - auto kvPool = torch_ext::buildFlashinferTrtllmGenPagedKvCacheBuffers(host_kv_cache_pool_pointers, - host_kv_cache_pool_mapping, layer_idx, num_kv_heads, tokens_per_block, head_dim, kv_factor, - total_num_blocks, kv_cache_quant_mode, dtype); - auto const mapping = torch_ext::readKvCachePoolMapping(host_kv_cache_pool_mapping, layer_idx); - auto blockTables = kv_cache_block_offsets.select(0, mapping.poolIndex).narrow(0, batch_start, batch_size); - return nb::make_tuple(nb::cast(kvPool), nb::cast(blockTables)); + at::Tensor kvPool; + std::optional kvScalePool; + at::Tensor blockTables; + { + nb::gil_scoped_release release; + std::tie(kvPool, kvScalePool) = torch_ext::buildFlashinferTrtllmGenPagedKvCacheBuffers( + host_kv_cache_pool_pointers, host_kv_cache_pool_mapping, layer_idx, num_kv_heads, tokens_per_block, + head_dim, kv_factor, total_num_blocks, kv_cache_quant_mode, dtype); + auto const mapping = torch_ext::readKvCachePoolMapping(host_kv_cache_pool_mapping, layer_idx); + blockTables = kv_cache_block_offsets.select(0, mapping.poolIndex).narrow(0, batch_start, batch_size); + } + return nb::make_tuple(nb::cast(kvPool), nb::cast(blockTables), optionalTensorToObject(kvScalePool)); }, nb::arg("host_kv_cache_pool_pointers"), nb::arg("host_kv_cache_pool_mapping"), nb::arg("kv_cache_block_offsets"), nb::arg("layer_idx"), nb::arg("num_kv_heads"), nb::arg("tokens_per_block"), diff --git a/cpp/tensorrt_llm/thop/attentionOp.cpp b/cpp/tensorrt_llm/thop/attentionOp.cpp index 2f6f986daf4e..c835ca438cb1 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.cpp +++ b/cpp/tensorrt_llm/thop/attentionOp.cpp @@ -1383,9 +1383,10 @@ common::op::KvCacheBuffers buildPagedKvCacheBuffers( quantMode.hasFp4KvCache()); } -at::Tensor buildFlashinferTrtllmGenPagedKvCacheBuffers(at::Tensor host_kv_cache_pool_pointers, - at::Tensor host_kv_cache_pool_mapping, int64_t layer_idx, int64_t num_kv_heads, int64_t tokens_per_block, - int64_t head_dim, int64_t kv_factor, int64_t total_num_blocks, int64_t kv_cache_quant_mode, at::ScalarType dtype) +std::tuple> buildFlashinferTrtllmGenPagedKvCacheBuffers( + at::Tensor host_kv_cache_pool_pointers, at::Tensor host_kv_cache_pool_mapping, int64_t layer_idx, + int64_t num_kv_heads, int64_t tokens_per_block, int64_t head_dim, int64_t kv_factor, int64_t total_num_blocks, + int64_t kv_cache_quant_mode, at::ScalarType dtype) { auto const mapping = readKvCachePoolMapping(host_kv_cache_pool_mapping, layer_idx); int32_t const poolIndex = mapping.poolIndex; @@ -1422,7 +1423,18 @@ at::Tensor buildFlashinferTrtllmGenPagedKvCacheBuffers(at::Tensor host_kv_cache_ auto kv_pool = torch::from_blob( poolPointers.primaryPoolPtr, {total_num_blocks, num_kv_heads, tokens_per_block, containerDim}, options); - return kv_pool; + std::optional kvScalePool = std::nullopt; + if (isFp4 && poolPointers.primaryBlockScalePoolPtr != nullptr) + { + auto scaleOptions + = at::TensorOptions() + .dtype(at::kFloat8_e4m3fn) + .device(c10::Device(at::kCUDA, static_cast(at::cuda::current_device()))); + kvScalePool = torch::from_blob(poolPointers.primaryBlockScalePoolPtr, + {total_num_blocks, num_kv_heads, tokens_per_block, head_dim / 16}, scaleOptions); + } + + return {kv_pool, kvScalePool}; } } // namespace torch_ext diff --git a/cpp/tensorrt_llm/thop/attentionOp.h b/cpp/tensorrt_llm/thop/attentionOp.h index f01ffd555a3e..193d80507125 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.h +++ b/cpp/tensorrt_llm/thop/attentionOp.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "tensorrt_llm/common/attentionOp.h" #include "tensorrt_llm/common/config.h" @@ -118,9 +119,10 @@ common::op::KvCacheBuffers buildPagedKvCacheBuffers( int64_t cyclic_attention_window_size, int64_t max_attention_window_size, int64_t beam_width, int64_t seq_offset, bool is_mla_enable, size_t elem_size); -at::Tensor buildFlashinferTrtllmGenPagedKvCacheBuffers(at::Tensor host_kv_cache_pool_pointers, - at::Tensor host_kv_cache_pool_mapping, int64_t layer_idx, int64_t num_kv_heads, int64_t tokens_per_block, - int64_t head_dim, int64_t kv_factor, int64_t total_num_blocks, int64_t kv_cache_quant_mode, at::ScalarType dtype); +std::tuple> buildFlashinferTrtllmGenPagedKvCacheBuffers( + at::Tensor host_kv_cache_pool_pointers, at::Tensor host_kv_cache_pool_mapping, int64_t layer_idx, + int64_t num_kv_heads, int64_t tokens_per_block, int64_t head_dim, int64_t kv_factor, int64_t total_num_blocks, + int64_t kv_cache_quant_mode, at::ScalarType dtype); // Layout manager for the thop attention workspace slices used by trtllm-gen. // Context follows AttentionOp::getWorkspaceSizeForContext() ordering. Generation diff --git a/cpp/tensorrt_llm/thop/trtllmGenFusedOps.h b/cpp/tensorrt_llm/thop/trtllmGenFusedOps.h index e2c651a4e48a..cb4db3535756 100644 --- a/cpp/tensorrt_llm/thop/trtllmGenFusedOps.h +++ b/cpp/tensorrt_llm/thop/trtllmGenFusedOps.h @@ -27,8 +27,8 @@ TRTLLM_NAMESPACE_BEGIN namespace torch_ext { -std::tuple, std::optional, at::Tensor, at::Tensor, at::Tensor, - int64_t, int64_t, int64_t> +std::tuple, std::optional, std::optional, + std::optional, std::optional, at::Tensor, at::Tensor, at::Tensor, int64_t, int64_t, int64_t> trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, torch::Tensor sequence_lengths, torch::Tensor context_lengths, std::optional kv_cache_block_offsets, std::optional host_kv_cache_pool_pointers, std::optional host_kv_cache_pool_mapping, @@ -57,8 +57,8 @@ void trtllmGenContextPostprocess(torch::Tensor qkv_input, torch::Tensor workspac int64_t position_embedding_type, double bmm1_scale, bool fp8_context_fmha, bool paged_context_fmha, bool is_mla_enable, int64_t attention_chunk_size, int64_t multi_processor_count); -std::tuple, std::optional, at::Tensor, std::optional, - int64_t, int64_t, int64_t, bool> +std::tuple, std::optional, std::optional, at::Tensor, + at::Tensor, at::Tensor, std::optional, int64_t, int64_t, int64_t, bool> trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, torch::Tensor sequence_lengths, std::optional spec_decoding_generation_lengths, std::optional spec_decoding_position_offsets, std::optional kv_cache_block_offsets, diff --git a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp index cb9125d3c7a2..b1401f8d4bec 100644 --- a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp +++ b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include TRTLLM_NAMESPACE_BEGIN @@ -183,6 +184,8 @@ struct ContextWorkspaceRawViews at::Tensor cuQSeqlens; at::Tensor cuKvSeqlens; at::Tensor qBuf; + std::optional fmhaBmm1Scale; + std::optional fmhaBmm2Scale; ContextWorkspaceRawPointers ptrs; }; @@ -197,6 +200,16 @@ ContextWorkspaceRawViews makeContextWorkspaceRawViews( { qBuf = workspaceView.tensor(layout.qBufOffset, layout.qBufSize, layout.qBufScalarType); } + std::optional fmhaBmm1Scale; + std::optional fmhaBmm2Scale; + if (layout.fmhaBmm1ScaleSize > 0) + { + fmhaBmm1Scale = workspaceView.tensor(layout.fmhaBmm1ScaleOffset, layout.fmhaBmm1ScaleSize, at::kFloat); + } + if (layout.fmhaBmm2ScaleSize > 0) + { + fmhaBmm2Scale = workspaceView.tensor(layout.fmhaBmm2ScaleOffset, layout.fmhaBmm2ScaleSize, at::kFloat); + } return ContextWorkspaceRawViews{ .trtllmGenWorkspace @@ -205,6 +218,8 @@ ContextWorkspaceRawViews makeContextWorkspaceRawViews( .cuKvSeqlens = workspaceView.tensor(layout.cuKvSeqlensOffset, layout.cuSeqlensSize, sizeof(int32_t), intOptions), .qBuf = qBuf, + .fmhaBmm1Scale = fmhaBmm1Scale, + .fmhaBmm2Scale = fmhaBmm2Scale, .ptrs = makeContextWorkspaceRawPointers(workspaceView, layout), }; } @@ -213,6 +228,8 @@ struct GenerationWorkspaceRawViews { at::Tensor trtllmGenWorkspace; at::Tensor qBuf; + at::Tensor bmm1Scale; + at::Tensor bmm2Scale; int* cuSeqlensPtr{}; int* cuKvSeqlensPtr{}; float* rotaryInvFreqBufPtr{}; @@ -235,6 +252,8 @@ GenerationWorkspaceRawViews makeGenerationWorkspaceRawViews( .trtllmGenWorkspace = workspaceView.tensor(layout.trtllmGenWorkspaceOffset, layout.trtllmGenWorkspaceSize, 1, byteOptions), .qBuf = workspaceView.tensor(layout.qBufOffset, layout.qBufSize, qBufItemSize, qBufOptions), + .bmm1Scale = workspaceView.tensor(layout.bmm1ScaleOffset, layout.bmm1ScaleSize, at::kFloat), + .bmm2Scale = workspaceView.tensor(layout.bmm2ScaleOffset, layout.bmm2ScaleSize, at::kFloat), .cuSeqlensPtr = workspaceView.ptr(layout.cuSeqlensOffset, layout.cuSeqlensSize), .cuKvSeqlensPtr = workspaceView.ptr(layout.cuKvSeqlensOffset, layout.cuKvSeqlensSize), .rotaryInvFreqBufPtr = workspaceView.ptr(layout.rotaryInvFreqOffset, layout.rotaryInvFreqSize), @@ -248,8 +267,8 @@ GenerationWorkspaceRawViews makeGenerationWorkspaceRawViews( } // anonymous namespace -std::tuple, std::optional, at::Tensor, at::Tensor, at::Tensor, - int64_t, int64_t, int64_t> +std::tuple, std::optional, std::optional, + std::optional, std::optional, at::Tensor, at::Tensor, at::Tensor, int64_t, int64_t, int64_t> trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, torch::Tensor sequence_lengths, torch::Tensor context_lengths, std::optional kv_cache_block_offsets, std::optional host_kv_cache_pool_pointers, std::optional host_kv_cache_pool_mapping, @@ -413,10 +432,11 @@ trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, tor } std::optional kvPool; + std::optional kvScalePool; std::optional blockTables; if (need_build_kv_cache_metadata) { - kvPool = buildFlashinferTrtllmGenPagedKvCacheBuffers(host_kv_cache_pool_pointers.value(), + std::tie(kvPool, kvScalePool) = buildFlashinferTrtllmGenPagedKvCacheBuffers(host_kv_cache_pool_pointers.value(), host_kv_cache_pool_mapping.value(), layer_idx, num_kv_heads, tokens_per_block, head_size, kv_factor, total_num_blocks, kv_cache_quant_mode, qkvScalarType); @@ -437,8 +457,9 @@ trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, tor zeroFlashinferTrtllmGenCounterWorkspaceAsync(views.trtllmGenWorkspace, stream); auto const windowLeft = computeWindowLeft(cyclic_attention_window_size, max_past_kv_length, attention_chunk_size); - return {qProcessed, kvPool, blockTables, views.trtllmGenWorkspace, views.cuQSeqlens, views.cuKvSeqlens, - input_seq_length, max_past_kv_length, windowLeft}; + return {qProcessed, kvPool, blockTables, kvScalePool, views.fmhaBmm1Scale, views.fmhaBmm2Scale, + views.trtllmGenWorkspace, views.cuQSeqlens, views.cuKvSeqlens, input_seq_length, max_past_kv_length, + windowLeft}; } void trtllmGenContextPostprocess(torch::Tensor qkv_input, torch::Tensor workspace, torch::Tensor sequence_lengths, @@ -556,8 +577,8 @@ void trtllmGenContextPostprocess(torch::Tensor qkv_input, torch::Tensor workspac } } -std::tuple, std::optional, at::Tensor, std::optional, - int64_t, int64_t, int64_t, bool> +std::tuple, std::optional, std::optional, at::Tensor, + at::Tensor, at::Tensor, std::optional, int64_t, int64_t, int64_t, bool> trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, torch::Tensor sequence_lengths, std::optional spec_decoding_generation_lengths, std::optional spec_decoding_position_offsets, std::optional kv_cache_block_offsets, @@ -733,10 +754,11 @@ trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, } std::optional kvPool; + std::optional kvScalePool; std::optional blockTables; if (need_build_kv_cache_metadata) { - kvPool = buildFlashinferTrtllmGenPagedKvCacheBuffers(host_kv_cache_pool_pointers.value(), + std::tie(kvPool, kvScalePool) = buildFlashinferTrtllmGenPagedKvCacheBuffers(host_kv_cache_pool_pointers.value(), host_kv_cache_pool_mapping.value(), layer_idx, num_kv_heads, tokens_per_block, head_size, kv_factor, total_num_blocks, kv_cache_quant_mode, qkvScalarType); @@ -748,8 +770,8 @@ trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, zeroFlashinferTrtllmGenCounterWorkspaceAsync(views.trtllmGenWorkspace, stream); auto const windowLeft = computeWindowLeft(cyclic_attention_window_size, max_past_kv_length, attention_chunk_size); - return {qProcessed, kvPool, blockTables, views.trtllmGenWorkspace, cuSeqlens, input_seq_length, max_past_kv_length, - windowLeft, isMultiTokenGen}; + return {qProcessed, kvPool, blockTables, kvScalePool, views.bmm1Scale, views.bmm2Scale, views.trtllmGenWorkspace, + cuSeqlens, input_seq_length, max_past_kv_length, windowLeft, isMultiTokenGen}; } } // namespace torch_ext diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index 7b819106eb0b..0ccd18f56ed7 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -26,7 +26,8 @@ PositionalEmbeddingParams, PredefinedAttentionMask, RopeParams, merge_attention_forward_args) -# Enable TRTLLM-Gen attention backend via environment variable (default: on). +# Enable TRTLLM-Gen attention backend by default. Set +# TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION=0 to force the thop.attention path. _TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION = (os.environ.get( "TRTLLM_ENABLE_TRTLLM_GEN_ATTENTION", "0") == "1") diff --git a/tensorrt_llm/_torch/attention_backend/trtllm_gen.py b/tensorrt_llm/_torch/attention_backend/trtllm_gen.py index 16ce98ef876a..a7fb8ccfa2fe 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm_gen.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm_gen.py @@ -64,7 +64,12 @@ class TrtllmGenSupportChecker: # Supported data types SUPPORTED_INPUT_DTYPES = {torch.float16, torch.bfloat16, torch.float8_e4m3fn} - SUPPORTED_KV_CACHE_DTYPES = {DataType.HALF, DataType.BF16, DataType.FP8} + SUPPORTED_KV_CACHE_DTYPES = { + DataType.HALF, + DataType.BF16, + DataType.FP8, + DataType.NVFP4, + } SUPPORTED_OUT_DTYPES = {torch.float16, torch.bfloat16, torch.float8_e4m3fn} # Supported Q:KV:O dtype combinations for trtllm-gen kernels @@ -81,6 +86,10 @@ class TrtllmGenSupportChecker: (torch.float8_e4m3fn, DataType.FP8, torch.float16), # e4m3:e4m3:bf16 (torch.float8_e4m3fn, DataType.FP8, torch.bfloat16), + # e4m3:nvfp4:* + (torch.float8_e4m3fn, DataType.NVFP4, torch.float8_e4m3fn), + (torch.float8_e4m3fn, DataType.NVFP4, torch.float16), + (torch.float8_e4m3fn, DataType.NVFP4, torch.bfloat16), } # Generation phase supported combinations (includes context + additional) @@ -96,6 +105,10 @@ class TrtllmGenSupportChecker: (torch.bfloat16, DataType.FP8, torch.bfloat16), # fp16:e4m3:fp16 (torch.float16, DataType.FP8, torch.float16), + # e4m3:nvfp4:* + (torch.float8_e4m3fn, DataType.NVFP4, torch.float8_e4m3fn), + (torch.float8_e4m3fn, DataType.NVFP4, torch.float16), + (torch.float8_e4m3fn, DataType.NVFP4, torch.bfloat16), } # Unsupported head sizes for context FMHA. @@ -231,19 +244,12 @@ def is_supported( if cross_attention: return False, "Cross attention is not supported by trtllm-gen backend." - has_fp4_kv = ( - quant_config.layer_quant_mode.has_fp4_kv_cache() - if quant_config is not None - else kv_cache_dtype == DataType.NVFP4 - ) - if has_fp4_kv: - return False, "NVFP4 KV cache is not supported by flashinfer trtllm-gen kernels." if q_dtype not in cls.SUPPORTED_INPUT_DTYPES: return False, f"Input dtype {q_dtype} not supported. Supported: FP16, BF16, FP8 (E4M3)." if kv_cache_dtype not in cls.SUPPORTED_KV_CACHE_DTYPES: return ( False, - f"KV cache dtype {kv_cache_dtype} not supported. Supported: FP16, BF16, FP8.", + f"KV cache dtype {kv_cache_dtype} not supported. Supported: FP16, BF16, FP8, NVFP4.", ) if out_dtype is not None and out_dtype not in cls.SUPPORTED_OUT_DTYPES: return False, f"Output dtype {out_dtype} not supported. Supported: FP16, BF16, FP8." @@ -610,12 +616,6 @@ def _get_kv_scale_params( return kv_scale_orig_quant, kv_scale_quant_orig - @staticmethod - def _get_mrope_rotary_cos_sin( - forward_args: AttentionForwardArgs, - ) -> Optional[torch.Tensor]: - return forward_args.mrope_rotary_cos_sin - def is_supported( self, q: torch.Tensor, @@ -655,8 +655,12 @@ def is_supported( has_sparse_attention = ( sparse_attention_config is not None and not has_skip_softmax_attention ) + q_dtype = q.dtype + if kv_cache_manager.dtype == DataType.NVFP4: + q_dtype = torch.float8_e4m3fn + result = self._checker.is_supported( - q_dtype=q.dtype, + q_dtype=q_dtype, kv_cache_dtype=kv_cache_manager.dtype, num_heads=self._num_heads, num_kv_heads=self._num_kv_heads, @@ -745,7 +749,10 @@ def attention( fp8_context_fmha = ( is_fp8_out or is_fp4_out - or (kv_cache_quant_mode.has_fp8_kv_cache() and use_paged_context_fmha) + or ( + (kv_cache_quant_mode.has_fp8_kv_cache() or kv_cache_quant_mode.has_fp4_kv_cache()) + and use_paged_context_fmha + ) ) num_tokens = q.size(0) @@ -956,6 +963,10 @@ def _get_kv_cache_metadata( blocks_in_primary_pool = max( int(primary) for primary, _ in blocks_per_window.values() ) + if blocks_in_primary_pool is None: + raise RuntimeError( + "trtllm-gen could not determine blocks_in_primary_pool from the KVCacheManager." + ) total_num_blocks = ( int(blocks_in_primary_pool) * kv_cache_manager.num_local_layers * kv_factor ) @@ -969,12 +980,15 @@ def run_context( params.forward, params.kv_cache_quant_mode ) attention_output_orig_quant = params.forward.out_scale - mrope_rotary_cos_sin = self._get_mrope_rotary_cos_sin(params.forward) + mrope_rotary_cos_sin = params.forward.mrope_rotary_cos_sin ( q_processed, kv_pool, block_tables, + kv_scale_pool, + bmm1_scale, + bmm2_scale, fmha_workspace, cu_q_seqlens, cu_kv_seqlens, @@ -1018,6 +1032,21 @@ def run_context( # FlashInfer accepts a split K/V tuple; TensorRT-LLM stores both views # in one flat paged KV pool, so both tuple entries intentionally alias. + kv_cache_sf = None + if kv_scale_pool is not None: + kv_cache_sf = (kv_scale_pool, kv_scale_pool) + + has_fp4_kv = QuantMode(params.kv_cache_quant_mode).has_fp4_kv_cache() + if has_fp4_kv: + q_processed = ( + q_processed.view(torch.uint8) + .flatten()[: params.num_tokens * self._num_heads * self._head_dim] + .view(torch.float8_e4m3fn) + .view(params.num_tokens, self._num_heads, self._head_dim) + ) + ctx_bmm1_scale = bmm1_scale if has_fp4_kv and bmm1_scale is not None else self._bmm1_scale + ctx_bmm2_scale = bmm2_scale if has_fp4_kv and bmm2_scale is not None else 1.0 + flashinfer.prefill.trtllm_batch_context_with_kv_cache( query=q_processed, kv_cache=(kv_pool, kv_pool), @@ -1026,8 +1055,8 @@ def run_context( seq_lens=params.sequence_lengths, max_q_len=max_q_len, max_kv_len=max_kv_len, - bmm1_scale=self._bmm1_scale, - bmm2_scale=1.0, + bmm1_scale=ctx_bmm1_scale, + bmm2_scale=ctx_bmm2_scale, batch_size=params.batch_size, cum_seq_lens_q=cu_q_seqlens, cum_seq_lens_kv=cu_kv_seqlens, @@ -1036,6 +1065,7 @@ def run_context( kv_layout=self._layout, sinks=params.forward.attention_sinks, uses_shared_paged_kv_idx=self.USE_SHARED_PAGED_KV_IDX, + kv_cache_sf=kv_cache_sf, enable_pdl=self._enable_pdl, ) @@ -1082,6 +1112,9 @@ def run_generation( q_processed, kv_pool, block_tables, + kv_scale_pool, + bmm1_scale, + bmm2_scale, fmha_workspace, cu_seqlens, max_q_len, @@ -1127,6 +1160,21 @@ def run_generation( decode_cu_seqlens = cu_seqlens if is_multi_token_gen else None # FlashInfer accepts a split K/V tuple; TensorRT-LLM stores both views # in one flat paged KV pool, so both tuple entries intentionally alias. + kv_cache_sf = None + if kv_scale_pool is not None: + kv_cache_sf = (kv_scale_pool, kv_scale_pool) + + has_fp4_kv = QuantMode(params.kv_cache_quant_mode).has_fp4_kv_cache() + if has_fp4_kv: + q_processed = ( + q_processed.view(torch.uint8) + .flatten()[: params.num_tokens * self._num_heads * self._head_dim] + .view(torch.float8_e4m3fn) + .view(params.num_tokens, self._num_heads, self._head_dim) + ) + gen_bmm1_scale = bmm1_scale if has_fp4_kv else self._bmm1_scale + gen_bmm2_scale = bmm2_scale if has_fp4_kv else 1.0 + flashinfer.decode.trtllm_batch_decode_with_kv_cache( query=q_processed, kv_cache=(kv_pool, kv_pool), @@ -1135,8 +1183,8 @@ def run_generation( seq_lens=params.sequence_lengths, max_seq_len=max_kv_len, out=params.context_buf, - bmm1_scale=self._bmm1_scale, - bmm2_scale=1.0, + bmm1_scale=gen_bmm1_scale, + bmm2_scale=gen_bmm2_scale, window_left=window_left, kv_layout=self._layout, sinks=params.forward.attention_sinks, @@ -1144,6 +1192,7 @@ def run_generation( max_q_len=decode_max_q_len, cum_seq_lens_q=decode_cu_seqlens, uses_shared_paged_kv_idx=self.USE_SHARED_PAGED_KV_IDX, + kv_cache_sf=kv_cache_sf, enable_pdl=self._enable_pdl, backend="trtllm-gen", ) @@ -1163,7 +1212,7 @@ def run_mla_generation( batch_beam = params.num_requests * params.beam_width if params.attention_input is None: raise RuntimeError("MLA generation requires attention_input.") - kv_cache, block_tables = thop.build_trtllm_gen_kv_cache_metadata( + kv_cache, block_tables, _ = thop.build_trtllm_gen_kv_cache_metadata( host_kv_cache_pool_pointers=params.host_kv_cache_pool_pointers, host_kv_cache_pool_mapping=params.host_kv_cache_pool_mapping, kv_cache_block_offsets=params.kv_cache_block_offsets, From 027eb72d85ae7733681d2227f294c0c9acf732ea Mon Sep 17 00:00:00 2001 From: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Date: Fri, 29 May 2026 11:02:38 -0700 Subject: [PATCH 130/308] [https://nvbugs/6185480][fix] autodeploy unwaive the test (#14716) Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f08fb36003d2..ea40c18d8c11 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -207,8 +207,6 @@ examples/test_visual_gen.py::test_wan21_t2v_lpips_against_golden SKIP (https://n examples/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6215688) examples/test_visual_gen.py::test_wan_t2v_example SKIP (https://nvbugs/6215688) examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) -full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] SKIP (https://nvbugs/6185480) -full:A100/accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] SKIP (https://nvbugs/6185480) full:B200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) From 6f2055f7c290fc7e88d305a34320f0f314026b8f Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Sat, 30 May 2026 02:15:54 +0800 Subject: [PATCH 131/308] [https://nvbugs/6136737][fix] Propagate external SWA window to FMHA kernel in V2 KV cache (#13719) Signed-off-by: svc_repair_bot Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: Thor Johnsen <41591019+thorjohnsen@users.noreply.github.com> --- tensorrt_llm/_torch/attention_backend/trtllm.py | 10 ++++++++++ tests/integration/test_lists/waives.txt | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index 0ccd18f56ed7..53dcfbc5af2a 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -1422,6 +1422,16 @@ def _run( if self.print_skip_softmax_stat: self.skip_softmax_stat.zero_() + # Propagate the KV cache manager's SWA window to the FMHA kernel when + # the model does not specify one, so paged-context attention does not + # read stale page-table entries (BAD_PAGE_INDEX). + if forward_args.attention_window_size is None and metadata.kv_cache_manager is not None: + window_vec = getattr(metadata.kv_cache_manager, + 'max_attention_window_vec', None) + if window_vec: + window = window_vec[self.local_layer_idx % len(window_vec)] + if window is not None: + forward_args.attention_window_size = window if forward_args.attention_window_size is None: forward_args.attention_window_size = metadata.max_seq_len diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index ea40c18d8c11..ac99a034439b 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -213,7 +213,6 @@ full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (ht full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) full:B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) -full:DGX_H100/kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked] SKIP (https://nvbugs/6136737) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) full:GB200/perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6194788) full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1] SKIP (https://nvbugs/4731514) From 91371ddc6b0f57578260623c53302c88a08be1f7 Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Fri, 29 May 2026 14:22:55 -0400 Subject: [PATCH 132/308] [https://nvbugs/5996024][fix] Enforce trust_remote_code flag (#13527) Signed-off-by: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> --- .../_torch/pyexecutor/model_engine.py | 1 + .../_torch/pyexecutor/model_loader.py | 4 ++-- tensorrt_llm/inputs/registry.py | 13 +++++++---- tensorrt_llm/inputs/utils.py | 9 ++++---- tensorrt_llm/llmapi/llm.py | 22 ++++++++++++------- tensorrt_llm/llmapi/mm_encoder.py | 9 +++++--- .../defs/accuracy/test_llm_api_pytorch.py | 8 +++++++ .../test_llm_api_pytorch_multimodal.py | 1 + .../host_perf_deepseek_v3_lite.yaml | 2 ++ .../test_modeling_nemotron_nano_v2_vl.py | 1 + 10 files changed, 49 insertions(+), 21 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 2c0c95ccb5ed..f452ccab4f97 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -267,6 +267,7 @@ def __init__( model_path, tokenizer=None, checkpoint_format=llm_args.checkpoint_format, + trust_remote_code=llm_args.trust_remote_code, **input_processor_kwargs) self.input_processor_with_hash = create_input_processor_with_hash( self.input_processor) diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 14d813a99dfd..7b547975933f 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -286,7 +286,7 @@ def load_config_and_apply_defaults( return llm_args config_kwargs = { - 'trust_remote_code': True, + 'trust_remote_code': llm_args.trust_remote_code, 'mm_encoder_only': llm_args.mm_encoder_only, } if llm_args.parallel_config: @@ -518,7 +518,7 @@ def _load_and_validate_config( """Loads and validates the model configuration.""" load_config_kwargs = dict( checkpoint_dir=checkpoint_dir, - trust_remote_code=True, + trust_remote_code=self.llm_args.trust_remote_code, mapping=self.mapping, enable_min_latency=self.llm_args.enable_min_latency, use_cuda_graph=self.llm_args.cuda_graph_config is not None, diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 5c99715e65ab..95ed84d4a96a 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -156,6 +156,7 @@ def __init__(self, self._model_path = model_path self._tokenizer = tokenizer self._use_fast: bool = kwargs.get('use_fast', True) + self._trust_remote_code = trust_remote_code self._multimodal_hashing_supported: Optional[bool] = None def attach_multimodal_embeddings( @@ -452,7 +453,8 @@ def get_dummy_prompt(self, input_seq_len: int): modality="image", prompts=[""], media=[[image]], - image_data_format="pt")[0] + image_data_format="pt", + trust_remote_code=self._trust_remote_code)[0] prompt_token_ids_single_img, _ = self(test_mm_prompt, None) @@ -694,6 +696,7 @@ def create_input_processor( model_path_or_dir: str, tokenizer, checkpoint_format: Optional[str] = "HF", + trust_remote_code: bool = True, **kwargs, ) -> Union[InputProcessor, BaseMultimodalInputProcessor]: """Create an input processor for a specific model. @@ -703,6 +706,8 @@ def create_input_processor( tokenizer: Tokenizer instance. checkpoint_format: Checkpoint format identifier. "HF" uses Hugging Face-style config loading; any other value skips HF config loading. Default is "HF". + trust_remote_code: Whether Hugging Face config/processor loading may run + model-provided Python code. **kwargs: Additional arguments passed to input processor constructors (e.g., video_pruning_rate for multimodal models). @@ -716,8 +721,8 @@ def create_input_processor( if checkpoint_format == "HF": try: - model_config = ModelConfig.from_pretrained(model_path_or_dir, - trust_remote_code=True) + model_config = ModelConfig.from_pretrained( + model_path_or_dir, trust_remote_code=trust_remote_code) config = model_config.pretrained_config except (ValueError, EnvironmentError) as e: logger.debug( @@ -745,7 +750,7 @@ def create_input_processor( return input_processor_cls(model_path_or_dir, config, tokenizer, - trust_remote_code=True, + trust_remote_code=trust_remote_code, **kwargs) return DefaultInputProcessor(None, None, tokenizer) diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py index 1132d7a13949..59483e625790 100644 --- a/tensorrt_llm/inputs/utils.py +++ b/tensorrt_llm/inputs/utils.py @@ -720,6 +720,7 @@ def default_multimodal_input_loader( List[List[torch.Tensor]]]] = None, device: str = "cpu", extract_audio: bool = False, + trust_remote_code: bool = True, ) -> List[dict[str, Union[str, torch.Tensor]]]: def convert_to_conversation_message( @@ -837,13 +838,13 @@ def convert_to_conversation_message( model_type) == ContentFormat.PASSTHROUGH) if tokenizer is None and not is_passthrough: - tokenizer = ModelLoader.load_hf_tokenizer(model_dir, use_fast=True) + tokenizer = ModelLoader.load_hf_tokenizer( + model_dir, trust_remote_code=trust_remote_code, use_fast=True) processor = None if not is_passthrough: - processor = AutoProcessor.from_pretrained(model_dir, - use_fast=True, - trust_remote_code=True) + processor = AutoProcessor.from_pretrained( + model_dir, use_fast=True, trust_remote_code=trust_remote_code) inputs = [] for prompt_idx, (prompt, diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 62f9be618222..2e6295bb5843 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -1161,7 +1161,8 @@ def _try_load_generation_config( def _try_load_hf_model_config( self) -> Optional[transformers.PretrainedConfig]: - return ModelLoader.load_hf_model_config(self.args.model) + return ModelLoader.load_hf_model_config( + self.args.model, trust_remote_code=self.args.trust_remote_code) @set_api_status("beta") def shutdown(self) -> None: @@ -1283,7 +1284,8 @@ def _build_model(self): # config.json uses TRT-LLM schema, not HF schema). if self._hf_model_dir is not None: self._hf_model_config = ModelLoader.load_hf_model_config( - self._hf_model_dir) + self._hf_model_dir, + trust_remote_code=self.args.trust_remote_code) else: self._hf_model_config = self._try_load_hf_model_config() self._generation_config = self._try_load_generation_config() @@ -1291,8 +1293,10 @@ def _build_model(self): # Multimodal special handling: # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor # 2. May need to modify model weights for MM (e.g., resize vocab embedding). We must do such operation via input processor's __init__ - self.input_processor = create_input_processor(self._hf_model_dir, - self.tokenizer) + self.input_processor = create_input_processor( + self._hf_model_dir, + self.tokenizer, + trust_remote_code=self.args.trust_remote_code) self._tokenizer = self.input_processor.tokenizer max_batch_size = self.args.max_batch_size @@ -1497,10 +1501,12 @@ def _build_model(self): if self.args.video_pruning_rate is not None: input_processor_kwargs[ 'video_pruning_rate'] = self.args.video_pruning_rate - self.input_processor = create_input_processor(self._hf_model_dir, - self.tokenizer, - checkpoint_format, - **input_processor_kwargs) + self.input_processor = create_input_processor( + self._hf_model_dir, + self.tokenizer, + checkpoint_format, + trust_remote_code=self.args.trust_remote_code, + **input_processor_kwargs) self._tokenizer = self.input_processor.tokenizer # Resolve encode_only mode (opt-in only) diff --git a/tensorrt_llm/llmapi/mm_encoder.py b/tensorrt_llm/llmapi/mm_encoder.py index 3ff85dd42bef..96b7420c31ae 100644 --- a/tensorrt_llm/llmapi/mm_encoder.py +++ b/tensorrt_llm/llmapi/mm_encoder.py @@ -52,9 +52,12 @@ def _build_model(self): # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor # 2. May need to modify model weights for MM (e.g., resize vocab embedding). We must do such operation via input processor's __init__ checkpoint_format = getattr(self.args, "checkpoint_format", None) - self.input_processor = create_input_processor(self._hf_model_dir, - self.tokenizer, - checkpoint_format) + trust_remote_code = self.args.trust_remote_code + self.input_processor = create_input_processor( + self._hf_model_dir, + self.tokenizer, + checkpoint_format, + trust_remote_code=trust_remote_code) self._tokenizer = self.input_processor.tokenizer assert isinstance(self.args, TorchLlmArgs) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index db94d7943bdc..ae173a79e0e1 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -6320,6 +6320,7 @@ class TestQwen3_5_4B(LlmapiAccuracyTestHarness): def test_bf16(self): model_path = f"{llm_models_root()}/Qwen3.5-4B" with LLM(model_path, + trust_remote_code=True, max_seq_len=4096, max_batch_size=128, kv_cache_config=self.kv_cache_config, @@ -6332,6 +6333,7 @@ def test_bf16(self): def test_fp8(self): model_path = f"{llm_models_root()}/Qwen3.5-4B-FP8" with LLM(model_path, + trust_remote_code=True, max_seq_len=4096, max_batch_size=128, kv_cache_config=self.kv_cache_config, @@ -6353,6 +6355,7 @@ def test_dflash(self): extra_acc_spec = "h20" if is_h20_gpu else None with LLM(target_model_path, + trust_remote_code=True, max_seq_len=4096, max_batch_size=8, kv_cache_config=self.kv_cache_config, @@ -6452,6 +6455,7 @@ def test_bf16(self, moe_backend, tp_size, mocker): extra_acc_spec = "h20" if is_h20_gpu else None with LLM(self.MODEL_PATH, + trust_remote_code=True, tensor_parallel_size=tp_size, moe_expert_parallel_size=1, max_seq_len=4096, @@ -6481,6 +6485,7 @@ def test_bf16_mtp(self, mtp_flag, mocker): ) if mtp_flag else None with LLM(self.MODEL_PATH, + trust_remote_code=True, tensor_parallel_size=1, moe_expert_parallel_size=1, max_seq_len=4096, @@ -6508,6 +6513,7 @@ def test_fp8(self, enable_block_reuse, mocker): cuda_graph_config = CudaGraphConfig(enable_padding=True, max_batch_size=128) with LLM(model_dir, + trust_remote_code=True, tensor_parallel_size=1, moe_expert_parallel_size=1, max_seq_len=4096, @@ -6547,6 +6553,7 @@ def test_bf16(self, mtp_flag, mocker): ) if mtp_flag else None with LLM(self.MODEL_PATH, + trust_remote_code=True, max_seq_len=4096, max_num_tokens=4096, max_batch_size=128, @@ -6608,6 +6615,7 @@ def test_nvfp4(self, tp_size, ep_size, cuda_graph, overlap_scheduler, if cuda_graph else None) with LLM(model_path, + trust_remote_code=True, tensor_parallel_size=tp_size, max_num_tokens=16384, max_batch_size=32, diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 05e887667df9..c4a1c29f44d6 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -718,6 +718,7 @@ def test_auto_dtype( ) -> None: with LLM( model_path, + trust_remote_code=True, kv_cache_config=kv_cache_config, enable_chunked_prefill=True, # Use a low-ish value for the below to force chunking for requests (helping to surface diff --git a/tests/scripts/perf-sanity/aggregated/host_perf_deepseek_v3_lite.yaml b/tests/scripts/perf-sanity/aggregated/host_perf_deepseek_v3_lite.yaml index d36851b53311..86ff5fac3352 100644 --- a/tests/scripts/perf-sanity/aggregated/host_perf_deepseek_v3_lite.yaml +++ b/tests/scripts/perf-sanity/aggregated/host_perf_deepseek_v3_lite.yaml @@ -27,6 +27,7 @@ server_configs: - name: "v3lite_fp8_bs1_128_128" model_name: "deepseek_v3_lite_fp8" tensor_parallel_size: 1 + trust_remote_code: true max_batch_size: 1 max_num_tokens: 256 moe_config: @@ -49,6 +50,7 @@ server_configs: - name: "v3lite_fp8_bs8_128_256" model_name: "deepseek_v3_lite_fp8" tensor_parallel_size: 1 + trust_remote_code: true max_batch_size: 8 max_num_tokens: 2048 moe_config: diff --git a/tests/unittest/_torch/modeling/test_modeling_nemotron_nano_v2_vl.py b/tests/unittest/_torch/modeling/test_modeling_nemotron_nano_v2_vl.py index 3e1db1e3bedf..3b89fe8ca204 100644 --- a/tests/unittest/_torch/modeling/test_modeling_nemotron_nano_v2_vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_nemotron_nano_v2_vl.py @@ -74,6 +74,7 @@ def nano_llm_model(): # we use the top-level LLM to create the engine to make things simpler. nano_llm = LLM( model=MODEL_PATH, + trust_remote_code=True, tensor_parallel_size=1, max_batch_size=24, cuda_graph_config=CudaGraphConfig(), From 562d6837e65ec9bc99b8e7fe9dc3337c3bd21222 Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Fri, 29 May 2026 14:46:21 -0400 Subject: [PATCH 133/308] [https://nvbugs/5979710][fix] Bound transfer destinations (#13525) Signed-off-by: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> --- .../agent_utils/connection.cpp | 138 ++++++++++++++++-- .../agent_utils/connection.h | 3 + 2 files changed, 127 insertions(+), 14 deletions(-) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp index d46defdf50ad..a21d470b898a 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,7 @@ #include "connection.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h" +#include #include #include #include @@ -54,11 +55,8 @@ std::string genUniqueAgentName() // layer num, since the buffer size is ratio is equal to the layer num ratio // except the VSWA case. -template -auto computeSendOffsetRatio( - CacheStateT const& peerCacheState, int peerIdx, CacheStateT const& selfCacheState, int connectionIdx) +auto computeSendOffsetRatio(TargetRanksInfo const& peerTargetInfo, int connectionIdx) { - auto peerTargetInfo = targetIRanks(selfCacheState, peerCacheState, peerIdx); size_t offsetLayer = 0; for (int i = 0; i < connectionIdx; i++) { @@ -69,6 +67,46 @@ auto computeSendOffsetRatio( return std::make_pair(offsetLayer, selfSendLayer); } +namespace +{ + +bool isValidBufferKind(uint8_t kind) +{ + switch (static_cast(kind)) + { + case batch_manager::BufferKind::kKV: + case batch_manager::BufferKind::kKV_INDEXER: + case batch_manager::BufferKind::kRNN: return true; + } + return false; +} + +AgentState const* findAgentState(CommState const& commState, std::string const& agentName) +{ + if (!commState.isAgentState()) + { + return nullptr; + } + for (auto const& agentState : commState.getAgentState()) + { + if (agentState.mAgentName == agentName) + { + return &agentState; + } + } + return nullptr; +} + +void validateConnectionIdx( + int connectionIdx, TargetRanksInfo const& targetInfo, char const* bufferKind, std::string const& remoteAgentName) +{ + TLLM_CHECK_WITH_INFO(static_cast(connectionIdx) < targetInfo.mIRanks.size(), + "AgentConnectionManager received %s connection index %d outside target rank count %zu from agent '%s'", + bufferKind, connectionIdx, targetInfo.mIRanks.size(), remoteAgentName.c_str()); +} + +} // namespace + AgentConnection::AgentConnection( std::string mAgentName, std::string mRemoteAgentName, AgentConnectionManager* mAgentConnectionManager) : mAgentName(mAgentName) @@ -132,7 +170,15 @@ void AgentConnection::send(DataContext const& ctx, void const* data, size_t size MemoryDescs srcDescs{MemoryType::kVRAM, {srcDesc}}; auto const& dstBaseDesc = mSenderState.activeBufferDesc(); auto const& offsetRatio = mSenderState.activeOffsetRatio(); - auto offset = size / offsetRatio.second * offsetRatio.first; + TLLM_CHECK_WITH_INFO(offsetRatio.second != 0, "AgentConnection::send offset ratio denominator cannot be 0"); + TLLM_CHECK_WITH_INFO(size <= dstBaseDesc.getLen(), "AgentConnection::send size exceeds destination buffer"); + auto const chunkSize = size / offsetRatio.second; + TLLM_CHECK_WITH_INFO(offsetRatio.first == 0 || chunkSize <= std::numeric_limits::max() / offsetRatio.first, + "AgentConnection::send offset calculation overflow"); + auto const offset = chunkSize * offsetRatio.first; + TLLM_CHECK_WITH_INFO(offset <= dstBaseDesc.getLen() - size, "AgentConnection::send destination out of bounds"); + TLLM_CHECK_WITH_INFO(dstBaseDesc.getAddr() <= std::numeric_limits::max() - offset, + "AgentConnection::send destination address overflow"); MemoryDesc dstDesc{dstBaseDesc.getAddr() + offset, size, dstBaseDesc.getDeviceId()}; TLLM_LOG_DEBUG( "send dstDesc: %p, size: %ld ,validSegmentIdx: %ld", dstDesc.getAddr(), size, mSenderState.validSegmentIdx); @@ -386,13 +432,75 @@ AgentConnection const* AgentConnectionManager::recvConnectionAndRequestInfo( erase = true; requestInfo = requestAndBufferInfo.mRequestInfo; - auto address = requestAndBufferInfo.mAddress; - auto bufferDescs = std::move(requestAndBufferInfo.mBufferDescs); + auto const& address = requestAndBufferInfo.mAddress; + auto const& bufferDescsRef = requestAndBufferInfo.mBufferDescs; auto metadataOpt = requestAndBufferInfo.mMetadata; - auto connectionIdx = requestAndBufferInfo.mValidConnectionIdx; - auto remoteAgentName = requestAndBufferInfo.mAgentName; + auto const connectionIdx = requestAndBufferInfo.mValidConnectionIdx; + auto const& remoteAgentName = requestAndBufferInfo.mAgentName; + auto const& bufferKindsRef = requestAndBufferInfo.mBufferKinds; + + TLLM_CHECK_WITH_INFO(agent == remoteAgentName, + "AgentConnectionManager received RequestAndBufferInfo from '%s' with embedded agent '%s'", + agent.c_str(), remoteAgentName.c_str()); + auto const* expectedAgentState = findAgentState(mCommState, remoteAgentName); + if (expectedAgentState != nullptr) + { + TLLM_CHECK_WITH_INFO(address == expectedAgentState->mConnectionInfo, + "AgentConnectionManager received mismatched connection info for agent '%s'", + remoteAgentName.c_str()); + } + else + { + std::scoped_lock remoteInfoLock(mRemoteConnectionInfoMutex); + auto [remoteInfoIt, inserted] = mRemoteConnectionInfo.emplace(remoteAgentName, address); + TLLM_CHECK_WITH_INFO(inserted || remoteInfoIt->second == address, + "AgentConnectionManager received mismatched connection info for dynamic agent '%s'", + remoteAgentName.c_str()); + } + TLLM_CHECK_WITH_INFO(connectionIdx >= 0, + "AgentConnectionManager received negative connection index for agent '%s'", + remoteAgentName.c_str()); + TLLM_CHECK_WITH_INFO(!bufferDescsRef.empty(), + "AgentConnectionManager received empty destination descriptors from agent '%s'", + remoteAgentName.c_str()); + TLLM_CHECK_WITH_INFO(bufferDescsRef.size() == bufferKindsRef.size(), + "AgentConnectionManager received %zu descriptors but %zu buffer kinds from agent '%s'", + bufferDescsRef.size(), bufferKindsRef.size(), remoteAgentName.c_str()); + TLLM_CHECK_WITH_INFO(bufferDescsRef.size() <= mCacheTransBufferManagers.size(), + "AgentConnectionManager received too many destination descriptors from agent '%s'", + remoteAgentName.c_str()); + for (auto const kind : bufferKindsRef) + { + TLLM_CHECK_WITH_INFO(isValidBufferKind(kind), + "AgentConnectionManager received invalid buffer kind %u from agent '%s'", + static_cast(kind), remoteAgentName.c_str()); + bool isConfiguredKind = false; + for (auto const configuredKind : mBufferKinds) + { + if (configuredKind == kind) + { + isConfiguredKind = true; + break; + } + } + TLLM_CHECK_WITH_INFO(isConfiguredKind, + "AgentConnectionManager received unconfigured buffer kind %u from agent '%s'", + static_cast(kind), remoteAgentName.c_str()); + } + TLLM_CHECK_WITH_INFO(requestInfo.getTransState().getCacheState().has_value(), + "AgentConnectionManager received request without cache state from agent '%s'", + remoteAgentName.c_str()); + TLLM_CHECK_WITH_INFO(requestInfo.getTransState().getCommState().has_value(), + "AgentConnectionManager received request without comm state from agent '%s'", + remoteAgentName.c_str()); + TLLM_LOG_DEBUG(" recv Address:%s", address.c_str()); auto connection = connect(remoteAgentName, address, metadataOpt, true); + TLLM_CHECK_WITH_INFO( + m_Agent->checkRemoteDescs(remoteAgentName, MemoryDescs{MemoryType::kVRAM, bufferDescsRef}), + "AgentConnectionManager received unregistered destination descriptors from agent '%s'", + remoteAgentName.c_str()); + auto bufferDescs = std::move(requestAndBufferInfo.mBufferDescs); auto bufferKinds = std::move(requestAndBufferInfo.mBufferKinds); std::optional> kvOffsetRatio; @@ -410,10 +518,11 @@ AgentConnection const* AgentConnectionManager::recvConnectionAndRequestInfo( { if (!kvOffsetRatio) { - kvOffsetRatio - = computeSendOffsetRatio(requestInfo.getTransState().getCacheState().value(), - requestInfo.getTransState().getCommState()->getSelfIdx(), mCacheState, - connectionIdx); + auto kvTargetInfo + = targetIRanks(mCacheState, requestInfo.getTransState().getCacheState().value(), + requestInfo.getTransState().getCommState()->getSelfIdx()); + validateConnectionIdx(connectionIdx, kvTargetInfo, "KV", remoteAgentName); + kvOffsetRatio = computeSendOffsetRatio(kvTargetInfo, connectionIdx); } offsetRatios.push_back(*kvOffsetRatio); break; @@ -425,6 +534,7 @@ AgentConnection const* AgentConnectionManager::recvConnectionAndRequestInfo( auto rnnTargetInfo = targetIRanksForRnn(mCacheState, requestInfo.getTransState().getCacheState().value(), requestInfo.getTransState().getCommState()->getSelfIdx()); + validateConnectionIdx(connectionIdx, rnnTargetInfo, "RNN", remoteAgentName); size_t rnnOffsetLayer = 0; for (int ri = 0; ri < connectionIdx; ri++) { diff --git a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h index 8ec948cfafeb..25283d1341a1 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h @@ -331,6 +331,9 @@ class AgentConnectionManager : public ConnectionManager private: std::map> mConnections; std::mutex mConnectionsMutex; + /// Connection info for dynamically discovered agents that are not listed in mCommState. + std::map mRemoteConnectionInfo; + std::mutex mRemoteConnectionInfoMutex; CommState mCommState; CacheState mCacheState; std::optional mRnnCacheState; From ebbbec419eb9d21e475bd05134e06059f541d910 Mon Sep 17 00:00:00 2001 From: Pernyyang <707042815@qq.com> Date: Sat, 30 May 2026 02:50:41 +0800 Subject: [PATCH 134/308] [None][fix] Resolve NVML device index mismatch in get_numa_aware_cpu_affinity when CUDA_VISIBLE_DEVICES is set (#12985) Signed-off-by: pernyyang Co-authored-by: Kanghwan <861393+karljang@users.noreply.github.com> --- tensorrt_llm/llmapi/utils.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/llmapi/utils.py b/tensorrt_llm/llmapi/utils.py index ef501cd0894f..82adc4f7da82 100644 --- a/tensorrt_llm/llmapi/utils.py +++ b/tensorrt_llm/llmapi/utils.py @@ -542,6 +542,9 @@ def get_numa_aware_cpu_affinity(device_id): Args: device_id: The CUDA device ID to query for optimal CPU affinity. + This is the logical CUDA device index (after + CUDA_VISIBLE_DEVICES remapping). The function will + resolve it to the physical NVML device index. Returns: List of CPU IDs representing the optimal CPU affinity mask for the device. @@ -563,6 +566,35 @@ def get_numa_aware_cpu_affinity(device_id): import pynvml pynvml.nvmlInit() + # Resolve the physical NVML device index from the logical CUDA + # device_id. NVML always enumerates *all* GPUs on the system + # regardless of CUDA_VISIBLE_DEVICES, so when the user restricts + # visibility (e.g. CUDA_VISIBLE_DEVICES=3,4), logical device 0 + # actually corresponds to physical GPU 3. + cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if cuda_visible is not None and cuda_visible.strip(): + visible_tokens = [ + x.strip() for x in cuda_visible.split(",") if x.strip() + ] + if 0 <= device_id < len(visible_tokens): + token = visible_tokens[device_id] + if token.isdigit(): + nvml_device_id = int(token) + else: + logger.warning( + f"CUDA_VISIBLE_DEVICES token '{token}' is non-numeric; " + f"falling back to device_id ({device_id}) as NVML index." + ) + nvml_device_id = device_id + else: + logger.warning( + f"device_id {device_id} exceeds CUDA_VISIBLE_DEVICES " + f"list length ({len(visible_tokens)}), falling back to " + f"device_id as NVML index.") + nvml_device_id = device_id + else: + nvml_device_id = device_id + # Get the number of bits per ulong c_ulong_bits = ctypes.sizeof(ctypes.c_ulong) * 8 @@ -571,7 +603,7 @@ def get_numa_aware_cpu_affinity(device_id): # Get the optimal CPU affinity for this device according to the NUMA # topology - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id) + handle = pynvml.nvmlDeviceGetHandleByIndex(nvml_device_id) affinity_masks = pynvml.nvmlDeviceGetCpuAffinity(handle, cpu_set_size) # Convert CPU masks to python list From 74d7c3acbb0cf1670dc3284970a5bf3a5b468187 Mon Sep 17 00:00:00 2001 From: tburt-nv <195370667+tburt-nv@users.noreply.github.com> Date: Fri, 29 May 2026 15:48:10 -0400 Subject: [PATCH 135/308] [None][infra] revert #13607 (#14757) --- docker/Dockerfile.multi | 9 +- docker/Makefile | 4 - jenkins/Build.groovy | 7 +- jenkins/BuildDockerImage.groovy | 56 +++-- scripts/generate_container_oss_attribution.sh | 2 +- scripts/generate_cpp_dependency_json.py | 221 ------------------ scripts/get_wheel_from_package.py | 5 - 7 files changed, 30 insertions(+), 274 deletions(-) delete mode 100644 scripts/generate_cpp_dependency_json.py diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index ab8d8ede5c5e..6e20b8f3d733 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -119,14 +119,8 @@ ENV CCACHE_DIR=/root/.cache/ccache ARG GITHUB_MIRROR="" ARG BUILD_WHEEL_ARGS="--clean --benchmarks" ARG BUILD_WHEEL_SCRIPT="scripts/build_wheel.py" -ARG PREPARE_WHEEL_FROM_BUILD_STAGE="" RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACHE_DIR} \ - --mount=type=secret,id=GITLAB_TOKEN \ - GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} && \ - if [ "${PREPARE_WHEEL_FROM_BUILD_STAGE}" != "true" ]; then \ - GITLAB_TOKEN=$(cat /run/secrets/GITLAB_TOKEN 2>/dev/null) \ - python3 scripts/generate_cpp_dependency_json.py --deps-dir cpp/build/_deps --output-dir ./; \ - fi + GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} FROM ${DEVEL_IMAGE} AS release @@ -136,7 +130,6 @@ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,from=wheel,sour COPY README.md ./ COPY --from=wheel /src/tensorrt_llm/build/tensorrt_llm*.whl ./ -COPY --from=wheel /src/tensorrt_llm/third-party-sources.json ./ COPY docs docs COPY cpp/include include diff --git a/docker/Makefile b/docker/Makefile index e2a74c15ac36..9a9c44ce8b56 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,5 +1,3 @@ -COMMA := , - # Default base image for the docker build as defined in Dockerfile.multi BASE_IMAGE ?= $(shell grep '^ARG BASE_IMAGE=' Dockerfile.multi | grep -o '=.*' | tr -d '="') BASE_TAG ?= $(shell grep '^ARG BASE_TAG=' Dockerfile.multi | grep -o '=.*' | tr -d '="') @@ -96,7 +94,6 @@ base_pull: $(if $(TRITON_BASE_TAG), --build-arg TRITON_BASE_TAG=$(TRITON_BASE_TAG)) \ $(if $(BUILD_WHEEL_ARGS), --build-arg BUILD_WHEEL_ARGS="$(BUILD_WHEEL_ARGS)") \ $(if $(BUILD_WHEEL_SCRIPT), --build-arg BUILD_WHEEL_SCRIPT="$(BUILD_WHEEL_SCRIPT)") \ - $(if $(PREPARE_WHEEL_FROM_BUILD_STAGE), --build-arg PREPARE_WHEEL_FROM_BUILD_STAGE="$(PREPARE_WHEEL_FROM_BUILD_STAGE)") \ $(if $(TORCH_INSTALL_TYPE), --build-arg TORCH_INSTALL_TYPE="$(TORCH_INSTALL_TYPE)") \ $(if $(CUDA_VERSION), --build-arg CUDA_VER="$(CUDA_VERSION)") \ $(if $(CUDNN_VERSION), --build-arg CUDNN_VER="$(CUDNN_VERSION)") \ @@ -111,7 +108,6 @@ base_pull: $(if $(SH_ENV), --build-arg SH_ENV="$(SH_ENV)") \ $(if $(BASH_ENV), --build-arg BASH_ENV="$(BASH_ENV)") \ $(if $(STAGE), --target $(STAGE)) \ - $(if $(GITLAB_TOKEN), --secret id=GITLAB_TOKEN$(COMMA)env=GITLAB_TOKEN) \ --file Dockerfile.multi \ --tag $(IMAGE_WITH_TAG) \ .. diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index a2cfc2bcb6c4..405280496af8 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -400,9 +400,6 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) withCredentials([usernamePassword(credentialsId: "urm-artifactory-creds", usernameVariable: 'CONAN_LOGIN_USERNAME', passwordVariable: 'CONAN_PASSWORD')]) { sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]} --benchmarks" } - withCredentials([string(credentialsId: 'svc_tensorrt_llm_oss_gitlab_token_secret', variable: 'GITLAB_TOKEN')]) { - sh "cd ${LLM_ROOT} && python3 scripts/generate_cpp_dependency_json.py --deps-dir cpp/build/_deps --output-dir ./ --token ${GITLAB_TOKEN}" - } if (is_linux_x86_64) { sh "cd ${LLM_ROOT} && python3 scripts/build_cpp_examples.py" } @@ -414,10 +411,8 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) def tritonShortTag = "r26.02" sh "cd ${LLM_ROOT}/triton_backend/inflight_batcher_llm && mkdir build && cd build && cmake .. -DTRTLLM_DIR=${llmPath} -DTRITON_COMMON_REPO_TAG=${tritonShortTag} -DTRITON_CORE_REPO_TAG=${tritonShortTag} -DTRITON_THIRD_PARTY_REPO_TAG=${tritonShortTag} -DTRITON_BACKEND_REPO_TAG=${tritonShortTag} -DUSE_CXX11_ABI=ON && make -j${buildJobs} install" - // Step 3.1: packaging wheels into tarfile + // Step 3: packaging wheels into tarfile sh "cp ${LLM_ROOT}/build/tensorrt_llm-*.whl TensorRT-LLM/" - // Step 3.2: packaging cmake dependency source json into tarfile - sh "cp ${LLM_ROOT}/third-party-sources.json TensorRT-LLM/" // Step 4: packaging tritonserver artifacts into tarfile sh "mkdir -p TensorRT-LLM/triton_backend/inflight_batcher_llm/" diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index c2cc7c7c1813..595ff8893f2c 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -236,7 +236,7 @@ def prepareWheelFromBuildStage(dockerfileStage, arch) { def wheelScript = 'scripts/get_wheel_from_package.py' def wheelArgs = "--arch ${arch} --timeout ${WAIT_TIME_FOR_BUILD_STAGE} --artifact_path " + env.uploadPath - return " BUILD_WHEEL_SCRIPT=${wheelScript} BUILD_WHEEL_ARGS='${wheelArgs}' PREPARE_WHEEL_FROM_BUILD_STAGE='true'" + return " BUILD_WHEEL_SCRIPT=${wheelScript} BUILD_WHEEL_ARGS='${wheelArgs}'" } def buildImage(config, imageKeyToTag) @@ -347,36 +347,34 @@ def buildImage(config, imageKeyToTag) sh "env | sort" def randomSleep = (Math.random() * 600 + 600).toInteger() trtllm_utils.llmExecStepWithRetry(this, script: "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}", sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) - withCredentials([string(credentialsId: 'svc_tensorrt_llm_oss_gitlab_token_secret', variable: 'GITLAB_TOKEN')]) { - try { - trtllm_utils.llmExecStepWithRetry(this, script: """ - cd ${LLM_ROOT} && make -C docker ${target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${imageWithTag} \ - STAGE=${dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} - """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) - } catch (InterruptedException ex) { + try { + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${imageWithTag} \ + STAGE=${dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} + """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) + } catch (InterruptedException ex) { + throw ex + } catch (Exception ex) { + if (buildWheelArgs.trim().isEmpty()) { throw ex - } catch (Exception ex) { - if (buildWheelArgs.trim().isEmpty()) { - throw ex - } - echo "Build failed with wheel arguments, retrying without them" - buildWheelArgs = "" - trtllm_utils.llmExecStepWithRetry(this, script: """ - cd ${LLM_ROOT} && make -C docker ${target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${imageWithTag} \ - STAGE=${dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} - """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) } - } // withCredentials + echo "Build failed with wheel arguments, retrying without them" + buildWheelArgs = "" + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${imageWithTag} \ + STAGE=${dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} ${buildWheelArgs} + """, sleepInSecs: randomSleep, numRetries: 6, shortCommondRunTimeMax: 7200) + } if (target == "ngc-release") { imageKeyToTag["NGC Release Image ${config.arch}"] = imageWithTag } diff --git a/scripts/generate_container_oss_attribution.sh b/scripts/generate_container_oss_attribution.sh index 978298891f99..f81a5127e2ea 100755 --- a/scripts/generate_container_oss_attribution.sh +++ b/scripts/generate_container_oss_attribution.sh @@ -47,7 +47,7 @@ mkdir -p "${OUTPUT_DIR}" # Generate the attribution file cat > "${OUTPUT_FILE}" << EOF -This container image includes open-source software whose source code is linked in /third-party-sources.json. +This container image includes open-source software whose source code is archived in the /third-party-source directory or at ${OSS_URL}. For further inquiries or assistance, contact us at oss-requests@nvidia.com EOF diff --git a/scripts/generate_cpp_dependency_json.py b/scripts/generate_cpp_dependency_json.py deleted file mode 100644 index 934a5832328d..000000000000 --- a/scripts/generate_cpp_dependency_json.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Generate a JSON manifest of third-party source URLs used in the CMake build. - -This script produces a record of third-party dependencies exactly as consumed -during the build. Each dependency is mirrored to the GitLab OSS components group -(https://gitlab.com/nvidia/tensorrt-llm/oss-components), and the resulting -third-party-sources.json is copied into the container image so that source -references are distributed alongside build artifacts, satisfying open-source -license obligations in a traceable and auditable way. -""" - -import argparse -import json -import logging -import os -import pathlib -import time -import urllib.parse -import urllib.request - -logger = logging.getLogger(__name__) - - -GITLAB_OSS_GROUP = "nvidia/tensorrt-llm/oss-components" -GITLAB_API_BASE = "https://gitlab.com/api/v4" - -REPO_URL_OVERWRITE = {"deep_ep_download": "https://github.com/deepseek-ai/DeepEP"} - -_FETCH_CONTENT_JSON = pathlib.Path(__file__).parent.parent / "3rdparty" / "fetch_content.json" - - -def _load_fetch_content_index() -> dict[str, dict]: - """Return a name->entry mapping from fetch_content.json.""" - data = json.loads(_FETCH_CONTENT_JSON.read_text()) - return {dep["name"]: dep for dep in data.get("dependencies", [])} - - -def get_source_info( - deps_dir: pathlib.Path, - package_name: str, - fetch_content_index: dict[str, dict] | None = None, -) -> dict[str, str]: - """Return {'url': ..., 'tag': ...} for package_name. - - Read directly from fetch_content.json. - """ - index = fetch_content_index or _load_fetch_content_index() - dep = index.get(package_name, {}) - repo = dep.get("git_repository", "").replace("${github_base_url}", "https://github.com") - return {"url": repo, "tag": dep.get("git_tag", "")} - - -def check_oss_components(package_name: str) -> tuple[str, int] | None: - """Return (web_url, project_id) if package_name exists in oss-components, else None.""" - project_path = urllib.parse.quote(f"{GITLAB_OSS_GROUP}/{package_name}", safe="") - url = f"{GITLAB_API_BASE}/projects/{project_path}" - req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) - try: - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return data["web_url"], data["id"] - except urllib.error.HTTPError as e: - if e.code == 404: - return None - raise - - -def commit_exists_in_project(project_id: int, ref: str) -> bool: - """Return True if ref (tag or commit SHA) exists in the GitLab project.""" - encoded_ref = urllib.parse.quote(ref, safe="") - url = f"{GITLAB_API_BASE}/projects/{project_id}/repository/commits/{encoded_ref}" - req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) - try: - with urllib.request.urlopen(req) as resp: - resp.read() - return True - except urllib.error.HTTPError as e: - if e.code == 404: - return False - raise - - -def get_namespace_id() -> int: - """Return the numeric namespace ID for GITLAB_OSS_GROUP.""" - group_path = urllib.parse.quote(GITLAB_OSS_GROUP, safe="") - url = f"{GITLAB_API_BASE}/groups/{group_path}" - req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return data["id"] - - -def create_oss_component( - package_name: str, namespace_id: int, upstream_url: str -) -> tuple[str, int]: - """Create a new project under oss-components with a pull mirror and return (web_url, project_id).""" - payload = json.dumps( - { - "name": package_name, - "namespace_id": namespace_id, - "visibility": "public", - "import_url": upstream_url, - "mirror": True, - } - ).encode() - req = urllib.request.Request( - f"{GITLAB_API_BASE}/projects", - data=payload, - headers={"PRIVATE-TOKEN": TOKEN, "Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return data["web_url"], data["id"] - - -def wait_for_mirror(project_id: int, poll_interval: int = 10, timeout: int = 300) -> None: - """Poll until the project mirror import finishes; return False if it times out.""" - url = f"{GITLAB_API_BASE}/projects/{project_id}" - req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": TOKEN}) - deadline = time.monotonic() + timeout - while True: - with urllib.request.urlopen(req) as resp: - status = json.loads(resp.read()).get("import_status") - if status == "finished": - return - if status == "failed": - raise RuntimeError(f"Mirror import failed for project {project_id}") - if time.monotonic() >= deadline: - raise TimeoutError( - f"Mirror import for project {project_id} still {status!r} after {timeout}s" - ) - logger.info("Mirror import status: %s, retrying in %ds...", status, poll_interval) - time.sleep(poll_interval) - - -def main(): - global TOKEN - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--deps-dir", - type=pathlib.Path, - required=True, - help="Path to the third party dependencies directory, e.g. ${CMAKE_BINARY_DIR}/_deps", - ) - parser.add_argument( - "--output-dir", - type=pathlib.Path, - required=True, - help="Path to the output directory where third party sources will be copied", - ) - parser.add_argument( - "--token", - default=os.environ.get("GITLAB_TOKEN"), - help="GitLab private token (defaults to $GITLAB_TOKEN env var)", - ) - - args = parser.parse_args() - if not args.token: - parser.error("A GitLab token is required: pass --token or set $GITLAB_TOKEN") - TOKEN = args.token - - src_dirs = list(sorted(args.deps_dir.glob("*-src"))) - if not src_dirs: - raise ValueError(f"No source directories found in {args.deps_dir}") - - namespace_id: int | None = None - fetch_content = json.loads(_FETCH_CONTENT_JSON.read_text()) - dep_source_index = {dep["name"]: dep for dep in fetch_content.get("dependencies", [])} - third_party_source_list = [] - - for src_dir in src_dirs: - package_name = src_dir.name.removesuffix("-src") - source_info = get_source_info(args.deps_dir, package_name) - if package_name in REPO_URL_OVERWRITE: - source_info["url"] = REPO_URL_OVERWRITE[package_name] - logger.info( - "%s -> upstream url=%s tag=%s", package_name, source_info["url"], source_info["tag"] - ) - result = check_oss_components(package_name) - if result is not None: - oss_url, project_id = result - logger.info("%s -> found in oss-components: %s", package_name, oss_url) - else: - logger.info("%s -> NOT found in oss-components, creating repo", package_name) - if namespace_id is None: - namespace_id = get_namespace_id() - oss_url, project_id = create_oss_component( - package_name, namespace_id, source_info["url"] - ) - logger.info("%s -> created: %s", package_name, oss_url) - logger.info("%s -> waiting for mirror import to finish", package_name) - wait_for_mirror(project_id) - logger.info("%s -> mirror import finished", package_name) - - tag = source_info["tag"] - if tag and commit_exists_in_project(project_id, tag): - logger.info("%s -> ref %r confirmed in oss-components, updating url", package_name, tag) - if package_name not in dep_source_index: - logger.warning("%s -> not found in fetch_content.json, skipping", package_name) - continue - third_party_source_list.append( - {**dep_source_index[package_name], "git_repository": oss_url} - ) - else: - logger.warning( - "%s -> ref %r not found in oss-components repo, keeping upstream url", - package_name, - tag, - ) - - args.output_dir.mkdir(parents=True, exist_ok=True) - output_path = args.output_dir / "third-party-sources.json" - with open(output_path, "w", encoding="utf-8") as f: - json.dump(third_party_source_list, f, indent=2) - f.write("\n") - logger.info("Wrote oss fetch_content to %s", output_path) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/scripts/get_wheel_from_package.py b/scripts/get_wheel_from_package.py index 043a3ab2bdcb..cb604482c27b 100644 --- a/scripts/get_wheel_from_package.py +++ b/scripts/get_wheel_from_package.py @@ -99,11 +99,6 @@ def get_wheel_from_package(arch, artifact_path, timeout): else: print(f"Warning: Benchmark file not found: {src_path}") - third_party_sources = tmp_dir / "third-party-sources.json" - if third_party_sources.exists(): - shutil.copy2(third_party_sources, llm_root / "third-party-sources.json") - print(f"Copied third-party-sources.json -> {build_dir}") - shutil.rmtree(tmp_dir) if os.path.exists(tarfile_name): From 097dab14969ac6bdd16accba18af939fb636cfbe Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Sat, 30 May 2026 03:10:59 +0000 Subject: [PATCH 136/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/auto_deploy/poetry.lock | 38 +-- .../llm-eval/lm-eval-harness/poetry.lock | 38 +-- .../models/contrib/hyperclovax/poetry.lock | 38 +-- .../examples/models/contrib/stdit/poetry.lock | 38 +-- .../examples/models/core/mixtral/poetry.lock | 38 +-- .../examples/models/core/mllama/poetry.lock | 38 +-- .../examples/models/core/qwenvl/poetry.lock | 38 +-- .../examples/models/core/whisper/poetry.lock | 38 +-- security_scanning/examples/serve/poetry.lock | 292 +++++++++++++++++- .../examples/serve/pyproject.toml | 2 +- .../examples/trtllm-eval/poetry.lock | 38 +-- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 105 +++---- security_scanning/pyproject.toml | 5 +- security_scanning/triton_backend/poetry.lock | 46 +-- 15 files changed, 528 insertions(+), 268 deletions(-) diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 47c0c73a4922..e5e04ecc6a41 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -544,31 +544,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index 8592cbac562b..cf6f9c6a75ae 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -465,31 +465,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index 5308046106bd..60410edfcb59 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -108,31 +108,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/stdit/poetry.lock b/security_scanning/examples/models/contrib/stdit/poetry.lock index 7104f94def36..dab3034c5c0e 100644 --- a/security_scanning/examples/models/contrib/stdit/poetry.lock +++ b/security_scanning/examples/models/contrib/stdit/poetry.lock @@ -581,31 +581,31 @@ ssh = ["bcrypt (>=3.1.5)"] [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock index 760f2f82a92b..6246a2e7b3c9 100644 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ b/security_scanning/examples/models/core/mixtral/poetry.lock @@ -197,31 +197,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/mllama/poetry.lock b/security_scanning/examples/models/core/mllama/poetry.lock index f90323bf02f5..32d54239440a 100644 --- a/security_scanning/examples/models/core/mllama/poetry.lock +++ b/security_scanning/examples/models/core/mllama/poetry.lock @@ -190,31 +190,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 8ee0142f486a..83eb04386171 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -592,31 +592,31 @@ test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist" [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/whisper/poetry.lock b/security_scanning/examples/models/core/whisper/poetry.lock index f8da1fe1653a..0997bda7deed 100644 --- a/security_scanning/examples/models/core/whisper/poetry.lock +++ b/security_scanning/examples/models/core/whisper/poetry.lock @@ -523,31 +523,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index 49dcf063e28d..a545112f8821 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -185,13 +185,13 @@ typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} [[package]] name = "aiperf" -version = "0.8.0" +version = "0.9.0" description = "AIPerf is a package for performance testing of AI models" optional = false python-versions = "<3.14,>=3.10" groups = ["main"] files = [ - {file = "aiperf-0.8.0-py3-none-any.whl", hash = "sha256:0a3ad3b557bd0058bb676b7a14367b5b375b40f02352d4556e2965bc8ba9fb2d"}, + {file = "aiperf-0.9.0-py3-none-any.whl", hash = "sha256:5f35c3223a6cb7d2377feea74fd0fb8ec286baef6b3d4fa91c353c60ce7a6d8d"}, ] [package.dependencies] @@ -213,6 +213,7 @@ matplotlib = ">=3.10.0" msgspec = ">=0.19.0,<1.0.0" numpy = ">=1.26.4,<3" nvidia-ml-py = "*" +optuna = ">=3.6" orjson = ">=3.11.7,<3.12.0" pandas = ">=2.3.3,<2.4.0" pillow = ">=12.2.0,<12.3.0" @@ -241,7 +242,34 @@ uvloop = {version = ">=0.22.1", markers = "platform_system != \"Windows\""} zstandard = ">=0.25.0" [package.extras] -dev = ["black (>=25.1.0)", "httpx (>=0.27.0)", "hypothesis (>=6.0.0)", "looptime (>=0.5)", "pre-commit (>=4.2.0)", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-xdist (>=3.8.0)", "ruff (>=0.14.0,<0.15.0)", "trustme (>=1.0.0)"] +accuracy = ["latex2sympy2-extended (>=1.0.6)", "lighteval (>=0.13.0)", "sympy (>=1.14.0)"] +botorch = ["botorch (>=0.10)", "gpytorch (>=1.11)", "optuna-integration (>=3.6)", "torch (>=2.0)"] +dev = ["black (>=25.1.0)", "httpx (>=0.27.0)", "hypothesis (>=6.0.0)", "jsonschema (>=4.0.0)", "looptime (>=0.5)", "mlflow (>=3.10.0,<4.0.0)", "opentelemetry-exporter-otlp-proto-http (>=1.24.0,<2.0.0)", "opentelemetry-sdk (>=1.24.0,<2.0.0)", "pre-commit (>=4.2.0)", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-xdist (>=3.8.0)", "ruff (>=0.14.0,<0.15.0)", "trustme (>=1.0.0)"] +mlflow = ["mlflow (>=3.10.0,<4.0.0)"] +optuna = ["botorch (>=0.10)", "gpytorch (>=1.11)", "optuna-integration (>=3.6)", "torch (>=2.0)"] +otel = ["opentelemetry-exporter-otlp-proto-http (>=1.24.0,<2.0.0)", "opentelemetry-sdk (>=1.24.0,<2.0.0)"] +test = ["httpx (>=0.27.0)", "hypothesis (>=6.0.0)", "jsonschema (>=4.0.0)", "looptime (>=0.5)", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-xdist (>=3.8.0)", "trustme (>=1.0.0)"] + +[[package]] +name = "alembic" +version = "1.18.4" +description = "A database migration tool for SQLAlchemy." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a"}, + {file = "alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc"}, +] + +[package.dependencies] +Mako = "*" +SQLAlchemy = ">=1.4.23" +tomli = {version = "*", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.12" + +[package.extras] +tz = ["tzdata"] [[package]] name = "annotated-doc" @@ -757,6 +785,24 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colorlog" +version = "6.10.1" +description = "Add colours to the output of Python's logging module." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "colorlog-6.10.1-py3-none-any.whl", hash = "sha256:2d7e8348291948af66122cff006c9f8da6255d224e7cf8e37d8de2df3bad8c9c"}, + {file = "colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + [[package]] name = "contourpy" version = "1.3.2" @@ -1504,6 +1550,100 @@ files = [ {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, ] +[[package]] +name = "greenlet" +version = "3.5.1" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" +files = [ + {file = "greenlet-3.5.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7eacb17a9d41538a2bc4912eba5ef13823c83cb69e4d141d0813debe7163187f"}, + {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5cc9606aa5f4e0bde0d3bd502b44f743864c3ffa5cfa1011b1e30f5aa02366f"}, + {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3d35f87c7253b715d13d679e0783d845910144f282cb939fe1ba4ac8616269c"}, + {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:00929c98ec525fd9bf075875d8c5f6a983a90906cdf78a66e6de2d8e466c2a19"}, + {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:540dae7b956209af4d70a3be35927b4055f617763771e5e84a5255bea934d2f5"}, + {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:001775efe7b8e758861294c7a27c28af87f3f3f1c20468a2bc618c45b346c061"}, + {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed8cdb691169715a9a492844a83246f090182247d1a5031dc78a403f68ba1e97"}, + {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d59e840387076a51016777a9328b3f2c427c6f9208a6e958bad251be50a648d"}, + {file = "greenlet-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:b9152fca4a6466e114aaec745ae61cba739903a109754a9d4e1262f01e9259b1"}, + {file = "greenlet-3.5.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:73f78f9b9f0a5c06e5c946ba1e8e36f5114923b6be109ee618c54f079c3ea14f"}, + {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0cbed8bb44e23c5b199f888f4e4ce096b45ad9f25ff74a7ad0213875e936bb2"}, + {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a203a8bd0acb0701653d3bbb26e404854a68674139ed5cbb778830f42b09bb33"}, + {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ebeb75c81211f5c702576cf81f315e77e23cfdb2c7c6fcb9dd143e6de35c360"}, + {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a271fcd66c74615cda6a964fda3f304267a12e50a084472218a39bb0376f563"}, + {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:017a544f0385d441e88714160d089d6900ef46c9eff9d99b6715a5ef2d127747"}, + {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ded7b068c7c31c1a8657d4fd42d886b3e051ae29f88b80c5ff9d502257b0f071"}, + {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0932b81d72f552ded9d810d00021b64d89f2195a91ce115b893f943b7a4ab3c"}, + {file = "greenlet-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:88e300d136eac057b2397aa1cfd7328b4c87c7eb66a09c7bc6a1292234db474e"}, + {file = "greenlet-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:cc6ab7e555c8a112ad3a76e368e86e12a2754bcae1652a5602e133ec7b635523"}, + {file = "greenlet-3.5.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:fa4f98af3a528f0c3fd592a26df7f376f93329c8f4d987f6bb979057af8bf5e2"}, + {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffea73584b216150eab159b6d12348fb253e68757974de1e2c40d8a318ac89ed"}, + {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1072b4f9edcc1e192d9283a66a3e68d6b84c561de33a83d7858beb9ba1effe10"}, + {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89101bfd5011e069be974903cb3a4e4523845e4ece2d62dcd8d358933c0ef249"}, + {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b"}, + {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:e6cd99ea59dd5d89f0c956606571d79bfe6f68c9eb7f4a4083a41a7f1587edee"}, + {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a5ea42a752d47a145eae922b605cd1634665ac3d5ec1e72402d5048e8d60d207"}, + {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823"}, + {file = "greenlet-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3c8bb982ad117d29478ef8f5533e97df21f1e2befd17a299257b0c96d1371c0b"}, + {file = "greenlet-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:80eb4b04dadc4e67df3fae179a32c4706a3f495bc7f22fc8a81115d5f5512188"}, + {file = "greenlet-3.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b"}, + {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ffdb3c0bb002c99cd8f298957e046c3dbf6006b5b7cdf11a4e19194624a0a0a"}, + {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283"}, + {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d40a890035c0058cadbdc4af7569800fd28a0e527a0fdbb7b5f9418f176846ce"}, + {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135"}, + {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:67821bb03e4e98664490edb787ff6af501194c29bbee0f5c1dfdcf1dc3d9d436"}, + {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd"}, + {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1"}, + {file = "greenlet-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9"}, + {file = "greenlet-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e"}, + {file = "greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07"}, + {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea"}, + {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2"}, + {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c09df69dc1712d131332054a858a3e5cca400967fa3a672e2324fbb0971448c"}, + {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c"}, + {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:a4764e0bfc6a4d114c865b32520805c16a990ef5f286a514413b05d5ecd6a23d"}, + {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0"}, + {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc"}, + {file = "greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3"}, + {file = "greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54"}, + {file = "greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad"}, + {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e"}, + {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986"}, + {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea37d5a157eb9493820d3792ac4ece28619a394391d2b9f2f78057d396ff0f0f"}, + {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e"}, + {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:8d8a23250ea3ec7b36de8fa4b541e9e2db3ee82915cc060ab0631609ad8b28de"}, + {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d"}, + {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78"}, + {file = "greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2"}, + {file = "greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541"}, + {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de"}, + {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64"}, + {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:10a9a1c0bfbc93d41156ffcb90c75fbc05544054faf15dcc1fdf9765f8b607f0"}, + {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5"}, + {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:ef08c1567c78074b22d1a200183d52d04a14df447bf70bcbb6a3507a48e776fc"}, + {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368"}, + {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26"}, + {file = "greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab"}, + {file = "greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6"}, + {file = "greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed"}, + {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244"}, + {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c"}, + {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a57b0d05a0448eed231d59c0ceb287dde984551e54cbc51ac2d4865712838e9c"}, + {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd"}, + {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:b0703c2cef53e01baec47f7a3868009913ad71ec678bbecb42a6f40895e4ce62"}, + {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e"}, + {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659"}, + {file = "greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e"}, + {file = "greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a"}, + {file = "greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil", "setuptools"] + [[package]] name = "h11" version = "0.16.0" @@ -1971,6 +2111,26 @@ files = [ {file = "logistro-2.0.1.tar.gz", hash = "sha256:8446affc82bab2577eb02bfcbcae196ae03129287557287b6a070f70c1985047"}, ] +[[package]] +name = "mako" +version = "1.3.12" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9"}, + {file = "mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -2657,6 +2817,33 @@ files = [ {file = "nvidia_ml_py-13.595.45.tar.gz", hash = "sha256:c9f34897fe0441ff35bc8f35baf80f830a20b0f4e6ce71e0a325bc0e66acf079"}, ] +[[package]] +name = "optuna" +version = "4.8.0" +description = "A hyperparameter optimization framework" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "optuna-4.8.0-py3-none-any.whl", hash = "sha256:c57a7682679c36bfc9bca0da430698179e513874074b71bebedb0334964ab930"}, + {file = "optuna-4.8.0.tar.gz", hash = "sha256:6f7043e9f8ecb5e607af86a7eb00fb5ec2be26c3b08c201209a73d36aff37a38"}, +] + +[package.dependencies] +alembic = ">=1.5.0" +colorlog = "*" +numpy = "*" +packaging = ">=20.0" +PyYAML = "*" +sqlalchemy = ">=1.4.2" +tqdm = "*" + +[package.extras] +checking = ["mypy", "mypy_boto3_s3", "ruff", "scipy-stubs ; python_version >= \"3.10\"", "types-PyYAML", "types-redis", "types-setuptools", "types-tqdm", "typing_extensions (>=3.10.0.0)"] +document = ["ase", "cmaes (>=0.12.0)", "fvcore", "kaleido (<0.4)", "lightgbm", "matplotlib (!=3.6.0)", "pandas", "pillow", "plotly (>=4.9.0)", "scikit-learn", "sphinx", "sphinx-copybutton", "sphinx-gallery", "sphinx-notfound-page", "sphinx_rtd_theme (>=1.2.0)", "torch", "torchvision"] +optional = ["boto3", "cmaes (>=0.12.0)", "google-cloud-storage", "greenlet", "grpcio", "matplotlib (!=3.6.0)", "pandas", "plotly (>=4.9.0)", "protobuf (>=5.28.1)", "redis", "scikit-learn (>=0.24.2)", "scipy", "torch"] +test = ["fakeredis[lua]", "greenlet", "grpcio", "kaleido (<0.4)", "moto", "protobuf (>=5.28.1)", "pytest", "pytest-xdist", "scipy (>=1.9.2)", "torch"] + [[package]] name = "orjson" version = "3.11.9" @@ -4618,6 +4805,103 @@ files = [ cffi = ">=1.0" numpy = "*" +[[package]] +name = "sqlalchemy" +version = "2.0.50" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "sqlalchemy-2.0.50-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7af6eeb84985bf840ba779018ff9424d61ff69b52e66b8789d3c8da7bf5341b2"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fe7822866f3a9fc5f3db21a290ce8961a53050115f05edf9402b6a5feb92a9f"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e1b0f6a4dcd9b4839e2320afb5df37a6981cbc20ff9c423ae11c5537bdbd21"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e195687f1af431c9515416288373b323b6eb599f774409814e89e9d603a56e39"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ea1a8a2db4b2217d456c8d7a873bfc605f06fe3584d315264ea18c2a17585d0b"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-win32.whl", hash = "sha256:68b154b08088b4ec32bb4d2958bfbb50e57549f91a4cd3e7f928e3553ed69031"}, + {file = "sqlalchemy-2.0.50-cp310-cp310-win_amd64.whl", hash = "sha256:66e374271ecb7101273f57af1a62446a953d327eec4f8089147de57c591bbacc"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1aa6e403663a9c43c8fef7ce4bdb4cf48bcd8d352e91deda2a99f963270bd508"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51b637a84f9fa35ae1f9017e786cb142974a25305085e1b378b3647a67f65ad3"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2dab927761d9108550f0cf8e66ff21af56f907a0ce0a689793db615e2b55f62c"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:545eae198d37bcf837a10ede3684e2af32458d6f35c597c35c2de7502dc38fc4"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fec460e18cdbb4c7773531122ce9a27e96c6ca17af3933941d94da475ad2c86"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-win32.whl", hash = "sha256:e6e814658818fd165e749e3d8490ef16cc7f379a118c37ada8b0589ffbaaac22"}, + {file = "sqlalchemy-2.0.50-cp311-cp311-win_amd64.whl", hash = "sha256:1c5f858fe79c9f5d8fda065c06186356acb7f8df3cd52dbd5ee3f200e4b144f5"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23ae23d8b9d344d30d0a92f06d45825024a5790f1c1dd4cf452636a50d3e58cb"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47b71b933e7b4ebad407c8fdfd70d2c4f08b78b3238bb30eebdd6eb32ca51b89"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:110fdac56ace278949f00de805edacbd6141e382d992f9ba28238b3a0827a600"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5e4ac70e9e757f6b3e87c0491ff034442ecd8dfd36d041a50564c322dafc0e"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:724f3dcbe53dd0151e3cb5e7ec4ba4c620bede579caacd16275dc35ce06e8615"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-win32.whl", hash = "sha256:1208050441471d003b7c8cb4054fb084f185cf35ac3f0ea270803865bca9939a"}, + {file = "sqlalchemy-2.0.50-cp312-cp312-win_amd64.whl", hash = "sha256:9d1af51558029a156a70986b7df88f042b3d158d7c8d8fb5072912d4b32d89c7"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:06a9210bdc5f4298cff0781087e2ff45683922252dacc452846373a58761f093"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b53784972ade4f8174b9aa661f31a06f8a936d2cfdd602913ff3c6dd40ae873"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31648fa14460537e768a7303b078e4344d208e0d23e06867c1f376a227ed82db"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03f4323c980ad0e918cc9e5369b015f759f4e534db5bbaf4dc36832c10d05064"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2b9dcc43afef8ac157cd92fce96985d6b8b0cfbd3df4d666f66b4d55a75d202f"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-win32.whl", hash = "sha256:60922d6599065ddca2c6f376b9aa2f41a6b85a271725e0909490bbc50b1998a5"}, + {file = "sqlalchemy-2.0.50-cp313-cp313-win_amd64.whl", hash = "sha256:287086e67275a212c4582d166a6fb03a65ccc5551d80866270ce0dd9f34eccd3"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c966932507a4d7d0a37314927dbfcd89720e3f37d2a1e3352e7ae7939fa8e8a0"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:faffef4bcc20a1892e65e155293d99d60855bbbc79250ab712819cfd56a8e6bb"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c206aec519a2e7bd08abbfb33436e325fd22c632d9c21a9047e376ce241646e"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bef4ac756363227ef6402a75fee025a4bc690f92328e825868939b3b3a446a6d"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96fbee6b19c19cd1556c8bf9419447cf2ec149ffcab7ab64348c23e54ef8547f"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-win32.whl", hash = "sha256:8f00e3eb43ba30eb1b238ee03a8a62309486d1321eda3328bb611e0340033ad8"}, + {file = "sqlalchemy-2.0.50-cp314-cp314-win_amd64.whl", hash = "sha256:15708c613cd5005b7dffe1f66ee6a63ee8f5e46799f71c70ebad74178c676a39"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3699dac4be410e97049a1658e9480da9cde956594aa0f3aebc60b88f21c5ba70"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96233858e3df43932ac11589e22520da6e8aeb624b03fedfeebb0e8ea213086"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4e70c46fad30c3bcc6a4708bc0130a3173e11a5b25f0ea4a9d8911b450f1f52"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1918a3cf564d16d95bca7301005f41ab2ad50b07cd3b9da50d3ed986db148d6a"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b00098cdbdbd38c7be3d568b0c9c3122b8c0ec62b911b57cd5e6e0254d60a76d"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-win32.whl", hash = "sha256:1fbd55a969d7ac44a98e3dec75016074f809fa08f871585ace58dde110d1bf3e"}, + {file = "sqlalchemy-2.0.50-cp314-cp314t-win_amd64.whl", hash = "sha256:c5c3cdb753a9004183e1ccb634b41611654c989e61bc68617ce878e46d6f1e51"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8e8af330cbb3a1931d3d6c91b239fc2ef135f7dd471dfa34c575028e0b1fa8"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eefd9a03cc0047b14153872d228499d048bd7deaf926109c9ec25b15157b8e23"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13b85b20f9ab714a666df9d8e72e253ec33c16c7e1e375c877e5bf6367a3e917"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:27b7062af702c61994e8806ad87e42d0a2c879e0a8e5c61c7f69d81dabe24fdf"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2c1920cde9d741ba3dda9b1aa5acd8c23ea17780ccfb2252d01878d5d0d628d3"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-win32.whl", hash = "sha256:7b1ddb7b5fc60dfa9df6a487f06a143c77def47c0351849da2bcea59b244a56c"}, + {file = "sqlalchemy-2.0.50-cp38-cp38-win_amd64.whl", hash = "sha256:0e104e196f457ec608eb8af736c5eb4c6bc58f481b546f485a7f9c628ee532be"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:409a8121b917116b035bedc5e532ad470c74a2d279f6c302100985b6304e9f9e"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9602c07b03e1449747ecb69f9998a7194a589124475788b370adce57c9e9a56e"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d10700bd519573f6ce5badbabbfe7f5baea84cdf370f2cbbfb4be28dfddbf1d"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7e36efdcc5493f8024ec873a4ee3855bfd2de0c5b19eba16f920e9d2a0d28622"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:adc0fe7d38d8c8058f7421c25508fcbc74df38233a42aa8324409844122dce8f"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-win32.whl", hash = "sha256:0a31c5963d58d3e3d11c5b97709e248305705de1fdf51ec3bf396674c5898b7e"}, + {file = "sqlalchemy-2.0.50-cp39-cp39-win_amd64.whl", hash = "sha256:83a9fce296b7e052316d8c6943237b31b9c00f58ca9c253f2d165df52637a293"}, + {file = "sqlalchemy-2.0.50-py3-none-any.whl", hash = "sha256:92064363517a3ff8212b5a93b8c62876579d8dfd1ca5b561335f30152d884fa9"}, + {file = "sqlalchemy-2.0.50.tar.gz", hash = "sha256:af5607d11ef90fd6a5c0549fe0045dce1663d427426bcfb506dcb5346a85a3b9"}, +] + +[package.dependencies] +greenlet = {version = ">=1", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (>=1)"] +aioodbc = ["aioodbc", "greenlet (>=1)"] +aiosqlite = ["aiosqlite", "greenlet (>=1)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (>=1)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (>=1)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (>=1)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + [[package]] name = "starlette" version = "1.2.0" @@ -5769,4 +6053,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "880201a6ac5b2bf802f6f343ea03ffe1c2281647874a70524aa4bb92d59ee96f" +content-hash = "f6d2b2afe4ccc3967e53e4bbbd0fbeb271d3bbacc8c715d70267fac8d85955f2" diff --git a/security_scanning/examples/serve/pyproject.toml b/security_scanning/examples/serve/pyproject.toml index 3355cee5040e..b69a799dca67 100644 --- a/security_scanning/examples/serve/pyproject.toml +++ b/security_scanning/examples/serve/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] requires-python = ">=3.10,<3.13" dependencies = [ - "aiperf (>=0.8.0,<0.9.0)" + "aiperf (>=0.9.0,<0.10.0)" ] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index e259f4424228..1ee122f0bc39 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -465,31 +465,31 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 0e9006bba4c3..0d258b14a124 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "fed47f1fb10667cb7b90f167574e79a8574059eb", - "timestamp": "2026-05-29T02:46:33Z" + "commit_hash": "74d7c3acbb0cf1670dc3284970a5bf3a5b468187", + "timestamp": "2026-05-30T02:47:10Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index 85b3bb692288..c3081a6ca191 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -1142,30 +1142,30 @@ ssh = ["bcrypt (>=3.1.5)"] [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] @@ -1224,22 +1224,22 @@ files = [ [[package]] name = "cuda-python" -version = "13.3.0" +version = "13.3.1" description = "CUDA Python: Performance meets Productivity" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_python-13.3.0-py3-none-any.whl", hash = "sha256:d21872920204eda83344f8e55644a57823c57c5553d23f9e334add7b8aac0cd0"}, + {file = "cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3"}, ] [package.dependencies] -cuda-bindings = ">=13.3.0,<13.4.0" +cuda-bindings = ">=13.3.1,<13.4.0" cuda-core = ">=1.0.0,<1.1.0" cuda-pathfinder = ">=1.1,<2.0" [package.extras] -all = ["cuda-bindings[all] (>=13.3.0,<13.4.0)"] +all = ["cuda-bindings[all] (>=13.3.1,<13.4.0)"] [[package]] name = "cuda-tile" @@ -1529,29 +1529,6 @@ typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "fastapi" -version = "0.121.3" -description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fastapi-0.121.3-py3-none-any.whl", hash = "sha256:0c78fc87587fcd910ca1bbf5bc8ba37b80e119b388a7206b39f0ecc95ebf53e9"}, - {file = "fastapi-0.121.3.tar.gz", hash = "sha256:0055bc24fe53e56a40e9e0ad1ae2baa81622c406e548e501e717634e2dfbc40b"}, -] - -[package.dependencies] -annotated-doc = ">=0.0.2" -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" -starlette = ">=0.40.0,<0.51.0" -typing-extensions = ">=4.8.0" - -[package.extras] -all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] -standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] - [[package]] name = "filelock" version = "3.29.0" @@ -2915,14 +2892,14 @@ dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setup [[package]] name = "mcp" -version = "1.27.1" +version = "1.27.2" description = "Model Context Protocol SDK" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f"}, - {file = "mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924"}, + {file = "mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5"}, + {file = "mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef"}, ] [package.dependencies] @@ -4611,19 +4588,19 @@ twisted = ["twisted"] [[package]] name = "prometheus-fastapi-instrumentator" -version = "7.1.0" +version = "8.0.0" description = "Instrument your FastAPI app with Prometheus metrics" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, - {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, + {file = "prometheus_fastapi_instrumentator-8.0.0-py3-none-any.whl", hash = "sha256:e3c967c402124dfe81cf5aad35208d9f96db5452c6cb752350d9358ca0a96198"}, + {file = "prometheus_fastapi_instrumentator-8.0.0.tar.gz", hash = "sha256:c31ed192db077e75467b28b2fe5055362517f53369b798cb8dac9ff85f3f1c38"}, ] [package.dependencies] prometheus-client = ">=0.8.0,<1.0.0" -starlette = ">=0.30.0,<1.0.0" +starlette = ">=1.0.0,<2.0.0" [[package]] name = "propcache" @@ -6370,14 +6347,14 @@ uvicorn = ["uvicorn (>=0.34.0)"] [[package]] name = "starlette" -version = "0.50.0" +version = "1.2.0" description = "The little ASGI library that shines." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-0.50.0-py3-none-any.whl", hash = "sha256:9e5391843ec9b6e472eed1365a78c8098cfceb7a74bfd4d6b1c0c0095efb3bca"}, - {file = "starlette-0.50.0.tar.gz", hash = "sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca"}, + {file = "starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7"}, + {file = "starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553"}, ] [package.dependencies] @@ -7080,25 +7057,25 @@ files = [ [[package]] name = "xdsl" -version = "0.64.0" +version = "0.65.0" description = "xDSL" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "xdsl-0.64.0-py3-none-any.whl", hash = "sha256:c8ebdcccdca4bb526830d3eedfdb8a202cce4c912e8019bd2ca29a58ff67056f"}, - {file = "xdsl-0.64.0.tar.gz", hash = "sha256:362d2aae14fd7b04848500128ee023c035aef3681f5704f111817de2bd86c1fd"}, + {file = "xdsl-0.65.0-py3-none-any.whl", hash = "sha256:1dd2686e3dd3c1d3d24af6dbd247ec6245b08e0716cfa9ac9a47549a3674556b"}, + {file = "xdsl-0.65.0.tar.gz", hash = "sha256:ac747c51e3c95f60da026cacb34a4eef6ec76343f5cfd0a46bcae7e0ec4ed916"}, ] [package.dependencies] immutabledict = "<4.3.2" -ordered-set = "4.1.0" +ordered-set = "4.1" typing-extensions = ">=4.7,<5" [package.extras] -dev = ["coverage (<8.0.0)", "filecheck (==1.0.3)", "ipykernel", "lit (<19.0.0)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8.0.0)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.409)", "pytest (<9.1)", "pytest-asyncio (==1.3.0)", "pytest-cov", "ruff (==0.15.12)", "sympy (==1.14.0)", "textual-dev (==1.8.0)", "toml (<0.11)"] +dev = ["coverage (<8)", "filecheck (==1.0.3)", "ipykernel", "lit (<19)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.409)", "pytest (<9.1)", "pytest-asyncio", "pytest-cov", "ruff (==0.15.15)", "sympy (==1.14)", "textual-dev (==1.8)", "toml (<0.11)"] gui = ["pyclip (==0.7)", "textual (>=8,<9)"] -heir = ["heir_py (==2026.5.1) ; python_version >= \"3.11\" and python_version < \"3.13\""] +heir = ["heir-py (==2026.5.18) ; python_version >= \"3.11\" and python_version < \"3.13\""] llvm = ["llvmlite (>=0.47.0,<0.48.0)"] [[package]] @@ -7497,4 +7474,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "25e38281b813618127f505b30af1ad9aa7f631945d858797c8870e5b81db432c" +content-hash = "0eb5c9722f31b299584cd13daae11eadb5c6a5eee4090d760ee53aada0623eab" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index afceed66e18b..f0c4c5c2dad3 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "nvidia-modelopt[torch] (>=0.37.0,<0.38.0)", "transformers (==5.5.4)", "prometheus-client (>=0.25.0,<0.26.0)", - "prometheus-fastapi-instrumentator (>=7.1.0,<8.0.0)", + "prometheus-fastapi-instrumentator (>=8.0.0,<9.0.0)", "pydantic (>=2.9.1)", "pydantic-settings[yaml] (>=2.14.1,<3.0.0)", "omegaconf (>=2.3.0,<3.0.0)", @@ -46,7 +46,6 @@ dependencies = [ "click-option-group (>=0.5.9,<0.6.0)", "aenum (>=3.1.17,<4.0.0)", "pyzmq (>=27.1.0,<28.0.0)", - "fastapi (>=0.120.1,<=0.121.3)", "starlette (>=0.49.1)", "uvicorn (>=0.48.0,<0.49.0)", "setuptools (<80)", @@ -74,7 +73,7 @@ dependencies = [ "plotly (>=6.7.0,<7.0.0)", "numexpr (>=2.14.1,<3.0.0)", "partial-json-parser (>=0.2.1.1.post7,<0.3.0.0)", - "mcp (>=1.27.1,<2.0.0)", + "mcp (>=1.27.2,<2.0.0)", "torch-c-dlpack-ext (==0.1.3)", "flash-attn-4 (==4.0.0b11)", "mistral-common (>=1.10.0)", diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 3cf7a8766832..9ffe8afebe53 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -491,30 +491,30 @@ files = [ [[package]] name = "cuda-bindings" -version = "13.3.0" +version = "13.3.1" description = "Python bindings for CUDA" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:326b31d88314068f0ead48141222d7ada094f4013723dd924af9db10150fa472"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:76edf9a22ad52c306d33204d16071f15203217640ffc4083f854425062a30a02"}, - {file = "cuda_bindings-13.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a26690cae5ff9a9cf46ab4619d097dd5c1000c7de9680652705a9a5d46b2409e"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:244a167d81a9d07d3209d02c8e02e848c2b7c38f4d01e8e4d1f9620b173ae006"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:820ab45be3f39a088f39c4a04eb8b852d0b339bff8c518da5c258882b8a4e21b"}, - {file = "cuda_bindings-13.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:2ef7180ea85405bf6d70e4b3924d293333faf6d5aa57b36e7ddf8aaae29a8a78"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f72f67f5790e7fa51e3f893e007e49567573ccdf4ed1ca988fbbbd36cd77847c"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a99c3b8d584f266c616bd0f30c7cd83e33553e3ef2abad41ff5a74fbc033a69a"}, - {file = "cuda_bindings-13.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:851e138181b3a6edf6f29ba30ce7481df9a0dbfffc655b603dd1bb31032707ca"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7698fcc4577aa96372866f4d0c9a6cf686cd5c90eab94581c29d37fe6600542"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abd908f651160d12c45c5714a38ee102a1173a55433c0d1509ec0e8293beb4a6"}, - {file = "cuda_bindings-13.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:97afb1ebe2f60b538d88a902a952749e97f999505cd8aca78d491ea7e8b038c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a801fa30e75d25b74252123aefc746b6c4275624d2b8640632dd1dfeeaa1f88"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94d40ef7b4bdd9dce0244a1baa132e0e538f1eb2c0d162fb3648a15e48515365"}, - {file = "cuda_bindings-13.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:e17a46d711233a8a8e4c33aaa773e13810abe5fe686e08ca3e3df52449e6d7eb"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5911bea15810b749a8077f8c45423ed785d51618b8e8664dea1fc8f5a2a76c8"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2870fed7707a37f8af0c02364b05f355ebe8921604e8c68eb56cf66867e0798"}, - {file = "cuda_bindings-13.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2ddb3b5c9b013f42b1304ba39c0fe4277f4c228987e62d850aff66353aeb0ee2"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, + {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, + {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, + {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, + {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, + {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, + {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, ] [package.dependencies] @@ -573,22 +573,22 @@ files = [ [[package]] name = "cuda-python" -version = "13.3.0" +version = "13.3.1" description = "CUDA Python: Performance meets Productivity" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_python-13.3.0-py3-none-any.whl", hash = "sha256:d21872920204eda83344f8e55644a57823c57c5553d23f9e334add7b8aac0cd0"}, + {file = "cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3"}, ] [package.dependencies] -cuda-bindings = ">=13.3.0,<13.4.0" +cuda-bindings = ">=13.3.1,<13.4.0" cuda-core = ">=1.0.0,<1.1.0" cuda-pathfinder = ">=1.1,<2.0" [package.extras] -all = ["cuda-bindings[all] (>=13.3.0,<13.4.0)"] +all = ["cuda-bindings[all] (>=13.3.1,<13.4.0)"] [[package]] name = "exceptiongroup" From 1f02e96aa46b1cd2552dbdaf91ffb1cc4d70a46d Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Fri, 29 May 2026 20:35:29 -0700 Subject: [PATCH 137/308] [TRTLLM-12535][chore] Refactor fast path (token ID space) preprocessing logic out to the input preprocessor methods only (#14370) Signed-off-by: Michal Guzek --- .../trtllm-model-onboard-multimodal/SKILL.md | 24 +- .../_torch/models/modeling_gemma3vl.py | 2 +- .../_torch/models/modeling_gemma4mm.py | 2 +- .../_torch/models/modeling_hyperclovax.py | 2 +- .../_torch/models/modeling_kimi_k25.py | 2 +- tensorrt_llm/_torch/models/modeling_llama.py | 4 +- .../_torch/models/modeling_llava_next.py | 7 +- .../_torch/models/modeling_mistral.py | 2 +- .../_torch/models/modeling_nemotron_nano.py | 6 +- tensorrt_llm/_torch/models/modeling_phi4mm.py | 2 +- .../_torch/models/modeling_qwen2vl.py | 2 +- .../_torch/models/modeling_qwen3vl.py | 2 +- tensorrt_llm/_torch/models/modeling_vila.py | 2 +- tensorrt_llm/inputs/registry.py | 340 +++++++++++------- tensorrt_llm/llmapi/llm.py | 36 +- .../multimodal/test_multimodal_runtime.py | 2 +- 16 files changed, 246 insertions(+), 191 deletions(-) diff --git a/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md b/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md index dcdbaf042a9c..6cbf73dad66a 100644 --- a/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md +++ b/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md @@ -31,7 +31,11 @@ metadata: asyncio.gather then decodes all media for one request in parallel. [2] Input pipeline (asyncio.to_thread, off the event loop) - BaseMultimodalInputProcessor.__call__: + BaseMultimodalInputProcessor.__call__ dispatches by input shape: a text + prompt goes to the per-model call_with_text_prompt; prompt_token_ids + + mm_data goes to the base-class call_with_token_ids fast path (or is + detokenized back to call_with_text_prompt when the model opts out). The + per-model HF processing lives in call_with_text_prompt: HF AutoProcessor → pixel_values + token_ids mm-token layout (positions / lengths / special_token_offsets) (mRoPE) mrope_position_ids + deltas computed on CPU @@ -79,7 +83,7 @@ metadata: When `@support_multimodal_disaggregated` is set and the deployment uses `TLLM_MULTIMODAL_DISAGGREGATED=1`: - **Encoder worker:** runs as a standalone `MultimodalEncoder` (`mm_encoder_only=True`). It executes only the multimodal encoder and ships `mm_embeddings` (+ mRoPE position ids/deltas) to prefill+decode workers as shared-tensor handles. -- **Prefill+decode worker:** the model's `__init__` skips constructing `self.mm_encoder` when `_is_disagg()` is true; the input processor's `attach_multimodal_embeddings()` override binds the encoder handles into the request. For context-only requests, the engine re-clones mrope tensors so IPC handles outlive the encoder worker's freed memory — replicate that pattern for any new GPU-resident mm tensors. +- **Prefill+decode worker:** the model's `__init__` skips constructing `self.mm_encoder` when `_is_disagg()` is true; the input processor's `_attach_multimodal_embeddings_impl()` override binds the encoder handles into the request (the base `attach_multimodal_embeddings` wrapper detokenizes tokenized inputs for non-fast-path VLMs, then delegates to your impl). For context-only requests, the engine re-clones mrope tensors so IPC handles outlive the encoder worker's freed memory — replicate that pattern for any new GPU-resident mm tensors. ### Templates to study @@ -159,11 +163,11 @@ Three-arg **`torch.where(cond, x, y)`** is fine when **`cond`** is built only on CPU-bound work (decode / resize / normalize / mel-spectrogram / frame extraction) must not compete with GPU work, block the request loop, or serialize across requests. -- HF AutoProcessor + image_processor + tokenizer run inside `BaseMultimodalInputProcessor.__call__` — *not* in the model worker. +- HF AutoProcessor + image_processor + tokenizer run inside the input processor's `call_with_text_prompt` (dispatched from `__call__`) — *not* in the model worker. - URL/bytes media goes through `async_load_image` / `async_load_video` / `async_load_audio` (all wrap blocking decode in `asyncio.to_thread`). Never call `PIL.Image.open(...).load()` / `cv2.VideoCapture` / `soundfile.read` synchronously on the request hot path. - Pin host tensors before H2D with `prefer_pinned()` (False under Confidential Compute (CC), True otherwise). The engine pins `multimodal_data` automatically via `to_device(..., pin_memory=prefer_pinned())`. - **Declare `multimodal_data_device_paths`** on the model — list of dotted paths (e.g. `["image.pixel_values", "image.image_grid_thw", "video.pixel_values_videos", "video.video_grid_thw", "multimodal_embedding"]`) telling the engine which fields go to CUDA. Anything not listed stays on CPU. -- Optional (refactor pending): `get_text_with_mm_placeholders` + `expand_prompt_token_ids_for_mm` enable the tokenized+MM fast path (`tokenized_multimodal_process`), skipping redundant detokenization. A cleaner alternative is being designed — skip unless you have a specific need. +- Optional tokenized+MM fast path: set `supports_token_id_mm_expansion = True` (a `ClassVar`, default `False`) and implement `get_text_with_mm_placeholders` + `expand_prompt_token_ids_for_mm`. The base-class `__call__` then routes `prompt_token_ids + multi_modal_data` (no `prompt`) requests through `call_with_token_ids`, skipping redundant detokenization. When the flag is `False` (most VLMs), the base class detokenizes `prompt_token_ids → prompt` and re-runs `call_with_text_prompt`, so token-ID inputs still work — just less efficiently. Only LlavaNext + NanoV2VL opt in today. - Forward `mm_processor_kwargs` from `inputs.get("mm_processor_kwargs", {})` to the HF processor (callers tune things like video sample rate via this). ### Contract 3 — Large media via shared tensors, never raw pickle @@ -302,7 +306,7 @@ class {Name}Model(PreTrainedModel): Subclass **both** `BaseMultimodalInputProcessor` (drives every real request) and `BaseMultimodalDummyInputsBuilder` (drives engine warmup / profiling — the base shrinks dummy image resolution until the synthetic prompt fits `input_seq_len`). Colocate in the modeling file. Reference: `Qwen3VLInputProcessorBase`. -`__call__(inputs, sampling_params)` does: +Implement `call_with_text_prompt(inputs, sampling_params)` — the per-model text-prompt path. **Don't override `__call__`**: the base class's concrete `__call__` dispatches here for text prompts, and also detokenizes `prompt_token_ids → prompt` and falls through to here for non-fast-path VLMs. `call_with_text_prompt` does: 1. Pull `text_prompt`, `mm_data`, `mm_processor_kwargs` from `inputs`. 2. `_preprocess(...)` — HF processor produces `pixel_values` / `pixel_values_videos` / `*_grid_thw` / `input_ids`. @@ -311,9 +315,9 @@ Subclass **both** `BaseMultimodalInputProcessor` (drives every real request) and 5. `_postprocess(input_ids)` rewrites HF's `image_token_id` / `video_token_id` to `tllm_multimodal_token_id = vocab_size + 1` (the OOV sentinel). Skip when `mm_data` is empty. 6. Return `(prompt_token_ids_list, {"multimodal_data": multimodal_data})`. -**Optional overrides (refactor pending; skip unless needed):** `get_text_with_mm_placeholders(mm_counts)` + `expand_prompt_token_ids_for_mm(prompt_token_ids, num_mm_tokens, ...)` enable the tokenized fast path. A cleaner replacement is being designed. +**Optional tokenized+MM fast path (skip unless needed):** set `supports_token_id_mm_expansion = True` (`ClassVar`) and implement `get_text_with_mm_placeholders(mm_counts)` + `expand_prompt_token_ids_for_mm(prompt_token_ids, num_mm_tokens, ...)`. The base-class `call_with_token_ids` then builds dummy placeholder text, runs `call_with_text_prompt` on it, expands the real token IDs, and merges any returned `mm_data_updates` (e.g. video `evs_ids`) into `multimodal_data`. Leave the flag `False` and the base class just detokenizes token-ID inputs and re-runs `call_with_text_prompt`. Only LlavaNext + NanoV2VL opt in today. -**EPD override (if `@support_multimodal_disaggregated`):** `attach_multimodal_embeddings(inputs, multimodal_embedding, sampling_params)` consumes encoder outputs in the prefill+decode worker. +**EPD override (if `@support_multimodal_disaggregated`):** override `_attach_multimodal_embeddings_impl(inputs, multimodal_embedding, sampling_params)` — **not** the `attach_multimodal_embeddings` wrapper — to consume encoder outputs in the prefill+decode worker. The base wrapper detokenizes tokenized inputs for non-fast-path VLMs before delegating to your impl. **Decorator stack** — bottom-up application; `register_vision_encoder` requires `register_auto_model` to have run first: @@ -429,9 +433,9 @@ Follow `CONTRIBUTING.md`. Title `[JIRA/NVBUG/None][type] description`, `git comm **Input processor** - [ ] Subclasses both `BaseMultimodalInputProcessor` and `BaseMultimodalDummyInputsBuilder`. -- [ ] `__call__` runs HF AutoProcessor + tokenizer, builds `multimodal_data` by modality, computes `mrope_config` on CPU, `_postprocess`-rewrites mm token ids to the OOV sentinel. -- [ ] `mm_processor_kwargs` flow-through preserved. (Tokenized fast-path overrides — `get_text_with_mm_placeholders` / `expand_prompt_token_ids_for_mm` — are optional; a cleaner replacement is being designed.) -- [ ] `attach_multimodal_embeddings` implemented if `@support_multimodal_disaggregated`. +- [ ] `call_with_text_prompt` (not `__call__` — that's the base-class dispatcher) runs HF AutoProcessor + tokenizer, builds `multimodal_data` by modality, computes `mrope_config` on CPU, `_postprocess`-rewrites mm token ids to the OOV sentinel. +- [ ] `mm_processor_kwargs` flow-through preserved. (Tokenized fast path is optional: set `supports_token_id_mm_expansion = True` + implement `get_text_with_mm_placeholders` / `expand_prompt_token_ids_for_mm`; otherwise the base class detokenizes token-ID inputs automatically.) +- [ ] `_attach_multimodal_embeddings_impl` implemented (not the `attach_multimodal_embeddings` wrapper) if `@support_multimodal_disaggregated`. **Performance contracts** - [ ] Grep clean for Contract 1 bans (`.item()` / `.cpu()` / `.tolist()` / `torch.nonzero` / single-arg `torch.where` / value-dependent `if`, etc.) in modeling `forward` paths — elementwise `torch.where(cond, x, y)` with GPU-only `cond` is fine. diff --git a/tensorrt_llm/_torch/models/modeling_gemma3vl.py b/tensorrt_llm/_torch/models/modeling_gemma3vl.py index a99d62a48b23..5f327c82c026 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma3vl.py +++ b/tensorrt_llm/_torch/models/modeling_gemma3vl.py @@ -102,7 +102,7 @@ def _preprocess(self, inputs): return input_ids, pixel_values @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: input_ids, pixel_values = self._preprocess(inputs) diff --git a/tensorrt_llm/_torch/models/modeling_gemma4mm.py b/tensorrt_llm/_torch/models/modeling_gemma4mm.py index 9bfbac089ac4..66ef1b03cfcc 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4mm.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4mm.py @@ -484,7 +484,7 @@ def _preprocess(self, inputs): ) @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams, diff --git a/tensorrt_llm/_torch/models/modeling_hyperclovax.py b/tensorrt_llm/_torch/models/modeling_hyperclovax.py index 0825212300bc..0fdfdb054623 100644 --- a/tensorrt_llm/_torch/models/modeling_hyperclovax.py +++ b/tensorrt_llm/_torch/models/modeling_hyperclovax.py @@ -756,7 +756,7 @@ def _preprocess(self, text_prompt: dict[str, any], images: List[Any], return input_ids, preprocessed_image @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: text_prompt, mm_data, mm_processor_kwargs = inputs.get("prompt"), \ diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index 314ba47fdc46..5ed83c6ab4b1 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -1205,7 +1205,7 @@ def _ensure_k25_slow_tokenizer(self) -> None: self._processor.tokenizer = slow_tok @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams, diff --git a/tensorrt_llm/_torch/models/modeling_llama.py b/tensorrt_llm/_torch/models/modeling_llama.py index 0bfbf65e1700..d77c22322de9 100644 --- a/tensorrt_llm/_torch/models/modeling_llama.py +++ b/tensorrt_llm/_torch/models/modeling_llama.py @@ -1305,7 +1305,7 @@ def processor(self) -> AutoProcessor: def dtype(self) -> torch.dtype: return self._dtype - def attach_multimodal_embeddings( + def _attach_multimodal_embeddings_impl( self, inputs: TextPrompt, multimodal_embedding: Dict[str, List[Dict[str, Any]]], @@ -1434,7 +1434,7 @@ def attach_multimodal_embeddings( return token_ids.tolist(), {"multimodal_data": multimodal_data} @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: text_prompt, mm_data = inputs.get("prompt"), inputs.get( diff --git a/tensorrt_llm/_torch/models/modeling_llava_next.py b/tensorrt_llm/_torch/models/modeling_llava_next.py index fd737c0386f3..910bc3c82a18 100644 --- a/tensorrt_llm/_torch/models/modeling_llava_next.py +++ b/tensorrt_llm/_torch/models/modeling_llava_next.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Union import numpy as np import torch @@ -37,6 +37,7 @@ class LlavaNextInputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): + supports_token_id_mm_expansion: ClassVar[bool] = True def __init__(self, model_path: str, @@ -328,7 +329,7 @@ def get_prompt_token_ids( return expanded_ids, mm_token_length, mm_token_offsets - def attach_multimodal_embeddings( + def _attach_multimodal_embeddings_impl( self, inputs: TextPrompt, multimodal_embedding: Dict[str, List[torch.Tensor]], sampling_params: SamplingParams @@ -375,7 +376,7 @@ def attach_multimodal_embeddings( } @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: text_prompt, mm_data = inputs.get("prompt"), inputs.get( diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 590d0a860f9a..7de6f2523d5f 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -414,7 +414,7 @@ def dtype(self) -> torch.dtype: return self._dtype @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], ExtraProcessedInputs | None]: images = inputs.get("multi_modal_data", {}).get("image") diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_nano.py b/tensorrt_llm/_torch/models/modeling_nemotron_nano.py index 6d03b2204d2e..c81407e77809 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_nano.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_nano.py @@ -4,7 +4,7 @@ import os import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Union import numpy as np import torch @@ -879,6 +879,8 @@ def _video_tubelet_geometry(self, t: int, T: int, ih: int, iw: int) -> Tuple[int class NanoV2VLInputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): + supports_token_id_mm_expansion: ClassVar[bool] = True + def __init__( self, model_path: str, @@ -2128,7 +2130,7 @@ def _compute_token_numbers_per_video(self, video_size_lst: List[Tuple]) -> List[ return num_tokens_per_frame_lst @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: text_prompt, mm_data = inputs.get("prompt"), inputs.get("multi_modal_data", {}) diff --git a/tensorrt_llm/_torch/models/modeling_phi4mm.py b/tensorrt_llm/_torch/models/modeling_phi4mm.py index b248c8ffdb6d..abcc0c1f5ff2 100644 --- a/tensorrt_llm/_torch/models/modeling_phi4mm.py +++ b/tensorrt_llm/_torch/models/modeling_phi4mm.py @@ -898,7 +898,7 @@ def _post_process(self, image_inputs: BatchFeature, return inputs @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: text_prompt, mm_data = inputs.get("prompt"), inputs.get( diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index 08523ffcecc5..7db60f085dfd 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -353,7 +353,7 @@ def get_mrope_config( @nvtx_range("Qwen2VLInputProcessorBase forward()") @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams, diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index 4e8ec6e2db37..8aba28bdb932 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -351,7 +351,7 @@ def get_mrope_config( @nvtx_range("Qwen3VLInputProcessorBase forward()") @torch.inference_mode() - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams, diff --git a/tensorrt_llm/_torch/models/modeling_vila.py b/tensorrt_llm/_torch/models/modeling_vila.py index 0efbc3e469a0..12c3eeefc652 100644 --- a/tensorrt_llm/_torch/models/modeling_vila.py +++ b/tensorrt_llm/_torch/models/modeling_vila.py @@ -1103,7 +1103,7 @@ def _postprocess(self, input_ids, mm_features): return fused_input_ids, mm_features @torch.inference_mode() # critical to minimize memory footprint - def __call__( + def call_with_text_prompt( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: """ diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 95ed84d4a96a..801e27b977eb 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -131,9 +131,10 @@ class BaseMultimodalInputProcessor(ABC): models. Specific processors can override these methods if they need custom logic. Optional tokenized+MM fast path: to support prompt_token_ids + multi_modal_data - without detokenizing, implement get_text_with_mm_placeholders(mm_counts) and + without detokenizing, set the `supports_token_id_mm_expansion` class flag to + True and implement both get_text_with_mm_placeholders(mm_counts) and expand_prompt_token_ids_for_mm(prompt_token_ids, num_mm_tokens_per_placeholder, ...). - If these are not implemented, the pipeline detokenizes the text prompt first and then + If the flag is False, the pipeline detokenizes the text prompt first and then processes the multimodal inputs. """ @@ -145,6 +146,12 @@ class BaseMultimodalInputProcessor(ABC): # tokens and tolerate splitting. Gemma4 sets True. mm_bidirectional_blocks: ClassVar[bool] = False + # Whether the subclass implements the tokenized+MM fast path — i.e. both + # `get_text_with_mm_placeholders` and `expand_prompt_token_ids_for_mm`. + # When True, `__call__` may route `prompt_token_ids + multi_modal_data` + # inputs to `call_with_token_ids` instead of detokenizing upstream. + supports_token_id_mm_expansion: ClassVar[bool] = False + def __init__(self, model_path, config, @@ -165,15 +172,65 @@ def attach_multimodal_embeddings( multimodal_embedding: Dict[str, List[torch.Tensor]], sampling_params: SamplingParams, ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + """Handle externally provided multimodal input embeddings. + + While inputs["multi_modal_data"] is handled by __call__, this method is + intended to process inputs["multi_modal_embeddings"]. + + Auto-detokenizes `prompt_token_ids` --> `prompt` for non-fast-path VLMs + before delegating to `_attach_multimodal_embeddings_impl`. Subclasses + override `_attach_multimodal_embeddings_impl` (not this wrapper). """ - Handle externally provided multimodal input embeddings. + if (inputs.get("prompt_token_ids") is not None + and inputs.get("prompt") is None + and not self.supports_token_id_mm_expansion): + inputs = self._detokenize_to_text_prompt(inputs, sampling_params) + return self._attach_multimodal_embeddings_impl(inputs, + multimodal_embedding, + sampling_params) + + def _attach_multimodal_embeddings_impl( + self, + inputs: TextPrompt, + multimodal_embedding: Dict[str, List[torch.Tensor]], + sampling_params: SamplingParams, + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + """Subclass hook for `attach_multimodal_embeddings`. - While inputs["multi_modal_data"] is handled by __call__, this method is intended to process - inputs["multi_modal_embeddings"]. + Subclasses that support `multi_modal_embeddings` override this method + rather than the wrapper `attach_multimodal_embeddings` itself. + Default raises NotImplementedError. """ raise NotImplementedError( "Input processor does not support multimodal embedding input") + def _detokenize_to_text_prompt( + self, + inputs: TextPrompt, + sampling_params: SamplingParams, + ) -> TextPrompt: + """Decode `prompt_token_ids` to text and rebuild `inputs` as a TextPrompt. + + Used by `__call__` and `attach_multimodal_embeddings` to normalize + tokenized inputs for non-fast-path VLMs before delegating to the + subclass-specific text-prompt path. + + Also flips `sampling_params.add_special_tokens` to False so the + subclass's re-tokenization doesn't double-add BOS/EOS on top of the + ones already encoded in `prompt_token_ids`. + """ + prompt = self.tokenizer.decode(inputs["prompt_token_ids"]) + if sampling_params is not None and sampling_params.add_special_tokens: + logger.debug( + "Setting add_special_tokens to False because prompt_token_ids " + "were provided to generate. VLMs will re-encode the prompt.") + sampling_params.add_special_tokens = False + return TextPrompt( + prompt=prompt, + multi_modal_data=inputs.get("multi_modal_data"), + mm_processor_kwargs=inputs.get("mm_processor_kwargs") or {}, + ) + @property @abstractmethod def processor(self) -> AutoProcessor: @@ -198,12 +255,124 @@ def dtype(self) -> torch.dtype: """The dtype for this model.""" ... - @abstractmethod def __call__( self, inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + """Dispatch between text-prompt and tokenized+MM fast paths. + + - Tokenized+MM fast path (`prompt_token_ids` set, `multi_modal_data` set, + no `prompt`): if the subclass opts in via + `supports_token_id_mm_expansion`, delegate to `call_with_token_ids`; + otherwise detokenize first and fall through to `call_with_text_prompt`. + - Else (text prompt with/without MM data, or tokenize-only requests): + delegate to `call_with_text_prompt`. + """ + if (inputs.get("prompt_token_ids") is not None + and inputs.get("multi_modal_data") is not None + and inputs.get("prompt") is None): + if self.supports_token_id_mm_expansion: + return self.call_with_token_ids(inputs, sampling_params) + # Fast path not supported by this VLM — detokenize and fall through + # to the text-prompt path. + inputs = self._detokenize_to_text_prompt(inputs, sampling_params) + return self.call_with_text_prompt(inputs, sampling_params) + + @abstractmethod + def call_with_text_prompt( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + """Process `inputs` containing a text `prompt` (and optionally `multi_modal_data`). + + Subclasses implement this with the HF processor pipeline that turns + `(text_prompt, mm_data)` into `(token_ids, extra_processed_inputs)`. + """ ... + def call_with_token_ids( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + """Fast path for `prompt_token_ids + multi_modal_data` without detokenizing. + + Builds dummy placeholder text, runs `call_with_text_prompt` on it to populate the + multimodal payload, then expands the real token IDs via + `expand_prompt_token_ids_for_mm`. + + Requires the subclass to opt in via the `supports_token_id_mm_expansion` + class flag and implement both of the following hooks; otherwise raises + `NotImplementedError`: + + - `get_text_with_mm_placeholders(mm_counts: Dict[str, int]) -> str` + Build a minimal placeholder text for the given per-modality item + counts. Used to drive the HF processor / vision encoder without the + real prompt. + - `expand_prompt_token_ids_for_mm( + prompt_token_ids: List[int], + num_mm_tokens_per_placeholder: List[int], + *, + hf_processor_mm_kwargs: Optional[Dict[str, Any]] = None, + mm_data: Optional[Dict[str, Any]] = None, + ) -> Tuple[List[int], Optional[Dict[str, Dict[str, Any]]]]` + Replace each MM placeholder token in `prompt_token_ids` with the + corresponding number of MM feature tokens. Returns the expanded + token IDs and optional `mm_data_updates` — per-modality fields to + merge into `extra_processed_inputs["multimodal_data"]` (e.g. video + `evs_ids` for Nano OMNI V3 rebuilt from the real prompt instead + of the dummy one). + """ + if not self.supports_token_id_mm_expansion: + raise NotImplementedError( + f"{type(self).__name__} does not implement the tokenized+MM " + "fast path (supports_token_id_mm_expansion is False). " + "Detokenize `prompt_token_ids` to a text prompt before calling." + ) + + prompt_token_ids = inputs["prompt_token_ids"] + mm_data = inputs["multi_modal_data"] + mm_counts = _mm_data_to_counts(mm_data) + + # Run the HF processor / vision encoder on a synthetic placeholder + # prompt to populate `extra_processed_inputs["multimodal_data"]`. Some + # auxiliary streams (e.g. video evs_ids) computed against this dummy + # text are stale w.r.t. the real prompt and get overwritten below. + extra_processed_inputs = _process_multimodal_with_dummy_placeholders( + self, + mm_data, + mm_counts, + inputs.get("mm_processor_kwargs"), + sampling_params, + ) + + num_mm_tokens = _get_single_mm_token_lengths( + mm_data, + self, + multimodal_data=(extra_processed_inputs + or {}).get("multimodal_data"), + ) + if num_mm_tokens is None: + raise ValueError( + "call_with_token_ids: find_mm_token_lengths returned no token " + "lengths for the provided multi_modal_data.") + + expanded_ids, mm_data_updates = self.expand_prompt_token_ids_for_mm( + prompt_token_ids, + num_mm_tokens, + hf_processor_mm_kwargs=inputs.get("mm_processor_kwargs"), + mm_data=mm_data, + ) + + # Merge any model-specific auxiliary streams (e.g. video evs_ids for + # EVS-enabled runs) into extra_processed_inputs["multimodal_data"], + # overwriting the stale values produced from the dummy prompt above so + # downstream consumers (`merge_evs_mm_embeds` etc.) read the values + # rebuilt from the real prompt token IDs. + if mm_data_updates: + mm_container = extra_processed_inputs.setdefault( + "multimodal_data", {}) + for modality, field_updates in mm_data_updates.items(): + mm_container.setdefault(modality, {}).update(field_updates) + + return expanded_ids, extra_processed_inputs + @property def use_fast(self) -> bool: """ @@ -795,6 +964,7 @@ def _get_single_mm_token_lengths( multimodal_data: Optional[Dict[str, Any]] = None, ) -> Optional[List[int]]: """Get the single set of MM token lengths (first value from find_mm_token_lengths). Returns None if empty.""" + # TODO: Just like in multimodal_hashing_process, here we assume there is only one modality for now num_mm_tokens_by_key = find_mm_token_lengths( mm_data, input_processor, multimodal_data=multimodal_data) if not num_mm_tokens_by_key: @@ -884,90 +1054,16 @@ def create_input_processor_with_hash( A wrapped processor that modifies prompts before processing. """ - def tokenized_multimodal_process( - inputs: TextPrompt, sampling_params: SamplingParams - ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: - """ - Process prompt_token_ids and multi_modal_data without detokenizing. - - Runs the input processor with dummy text placeholders for multi-modal slots, - then replaces placeholder token IDs with the actual feature token IDs and - delegates to multimodal_hashing_process. - - Args: - inputs: TextPrompt with "prompt_token_ids" and "multi_modal_data" (and optional "mm_processor_kwargs"). - sampling_params: Sampling parameters for the input processor. - - Returns: - (prompt_token_ids, extra_processed_inputs) from multimodal_hashing_process. - ([], None) if multi-modal token lengths cannot be determined. - """ - prompt_token_ids = inputs["prompt_token_ids"] - mm_data = inputs["multi_modal_data"] - mm_counts = _mm_data_to_counts(mm_data) - extra_processed_inputs = _process_multimodal_with_dummy_placeholders( - input_processor, - mm_data, - mm_counts, - inputs.get("mm_processor_kwargs"), - sampling_params, - ) - num_mm_tokens = _get_single_mm_token_lengths( - mm_data, - input_processor, - multimodal_data=(extra_processed_inputs - or {}).get("multimodal_data"), - ) - if num_mm_tokens is None: - raise ValueError( - "tokenized_multimodal_process: find_mm_token_lengths returned " - "no token lengths for the provided multi_modal_data.") - - expanded_ids, mm_data_updates = input_processor.expand_prompt_token_ids_for_mm( - prompt_token_ids, - num_mm_tokens, - hf_processor_mm_kwargs=inputs.get("mm_processor_kwargs"), - mm_data=mm_data, - ) - - # Merge any model-specific auxiliary streams (e.g. video evs_ids for - # EVS-enabled runs) into extra_processed_inputs["multimodal_data"]. - # The dummy-placeholder pass above produced these against the synthetic - # placeholder text, so they're stale w.r.t. the real prompt; overwrite - # them with the values rebuilt from `prompt_token_ids` here so - # `merge_evs_mm_embeds` picks up the correct stream at LLM forward time. - if mm_data_updates: - mm_container = extra_processed_inputs.setdefault( - "multimodal_data", {}) - for modality, field_updates in mm_data_updates.items(): - mm_container.setdefault(modality, {}).update(field_updates) - - return multimodal_hashing_process( - inputs, - sampling_params, - precomputed_token_ids=expanded_ids, - precomputed_extra=extra_processed_inputs, - precomputed_num_mm_tokens=num_mm_tokens, - ) - def multimodal_hashing_process( - inputs: TextPrompt, - sampling_params: SamplingParams, - *, - precomputed_token_ids: Optional[List[int]] = None, - precomputed_extra: Optional[ExtraProcessedInputs] = None, - precomputed_num_mm_tokens: Optional[List[int]] = None, + inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: """ Process multimodal hashing for media tokens if possible. - precomputed_token_ids and precomputed_extra must be provided together or - both be None. When both are provided (tokenized+MM path), skips the - input_processor call and uses them; when both are None, calls input_processor. - - precomputed_num_mm_tokens is optional and independent: if provided, skips - the otherwise-duplicate find_mm_token_lengths call (the tokenized+MM path - already computes it upstream to expand the prompt). + Delegates the raw `(token_ids, extra_processed_inputs)` production to + the input processor's `__call__` (which itself dispatches between the + text-prompt and tokenized+MM fast paths), then computes hashes, + positions, and masks on top. Supports optional user-provided UUIDs via 'multi_modal_uuids' in inputs. When a UUID is provided for a multimodal item, it will be used as the @@ -981,32 +1077,29 @@ def multimodal_hashing_process( mm_hashes, mm_uuid_list = apply_mm_hashes(mm_data, mm_uuids, hash_lib) - if precomputed_token_ids is not None and precomputed_extra is not None: - prompt_token_ids = precomputed_token_ids - extra_processed_inputs = precomputed_extra - elif precomputed_token_ids is None and precomputed_extra is None: - prompt_token_ids, extra_processed_inputs = input_processor( - inputs, sampling_params) - else: - raise ValueError( - "precomputed_token_ids and precomputed_extra must be provided " - "together or both be None; got one without the other.") + prompt_token_ids, extra_processed_inputs = input_processor( + inputs, sampling_params) - if precomputed_num_mm_tokens is not None: - num_mm_tokens = precomputed_num_mm_tokens - else: - # TODO: here we assume there is only one modality for now - num_mm_tokens_by_key = find_mm_token_lengths( - mm_data, - input_processor, - multimodal_data=(extra_processed_inputs - or {}).get("multimodal_data"), - ) - if not num_mm_tokens_by_key: - return [], None - num_mm_tokens = next(iter(num_mm_tokens_by_key.values())) + # TODO: here we assume there is only one modality for now + num_mm_tokens_by_key = find_mm_token_lengths( + mm_data, + input_processor, + multimodal_data=(extra_processed_inputs + or {}).get("multimodal_data"), + ) + # Raise (rather than returning ([], None)) so process_prompt_maybe_hash + # treats this as a failed hashing attempt: it falls back to the basic + # input processor and caches multimodal_hashing_supported=False instead + # of marking hashing as supported and forwarding an empty prompt + # downstream. + if not num_mm_tokens_by_key: + raise ValueError( + "multimodal hashing could not determine multimodal token " + "lengths for the provided input.") + num_mm_tokens = next(iter(num_mm_tokens_by_key.values())) if len(num_mm_tokens) <= 0: - return [], None + raise ValueError("multimodal hashing produced an empty multimodal " + "token-length list.") vocab_size = input_processor.get_vocab_size() mm_ids = input_processor.get_mm_token_ids() @@ -1056,15 +1149,6 @@ def multimodal_hashing_process( item_run_cu_offsets, run_positions, run_lengths) return prompt_token_ids, extra_processed_inputs - def process_tokenized_prompt_maybe_hash( - inputs: TextPrompt, sampling_params: SamplingParams - ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: - try: - return tokenized_multimodal_process(inputs, sampling_params) - except Exception as e: - logger.warning(f"Tokenized+MM path failed: {e}") - raise - def process_prompt_maybe_hash( inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: @@ -1121,21 +1205,11 @@ def process_prompt_maybe_hash( def input_processor_wrapper( inputs: TextPrompt, sampling_params: SamplingParams ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: - # Tokenized prompt + multi_modal_data path: requires the optional hooks. - # If the processor lacks them, fall through to the regular prompt path. - has_tokenized_multimodal_prompt = ( - inputs.get("prompt_token_ids") is not None - and inputs.get("multi_modal_data") is not None - and inputs.get("prompt") is None - and hasattr(input_processor, "get_text_with_mm_placeholders") - and hasattr(input_processor, "expand_prompt_token_ids_for_mm")) - - if has_tokenized_multimodal_prompt: - prompt_token_ids, extra_processed_inputs = ( - process_tokenized_prompt_maybe_hash(inputs, sampling_params)) - else: - prompt_token_ids, extra_processed_inputs = ( - process_prompt_maybe_hash(inputs, sampling_params)) + # The input processor's __call__ dispatches between the text-prompt + # and tokenized+MM fast paths internally; this wrapper only adds + # hashing/masking/cumsum on top. + prompt_token_ids, extra_processed_inputs = process_prompt_maybe_hash( + inputs, sampling_params) maybe_compute_mm_embed_cumsum(prompt_token_ids, extra_processed_inputs, input_processor) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 2e6295bb5843..9c1481099449 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -17,10 +17,8 @@ from transformers import PreTrainedTokenizerBase from tensorrt_llm._utils import mpi_disabled -from tensorrt_llm.inputs.data import TextPrompt from tensorrt_llm.inputs.multimodal import MultimodalInput, MultimodalParams -from tensorrt_llm.inputs.registry import (BaseMultimodalInputProcessor, - DefaultInputProcessor) +from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor from tensorrt_llm.llmapi import tracing from tensorrt_llm.metrics.enums import MetricNames @@ -550,34 +548,10 @@ def _preprocess( """ inputs = prompt_inputs(inputs) - # A fast path for token IDs & MM data is available for a VLM if the input processor has the following methods. - # TODO: Once all the VLMs support the fast path, remove this flag and modify the remaining logic accordingly. - use_token_ids_for_mm_placeholders = ( - hasattr(self.input_processor, "get_text_with_mm_placeholders") - and hasattr(self.input_processor, "expand_prompt_token_ids_for_mm")) - - # This IF branch is applicable, whenever: - # - multimodal data is present (whether through embeddings or as preprocessed data), AND - # - token IDs are present, AND - # - two methods defining the placeholder token IDs expansion logic are not available. - if not inputs.get("prompt") and inputs.get("prompt_token_ids") and ( - inputs.get("multi_modal_data") - or inputs.get("multi_modal_embeddings")) and not isinstance( - self.input_processor, DefaultInputProcessor - ) and not use_token_ids_for_mm_placeholders: - # VLMs need to process/tokenize the prompt in their own way, - # if they don't have the fast path for token IDs & MM data implemented yet. - # TODO: Once all the VLMs support the fast path, we can remove this detokenization step entirely. - prompt = self.tokenizer.decode(inputs['prompt_token_ids']) - inputs = TextPrompt( - prompt=prompt, - multi_modal_data=inputs.get("multi_modal_data"), - mm_processor_kwargs=inputs.get("mm_processor_kwargs") or {}) - if sampling_params.add_special_tokens: - logger.debug( - "Setting add_special_tokens to False because prompt_token_ids were provided to generate. VLMs will re-encode the prompt." - ) - sampling_params.add_special_tokens = False + # Detokenization for non-fast-path VLMs (prompt_token_ids + MM payload, + # no prompt) is handled inside BaseMultimodalInputProcessor.__call__ + # and BaseMultimodalInputProcessor.attach_multimodal_embeddings, gated + # on the `supports_token_id_mm_expansion` class flag. is_mm_disagg = (disaggregated_params is not None and disaggregated_params.multimodal_embedding_handles diff --git a/tests/unittest/_torch/multimodal/test_multimodal_runtime.py b/tests/unittest/_torch/multimodal/test_multimodal_runtime.py index 3e2b9f1d248a..0f5ce64af4d8 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_runtime.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_runtime.py @@ -781,7 +781,7 @@ def config(self): def dtype(self): return torch.float32 - def __call__(self, inputs, sampling_params): + def call_with_text_prompt(self, inputs, sampling_params): raise NotImplementedError("This fake only supports token queries.") def get_vocab_size(self): From 15bb7915c5a5a1d157465ef05ca12aafd2e0b28b Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Sat, 30 May 2026 11:37:40 +0800 Subject: [PATCH 138/308] [None][infra] Waive 2 failed cases for main in post-merge 2741 (#14737) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index ac99a034439b..475983ef3a45 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -14,6 +14,7 @@ accuracy/test_llm_api.py::TestMistralNemo12B::test_fp8 SKIP (https://nvbugs/5413 accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True] SKIP (https://nvbugs/6240561) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) @@ -272,6 +273,7 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6189928) perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236108) +perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236094) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) From c27da7553c6475041a0343724e70e305087e4257 Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Sat, 30 May 2026 13:59:54 +0800 Subject: [PATCH 139/308] [None][infra] Waive 1 failed cases for main in pre-merge 40562 (#14776) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 475983ef3a45..015efc6e97e6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -276,6 +276,7 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_bl perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236094) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6227472) +perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6244474) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) From 4ec2942c7c200d2ee2bda013956395fef5fe74d5 Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Fri, 29 May 2026 23:03:48 -0700 Subject: [PATCH 140/308] [TRTLLM-12440][feat] Add GMS-only weight sharing support (#13926) Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- tensorrt_llm/_torch/memory/__init__.py | 18 + .../_torch/memory/gpu_memory_backend.py | 581 ++++++++++++++++++ .../checkpoints/mx/checkpoint_loader.py | 48 +- .../_torch/pyexecutor/model_engine.py | 89 ++- .../_torch/pyexecutor/model_loader.py | 296 ++++++++- tensorrt_llm/_torch/pyexecutor/py_executor.py | 10 + tensorrt_llm/llmapi/llm_args.py | 228 +++++-- .../_torch/memory/test_gms_backend.py | 411 +++++++++++++ .../mx/test_mx_checkpoint_loader.py | 39 +- .../pyexecutor/test_model_loader_gms.py | 382 ++++++++++++ .../api_stability/references/llm.yaml | 4 + tests/unittest/llmapi/test_gms_args.py | 147 +++++ 12 files changed, 2171 insertions(+), 82 deletions(-) create mode 100644 tensorrt_llm/_torch/memory/__init__.py create mode 100644 tensorrt_llm/_torch/memory/gpu_memory_backend.py create mode 100644 tests/unittest/_torch/memory/test_gms_backend.py create mode 100644 tests/unittest/_torch/pyexecutor/test_model_loader_gms.py create mode 100644 tests/unittest/llmapi/test_gms_args.py diff --git a/tensorrt_llm/_torch/memory/__init__.py b/tensorrt_llm/_torch/memory/__init__.py new file mode 100644 index 000000000000..9461d06de740 --- /dev/null +++ b/tensorrt_llm/_torch/memory/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .gpu_memory_backend import GMSBackend, GPUMemoryBackend + +__all__ = ["GPUMemoryBackend", "GMSBackend"] diff --git a/tensorrt_llm/_torch/memory/gpu_memory_backend.py b/tensorrt_llm/_torch/memory/gpu_memory_backend.py new file mode 100644 index 000000000000..7d0ab04808f1 --- /dev/null +++ b/tensorrt_llm/_torch/memory/gpu_memory_backend.py @@ -0,0 +1,581 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GPU Memory Backend protocol and GMS implementation. + +Defines a thin protocol that adapts the upstream GPU Memory Service +(``gpu_memory_service`` from ``ai-dynamo/dynamo``) into TRT-LLM's +``ModelLoader`` pipeline. The protocol intentionally exposes only the +operations TRT-LLM needs at specific points in the loading lifecycle — +all heavy lifting (CUDA VMM, FD passing, zero-copy tensor construction) +is delegated to the GMS library's stable public Python primitives. + +Design notes: +- We deliberately avoid the upstream ``setup_gms()`` monkey-patch entry + point. TRT-LLM owns the integration policy in ``ModelLoader``; this + backend is a thin adapter over the GMS library's primitive API. +- The protocol has a single point of contact per concern, so when the + upstream GMS Python API drifts, only the methods on ``GMSBackend`` + need to change — the call sites in ``model_loader.py`` are stable. + +Operating modes: +- **RW (Read-Write)**: First worker loads weights via the normal + checkpoint loader pipeline, with allocations captured by a GMS-managed + CUDA memory pool. After loading, weights are committed for read-only + access by other workers and the client transitions to RO mode in place. +- **RO (Read-Only)**: Subsequent workers zero-copy import already-committed + weights from the GMS pool. ``post_load_weights()`` must run BEFORE + materialization so that module aliases are set up correctly. +""" + +from contextlib import contextmanager +from typing import Iterator, Optional, Protocol, runtime_checkable + +import torch +from torch import nn + +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping + + +@runtime_checkable +class GPUMemoryBackend(Protocol): + """Thin abstraction over GPU memory services for API stability.""" + + def connect(self) -> bool: + """Establish a session with the backend. Returns True on success.""" + ... + + @property + def is_rw(self) -> Optional[bool]: + """Whether this backend was granted RW (vs RO). None pre-connect.""" + ... + + def has_committed_weights(self) -> bool: + """Whether committed weights are ready to be imported (RO-ready).""" + ... + + def mem_pool_scope( + self, + device: Optional[torch.device] = None, + ) -> Iterator[None]: + """Return a context manager scoping CUDA allocations to the backend pool.""" + ... + + def materialize_module(self, model: nn.Module) -> None: + """Zero-copy import committed weights into model params (RO path).""" + ... + + def finalize_write(self, model: nn.Module) -> int: + """Register, commit, and transition to RO. Returns bytes committed.""" + ... + + def move_untracked_params(self, model: nn.Module) -> None: + """Move stray params not in the backend pool into shared memory.""" + ... + + def cleanup(self) -> None: + """Release all resources and disconnect.""" + ... + + +# String mode -> upstream RequestedLockType. Resolved lazily inside connect() +# so importing this module does not require the GMS library to be installed. +_MODE_ALIASES = ("rw", "ro", "auto") + + +class GMSBackend: + """Concrete ``GPUMemoryBackend`` using ``gpu_memory_service`` (GMS). + + GMS is a multi-process GPU memory manager (out-of-process server, + in-process clients) that lets multiple inference instances share weight + bytes via CUDA VMM mappings and Unix-socket FD passing. + + This adapter calls the GMS library's stable per-call primitives + (``GMSClientMemoryManager`` + the helpers in + ``gpu_memory_service.client.torch.allocator`` and ``.module``) and + composes them into the methods TRT-LLM's ``ModelLoader`` invokes for + the ``LoadFormat.GMS`` branch. We intentionally do **not** use the + upstream ``setup_gms()`` monkey-patch — TRT-LLM owns the integration + policy and we keep the dependency surface narrow and explicit. + """ + + DEFAULT_TAG = "weights" + + def __init__( + self, + socket_path: Optional[str], + mapping: Mapping, + mode: str = "auto", + tag: str = DEFAULT_TAG, + ) -> None: + """Initialize the GMS backend. + + Args: + socket_path: Unix domain socket path for the per-GPU GMS daemon. + When ``None``, the default per-GPU UUID-keyed path from + ``gpu_memory_service.common.utils.get_socket_path`` is used + (resolved lazily inside :meth:`connect`). + mapping: TRT-LLM distributed ``Mapping`` for TP/PP rank info. + mode: Operating mode — ``"auto"`` (RW first, RO after committed), + ``"rw"`` (require RW), or ``"ro"`` (require RO). + tag: Logical tag identifying this weight set in the GMS server. + Default ``"weights"`` matches the GMS library convention. + """ + if mode not in _MODE_ALIASES: + raise ValueError(f"GMS mode must be one of {_MODE_ALIASES}, got {mode!r}") + + self._socket_path = socket_path + self._mapping = mapping + self._mode = mode + self._tag = tag + self._device_index = torch.cuda.current_device() + + # Late-bound state, populated by connect() + self._client = None # GMSClientMemoryManager (lazy-typed) + self._is_rw: Optional[bool] = None + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def connect(self) -> bool: + """Connect to the per-GPU GMS daemon and acquire a session. + + Returns ``True`` on success. On import failure (library not + installed) or socket failure, returns ``False`` and logs a warning + — callers are expected to surface a useful error to the user. + """ + try: + from gpu_memory_service.client.torch.allocator import ( + get_or_create_gms_client_memory_manager, + ) + from gpu_memory_service.common.locks import GrantedLockType, RequestedLockType + from gpu_memory_service.common.utils import get_socket_path + from gpu_memory_service.integrations.common.patches import patch_empty_cache + except ImportError: + logger.warning( + "gpu_memory_service library not installed; cannot use " + "LoadFormat.GMS. Install from " + "https://github.com/ai-dynamo/dynamo/tree/main/lib/gpu_memory_service" + ) + return False + + mode_map = { + "rw": RequestedLockType.RW, + "ro": RequestedLockType.RO, + "auto": RequestedLockType.RW_OR_RO, + } + + # Resolve default socket path against the upstream convention so + # multiple TRT-LLM instances on the same node automatically share + # the same per-GPU daemon. + socket_path = self._socket_path + if socket_path is None: + socket_path = get_socket_path(self._device_index, self._tag) + self._socket_path = socket_path + + try: + self._client = get_or_create_gms_client_memory_manager( + socket_path, + self._device_index, + mode=mode_map[self._mode], + tag=self._tag, + ) + except Exception as e: + # TODO(GMS-API): narrow once upstream documents the expected + # client-creation failure modes (socket missing, connection + # refused, daemon crashed). Today the upstream API doesn't + # enumerate them, so the broad catch protects the + # "GMS optional / fall through to disk" guarantee at the cost + # of also swallowing programming bugs (e.g. signature drift). + logger.warning( + "Failed to connect to GMS at %s (mode=%s, tag=%s): %s", + socket_path, + self._mode, + self._tag, + e, + ) + self._client = None + self._is_rw = None + return False + + self._is_rw = self._client.granted_lock_type == GrantedLockType.RW + + # Process-wide safety patch: torch.cuda.empty_cache() can segfault + # when it tries to free GMS-backed VMM regions. Upstream's + # patch_empty_cache() rebinds torch.cuda.empty_cache to a GMS-aware + # variant that skips those regions. + # + # Process-wide is intentional, not a leak: torch internals, MoE + # balancer, draft-model setup, and unrelated TRT-LLM call sites all + # invoke empty_cache and would crash otherwise. A per-call-site + # wrapper would miss the calls we don't own. + # + # TODO(GMS-API): replace with a TRT-LLM-owned implementation once + # upstream's API stabilizes (see TODO on move_untracked_params). + # Idempotent and cheap. + try: + patch_empty_cache() + except Exception as e: + logger.debug("GMS patch_empty_cache failed (non-fatal): %s", e) + + logger.info( + "Connected to GMS at %s (mode=%s, granted=%s, tag=%s)", + socket_path, + self._mode, + "RW" if self._is_rw else "RO", + self._tag, + ) + return True + + @property + def is_rw(self) -> Optional[bool]: + """Whether this client holds the writer lock for the GMS pool. + + Returns: + ``True`` if RW (writer) was granted, ``False`` if RO (reader) + was granted, ``None`` before :meth:`connect` has been called + successfully or after :meth:`cleanup` has run. Transitions + from ``True`` to ``False`` after :meth:`finalize_write`. + """ + return self._is_rw + + def has_committed_weights(self) -> bool: + """Whether committed weights are available for RO materialization. + + Reports the granted lock type at the time of call, which only + becomes ``RO`` once a writer in this or another process has + published a finalized layout for ``tag``. + + Returns: + ``True`` if the granted lock is ``RO`` (committed weights + exist and can be zero-copy mapped); ``False`` otherwise, + including pre-connect, post-cleanup, and any error paths. + Never raises — best-effort by design so callers can use it + as a precondition check before invoking + :meth:`materialize_module`. + """ + if self._client is None: + return False + try: + from gpu_memory_service.common.locks import GrantedLockType + + return self._client.granted_lock_type == GrantedLockType.RO + except Exception: + return False + + # ------------------------------------------------------------------ + # RW path: scope allocations into the GMS pool, then finalize + # ------------------------------------------------------------------ + + @contextmanager + def mem_pool_scope( + self, + device: Optional[torch.device] = None, + ) -> Iterator[None]: + """Context manager scoping CUDA allocations to the GMS pool. + + All ``torch.empty`` / ``torch.zeros`` / etc. allocations inside + this scope are routed through the GMS pluggable allocator and + land in the shared memory region for the configured ``tag``. + Allocations made via paths that bypass the active allocator + (e.g. C++ ops that call ``cudaMalloc`` directly) escape this + scope and must be swept by :meth:`move_untracked_params` + before :meth:`finalize_write` is called. + + Args: + device: Target CUDA device. Defaults to + ``torch.device('cuda', current_device())``. + + Yields: + ``None`` — the value is unused; this is a scoping context. + + Raises: + RuntimeError: If ``connect()`` has not been called yet, or + if the granted lock is RO (only the writer may allocate + into the pool). + """ + if self._client is None: + raise RuntimeError("GMS client not connected. Call connect() first.") + if self._is_rw is False: + raise RuntimeError( + "GMS mem_pool_scope() is only valid in RW mode (this client was granted RO)." + ) + + from gpu_memory_service.client.torch.allocator import gms_use_mem_pool + + target_device = device + if target_device is None: + target_device = torch.device("cuda", self._device_index) + + with gms_use_mem_pool(self._tag, target_device): + yield + + def move_untracked_params(self, model: nn.Module) -> None: + """Migrate parameters allocated outside the GMS pool into it. + + Some parts of TRT-LLM's loading pipeline (e.g. ``post_load_weights``, + ``model.to("cuda")``) may allocate buffers outside the + ``mem_pool_scope`` context. ``finalize_write`` requires every + registered tensor to live in the GMS pool, so we copy any stray + CUDA params into freshly created GMS mappings of the same size. + + Mirrors ``gpu_memory_service.integrations.trtllm.model_loader. + _move_untracked_params`` so behavior matches the upstream + reference integration. + + Args: + model: The ``nn.Module`` to scan for stray CUDA parameters. + Buffers (``tensor_type != 'parameter'``), CPU tensors, + and tensors already backed by a GMS mapping are skipped. + + Raises: + RuntimeError: If ``connect()`` has not been called yet. + """ + # TODO(GMS-API): Clean up once the stable GMS API becomes available. + if self._client is None: + raise RuntimeError("GMS client not connected. Call connect() first.") + + from gpu_memory_service.client.torch.module import _iter_module_tensors + from gpu_memory_service.client.torch.tensor import _tensor_from_pointer + + gms_client = self._client + device_index = self._device_index + # Map original storage base ptr -> new GMS base_va. Serves two roles: + # 1. Dedups create_mapping() / copy_() calls when multiple tensors + # share a storage (e.g. tied embeddings, sliced fused params). + # 2. Lets us rebind every view of a shared storage to the SAME + # GMS mapping, so all tied/aliased parameters end up backed + # by the GMS pool. Without this, only the first encountered + # view was rebound; subsequent views still pointed at the + # original non-GMS storage and were missed by finalize_write. + storage_to_gms_va: dict[int, int] = {} + + # No torch.no_grad() guard: ``tensor.data = X`` bypasses autograd by + # definition (it's a data-attribute assignment, not an autograd-tracked + # operation), and ``replacement.copy_(tensor)`` is an in-place op on a + # freshly created tensor with no autograd history. Model loading also + # runs outside any grad-enabled context. + for _name, tensor, tensor_type in _iter_module_tensors(model): + if tensor_type != "parameter" or tensor is None or not tensor.is_cuda: + continue + + # GMS tracks whole storage allocations, so use the storage + # base pointer instead of a tensor view's offset pointer. + storage_base_ptr = tensor.untyped_storage().data_ptr() + if _ptr_in_gms(gms_client, int(storage_base_ptr)): + continue + + base_va = storage_to_gms_va.get(storage_base_ptr) + first_for_this_storage = base_va is None + if first_for_this_storage: + nbytes = _storage_nbytes(tensor) + base_va = gms_client.create_mapping(size=nbytes, tag=self._tag) + storage_to_gms_va[storage_base_ptr] = base_va + + replacement = _tensor_from_pointer( + int(base_va), + list(tensor.shape), + list(tensor.stride()), + tensor.dtype, + device_index, + ) + # Copy original bytes exactly once per storage; subsequent + # views of the same storage just rebind onto the same GMS + # mapping without recopying. + if first_for_this_storage: + replacement.copy_(tensor) + tensor.data = replacement + + def finalize_write(self, model: nn.Module) -> int: + """Register tensors, commit them, and transition this client to RO. + + After this returns successfully, ``is_rw`` flips to ``False`` + and other instances on the same node connecting in ``"auto"`` + or ``"ro"`` mode will receive a zero-copy RO mapping of the + committed layout. The transition is in-place: this client + keeps the same socket connection. + + Mirrors ``gpu_memory_service.integrations.common.utils. + finalize_gms_write``. + + Args: + model: The fully-loaded ``nn.Module`` whose CUDA parameters + will be registered with the GMS daemon. Every parameter + must already live in the GMS pool — call + :meth:`move_untracked_params` first to sweep strays + allocated outside :meth:`mem_pool_scope`. + + Returns: + Total bytes committed to the GMS pool for this ``tag``. + + Raises: + RuntimeError: If ``connect()`` has not been called yet, or + if the granted lock is RO (only the writer may commit). + """ + if self._client is None: + raise RuntimeError("GMS client not connected. Call connect() first.") + if self._is_rw is False: + raise RuntimeError("GMS finalize_write() is only valid in RW mode.") + + from gpu_memory_service.integrations.common.utils import finalize_gms_write + + bytes_committed = int(finalize_gms_write(self._client, model)) + # finalize_gms_write internally commits and reconnects in RO mode, + # so we mirror that state transition here. + self._is_rw = False + logger.info( + "GMS RW->RO: committed %.2f GiB at %s (tag=%s)", + bytes_committed / (1 << 30), + self._socket_path, + self._tag, + ) + return bytes_committed + + # ------------------------------------------------------------------ + # RO path: zero-copy import committed weights into model params + # ------------------------------------------------------------------ + + def materialize_module(self, model: nn.Module) -> None: + """Zero-copy import committed weights into model params (RO path). + + Replaces meta-initialized parameters with CUDA tensors backed + by GPU pointers from the shared memory region — no data copies, + no disk I/O, just CUDA VMM remapping. The model's submodule + layout must already match the writer's at commit time, including + any aliases / derived buffers introduced by ``post_load_weights``. + + Args: + model: The ``nn.Module`` to materialize. Walks the full + module tree (including submodules like ``draft_model`` + added for speculative decoding) and rebinds matching + parameters to GMS-backed storage. + + Raises: + RuntimeError: If ``connect()`` has not been called yet. + + Note: + ``post_load_weights()`` must be called on the model BEFORE + this method. The order ensures that any aliases / derived + parameters created by post-load hooks are present on the + module tree at materialization time, so they are bound to + the same GMS storage as their primary tensor. + """ + if self._client is None: + raise RuntimeError("GMS client not connected. Call connect() first.") + + from gpu_memory_service.client.torch.module import materialize_module_from_gms + + materialize_module_from_gms(self._client, model, device_index=self._device_index) + + logger.info( + "GMS RO: materialized weights from %s (tag=%s, tp_rank=%d/%d, total_bytes=%.2f GiB)", + self._socket_path, + self._tag, + self._mapping.tp_rank, + self._mapping.tp_size, + int(self._client.total_bytes) / (1 << 30), + ) + + # ------------------------------------------------------------------ + # Teardown + # ------------------------------------------------------------------ + + def cleanup(self) -> None: + """Release the GMS session and evict it from the per-tag registry. + + Idempotent: a second call after a successful cleanup is a no-op + because the client handle is dropped. Best-effort: any failure + in upstream ``client.close()`` or ``evict_gms_client_memory_manager`` + is logged (warning for the outer error, debug for ``close()``) + and swallowed so callers — including ``__del__`` paths in + ``ModelLoader`` and ``PyTorchModelEngine`` — never raise from + teardown. + + After return: + - The local client handle is unset (``_client is None``). + - A subsequent :meth:`connect` may re-establish a fresh + session against the same daemon. + - GMS-backed CUDA mappings remain alive on-device for any + other process holding an RO lock on this ``tag``. + """ + if self._client is None: + return + try: + from gpu_memory_service.client.torch.allocator import evict_gms_client_memory_manager + + client = self._client + try: + client.close() + except Exception as e: + # Best-effort: even if close() fails, evict so a future + # connect() in the same process can re-establish state. + # Log at debug so unrelated shutdown noise doesn't drown + # out the warning we already emit for the outer try. + logger.debug("GMS client.close() failed (best-effort): %s", e) + evict_gms_client_memory_manager(client) + logger.info("GMS: disconnected from %s", self._socket_path) + except Exception as e: + logger.warning("GMS cleanup error: %s", e) + finally: + self._client = None + self._is_rw = None + + +def _ptr_in_gms(gms_client, ptr: int) -> bool: + """Whether a raw CUDA pointer falls inside an existing GMS mapping. + + Mirrors ``gpu_memory_service.integrations.trtllm.model_loader._ptr_in_gms``. + Tolerates absence of the ``_mappings`` attribute on older GMS + releases by returning ``False`` (treating the pointer as "untracked" + so the caller's stray-handling path runs). + + Args: + gms_client: A connected ``GMSClientMemoryManager``. + ptr: A raw CUDA virtual address (``tensor.data_ptr()`` or a + storage base pointer). + + Returns: + ``True`` if ``ptr`` lies inside any mapping the client tracks + for any tag, ``False`` otherwise. Never raises. + """ + mappings = getattr(gms_client, "_mappings", None) + if not mappings: + return False + for mapping in mappings.values(): + base = int(getattr(mapping, "va", 0)) + size = int(getattr(mapping, "size", 0)) + if base and size and base <= ptr < base + size: + return True + return False + + +def _storage_nbytes(tensor: torch.Tensor) -> int: + """Bytes owned by ``tensor``'s underlying storage, including any padding. + + Used to size GMS mappings created in :meth:`GMSBackend.move_untracked_params` + so the replacement allocation matches the original storage exactly, + not the (possibly smaller) view the tensor exposes. + + Args: + tensor: Any CUDA ``torch.Tensor``. View-vs-base distinction is + intentional: we always want the whole storage size. + + Returns: + Total bytes of the underlying ``UntypedStorage``. + """ + storage = tensor.untyped_storage() + return int(storage.nbytes()) diff --git a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py index 304059794a3b..e5123b7cd40d 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py @@ -149,20 +149,33 @@ def model_name(self) -> Optional[str]: def query_timeout_s(self) -> Optional[int]: return self._query_timeout_s - @property - def p2p_succeeded(self) -> bool: - """Whether the last load_weights() call used P2P transfer. + def is_weights_preloaded(self) -> bool: + """Whether the last :meth:`load_weights` call wired weights directly into the model. + + Reports the result of the most recent ``load_weights()`` invocation + on this loader instance. ``ModelLoader`` consults this signal to + decide whether to run the standard weight-mapping pipeline: + + - ``True``: MX P2P transfer succeeded; weights already live in + model parameter buffers via direct writes from the upstream + ``MxLiveWeightLoader``. The mapping pipeline is skipped for + all parameters covered by P2P. + - ``False``: either P2P was never attempted (no MX server URL, + no model reference, library missing) or it failed and we + fell back to disk; weights still need to flow through + ``model.load_weights(...)`` via the standard mapper. + + Note this is a per-loader-instance flag, not a global one. The + flag is reset to ``False`` at the start of each ``load_weights`` + call, so the value is only meaningful immediately after a + successful call. - ``True`` means weights are already in model parameter buffers - and the standard weight-mapping pipeline should be skipped - for those parameters. + Returns: + ``True`` iff the last ``load_weights`` populated the model + via P2P; ``False`` before any call and on any fallback path. """ return self._p2p_succeeded - def is_weights_preloaded(self) -> bool: - """Whether the last MX load wrote weights directly into the model.""" - return self._p2p_succeeded - def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[str, Any]: """Load weights, preferring MX P2P transfer when available. @@ -404,7 +417,20 @@ def publish_as_source( def post_load_publish( self, model, *, checkpoint_dir: str, weights_preloaded: bool = False ) -> None: - """Publish only workers that loaded locally, not MX P2P receivers.""" + """Publish locally loaded weights as an MX source when appropriate. + + Args: + model: The loaded model whose parameters should be published for + future MX P2P receivers. + checkpoint_dir: Checkpoint directory used as a fallback model + identity when no explicit MX model name is configured. + weights_preloaded: Whether this worker already received weights + through MX P2P. When true, this worker is an MX receiver and + should not republish the same weights as a source. + + Returns: + None. + """ if weights_preloaded: return self.publish_as_source(model, checkpoint_dir=checkpoint_dir) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index f452ccab4f97..ae88b6d62e80 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -1558,17 +1558,96 @@ def _set_up_spec_metadata( max_seq_len=self.max_seq_len) return self.spec_metadata - def __del__(self) -> None: + def cleanup(self) -> None: + """Release resources owned by this model engine. + + Tears down, in order: + + 1. The optional ``ModelLoader`` (which in turn releases any + GMS client; see :meth:`ModelLoader.cleanup`). + 2. The model module reference. + 3. CUDA Graph captures (via :meth:`_release_cuda_graphs`). + 4. Input processors. + 5. Userbuffers (``ub.ub_deallocate`` per buffer); on per-buffer + failure the unfreed buffers are kept attached so a deterministic + retry doesn't double-free already-released ones, and the + collected errors are re-raised after the loop. + + Idempotency: + Subsequent calls are no-ops (guarded by ``_cleanup_done``). + The flag is set only at the end, so a partial cleanup that + raises mid-way will be retried on the next call. + + Raises: + RuntimeError: If one or more userbuffer deallocations fail + (chained from the first error). All other steps are + best-effort and either succeed or leak silently with + their errors logged at warning level by callees. + + Called from: + - :meth:`PyExecutor.shutdown` (deterministic teardown). + - :meth:`__del__` (best-effort fallback during garbage + collection / interpreter shutdown). + """ + if getattr(self, "_cleanup_done", False): + return + + # Cleanup is not truly atomic: released CUDA/GMS resources cannot be + # rolled back. Keep each handle live until its own release succeeds, + # so a failed cleanup can be retried without double-freeing resources + # that were already released. + model_loader = getattr(self, "model_loader", None) + if model_loader is not None: + model_loader.cleanup() + self.model_loader = None + self.model = None - self.model_loader = None + self._release_cuda_graphs() self.input_processor = None self.input_processor_with_hash = None - if getattr(self, 'ub_buffers', None): - for u in self.ub_buffers: - ub.ub_deallocate(u.addr) + + ub_buffers = getattr(self, 'ub_buffers', None) + if ub_buffers: + remaining_ub_buffers = [] + ub_errors = [] + for u in ub_buffers: + try: + ub.ub_deallocate(u.addr) + except RuntimeError as e: + # Keep failed buffers attached so a deterministic + # cleanup() call can retry without double-freeing buffers + # that were already deallocated successfully. + remaining_ub_buffers.append(u) + ub_errors.append(e) + self.ub_buffers = remaining_ub_buffers or None + if ub_errors: + raise RuntimeError( + "Failed to deallocate one or more userbuffers during " + "PyTorchModelEngine cleanup") from ub_errors[0] + # Release model weights. release_gc() + self._cleanup_done = True + + def __del__(self) -> None: + """Best-effort cleanup during garbage collection. + + Delegates to :meth:`cleanup`. Catches ``RuntimeError`` (raised + when one or more userbuffer deallocations fail) and + ``AttributeError`` (typical on partially-initialized engines + torn down during interpreter shutdown when module references + have already been cleared); both are logged and swallowed + because destructors cannot reliably surface exceptions. + + Deterministic callers (``PyExecutor.shutdown``) should call + :meth:`cleanup` directly so they see any failure. + """ + try: + self.cleanup() + except (RuntimeError, AttributeError) as e: + logger.warning( + "PyTorchModelEngine cleanup failed during destruction: %s", e) def _init_max_seq_len(self): # Allow user to override the inferred max_seq_len with a warning. diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 7b547975933f..5e98e730b6b8 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -275,7 +275,9 @@ def __init__(self, self.lora_config = lora_config self.model_weights_memory_tag = model_weights_memory_tag self.model_weights_restore_mode = model_weights_restore_mode + self.weight_mapper = None self._weight_pool_proxy = None + self._gms_backend = None @staticmethod def load_config_and_apply_defaults( @@ -349,7 +351,8 @@ def load( memo = dict() - if self.model_weights_memory_tag is not None: + if (self.model_weights_memory_tag is not None + and load_format != LoadFormat.GMS): # Allocate buffers to the outer virtual_memory_scope, # but parameters (weights) to the dedicated inner virtual_memory_scope. @@ -396,7 +399,7 @@ def allocate_weights_on_cuda(t: torch.Tensor): self.model_weights_restore_mode) as pool: model._apply(allocate_weights_on_cuda) self._weight_pool_proxy = pool - elif is_meta_init: + elif is_meta_init and load_format != LoadFormat.GMS: def init_meta_tensor(t: torch.Tensor): if t.device != torch.device('meta'): @@ -410,7 +413,8 @@ def init_meta_tensor(t: torch.Tensor): # Ensure everything is at least on CUDA # No-op if worked as expected - model.to("cuda") + if load_format != LoadFormat.GMS: + model.to("cuda") del memo rank_model_storage = get_rank_model_storage(model) @@ -418,6 +422,13 @@ def init_meta_tensor(t: torch.Tensor): f"Use {rank_model_storage / (1024**3):.2f} GB for model weights." ) weights_preloaded = False + # Set when either GMS RW or GMS RO branch has already run the + # post_load_* hooks itself, so the shared post-load block below + # must skip them. RW handles them inside ``mem_pool_scope`` so the + # committed pool reflects the post-post_load layout; RO runs + # ``module.post_load_weights()`` before ``materialize_module`` to + # wire aliases prior to zero-copy mapping. + gms_post_load_handled = False if load_format == LoadFormat.AUTO: # Pass model= so format-specific loaders (e.g. MX) can # write weights directly into parameter buffers via P2P. @@ -462,6 +473,224 @@ def init_meta_tensor(t: torch.Tensor): self._call_load_weights(model.load_draft_weights, weights, draft_weight_mapper) + elif load_format == LoadFormat.GMS: + # GPU Memory Service path: weight tensors live in a + # node-shared GPU memory pool so multiple TRT-LLM instances + # zero-copy share the same weight bytes. Two roles: + # - RW (writer): opens ``mem_pool_scope`` BEFORE meta- + # materialization and to(cuda), runs the entire model + # bring-up (load + draft load + post_load_*) inside the + # pool, then commits via ``finalize_write`` so RO peers + # receive the post-post_load layout. + # - RO (reader): zero-copy materializes weights from the + # pool into the model via ``materialize_module``; no + # disk I/O, no per-instance copies. + # Imported lazily so the optional ``gpu_memory_service`` + # dependency only loads when GMS is actually selected. + from tensorrt_llm._torch.memory import GMSBackend + + gms_backend = GMSBackend( + socket_path=self.llm_args.gms_config.socket_path, + mapping=self.mapping, + mode=self.llm_args.gms_config.mode, + tag=self.llm_args.gms_config.tag, + ) + + if not gms_backend.connect(): + raise RuntimeError( + "Failed to connect to GMS at " + f"{self.llm_args.gms_config.socket_path}") + + self._gms_backend = gms_backend + try: + # ``is_rw`` is ``Optional[bool]`` on the GPU memory backend + # protocol: True = RW lock granted, False = RO lock granted, + # None = unset. After a successful ``connect()`` it must be + # True or False; ``None`` is treated as an adapter bug and + # surfaces as a hard error rather than silently falling + # through to the RO path. + if gms_backend.is_rw is True: + # RW path: open the GMS pool BEFORE meta-tensor + # materialization and to(cuda), so every CUDA + # allocation needed to bring the model up — meta + # materialization, to(cuda), checkpoint load, draft + # load, post_load hooks — lands in the GMS pool. After + # the pool is closed, finalize_write commits the + # post-post_load layout for RO peers to map zero-copy. + # Mirrors the upstream Dynamo TRT-LLM shim pattern. + with gms_backend.mem_pool_scope(torch.device("cuda")): + # Materialize meta tensors inside the pool. Local + # memo because the outer one was deleted (and the + # outer ``init_meta_tensor`` branch is gated off + # for GMS so it never ran). + if is_meta_init: + gms_memo: dict = {} + + def init_meta_tensor_in_pool(t: torch.Tensor): + if t.device != torch.device('meta'): + return t + if t not in gms_memo: + gms_memo[t] = torch.empty_like( + t, device='cuda') + return gms_memo[t] + + model._apply(init_meta_tensor_in_pool) + + # Catch-all for any tensors not on CUDA yet. + model.to("cuda") + + weight_source = (model.llm_checkpoint_dir + if hasattr(model, + 'llm_checkpoint_dir') + else checkpoint_dir) + # Pass model= for symmetry with the LoadFormat.AUTO + # primary load above. Generic loaders (HF) ignore + # this kwarg; loaders that consume a live module + # reference (MX) use it for direct P2P writes into + # parameter buffers. Keeping the call shape + # consistent here avoids forgetting it when MX+GMS + # composition lands later. + weights = checkpoint_loader.load_weights( + weight_source, + mapping=self.mapping, + model=model) + + # ``weights`` may be: + # - non-empty dict: standard mapping pipeline runs + # - empty/None + ``is_weights_preloaded()=True``: + # the loader populated the model directly + # (e.g. MX P2P), so no mapping is needed + # - empty/None + ``is_weights_preloaded()=False``: + # loader returned nothing AND didn't preload — + # the model would be committed in an + # uninitialized state, so abort. Per-tensor + # partial loads (missing keys in a non-empty + # dict) are caught downstream by + # ``model.load_weights`` strict checking. + weights_preloaded = checkpoint_loader.is_weights_preloaded( + ) + if weights: + self.weight_mapper = checkpoint_loader.get_initialized_weight_mapper( + model, config) + self._call_load_weights(model.load_weights, + weights, + self.weight_mapper) + elif not weights_preloaded: + raise RuntimeError( + f"GMS RW: checkpoint loader " + f"'{checkpoint_loader.checkpoint_format}' " + f"returned empty weights ({weights!r}) and " + "did not preload the model. Refusing to " + "commit an unpopulated model to the GMS " + "pool.") + + if self.spec_config is not None and self.spec_config.spec_dec_mode.need_load_draft_weights( + ): + draft_weights = checkpoint_loader.load_weights( + self.spec_config.speculative_model, + mapping=self.mapping) + + draft_model_arch = model.draft_config.pretrained_config.architectures[ + 0] + draft_weight_mapper = AutoCheckpointMapper.get( + checkpoint_loader.checkpoint_format, + draft_model_arch) + draft_weight_mapper.init_model_and_config( + model.draft_model, model.draft_config) + + self._call_load_weights( + model.load_draft_weights, draft_weights, + draft_weight_mapper) + + # Run post_load hooks INSIDE the pool so any + # tensors they create or rebind (fused QKV, + # quantization scales, derived aliases) land in + # GMS and become part of the committed layout + # that RO peers receive. Closes hhzhang16's + # narrow-scope and commit-ordering concerns. + checkpoint_loader.post_load_apply( + model, weights_preloaded=weights_preloaded) + checkpoint_loader.post_load_publish( + model, + checkpoint_dir=checkpoint_dir, + weights_preloaded=weights_preloaded) + + for module in model.modules(): + if hasattr( + module, + 'post_load_weights') and not getattr( + module, '_weights_removed', False): + module.post_load_weights() + + # Defensive last-mile sweep: catches strays from + # C++ ops that bypassed the active torch + # allocator (e.g. native cudaMalloc). + gms_backend.move_untracked_params(model) + # Safe with active GMS mappings: GMSBackend.connect() + # installed upstream's patch_empty_cache(), which makes + # torch.cuda.empty_cache() skip GMS-backed VMM regions. + # Frees the non-GMS originals dropped by + # move_untracked_params (tensor.data was rebound to + # GMS-backed replacements above) before commit so the + # cached size doesn't show as live in memory accounting. + torch.cuda.empty_cache() + + # Pool closed. Commit the post-post_load layout. + gms_backend.finalize_write(model) + gms_post_load_handled = True + logger.info( + "LoadFormat.GMS (RW): loaded and committed weights via %s", + checkpoint_loader.checkpoint_format) + elif gms_backend.is_rw is False: + # RO path: weights are coming from a GMS donor that + # has already committed the post-post_load layout, so + # the receiver flags ``weights_preloaded=True`` on the + # checkpoint-loader hooks. ``MXCheckpointLoader.post_load_publish`` + # (and any other format-aware loader) honors this flag + # to early-return and not re-publish, while still + # letting ``post_load_apply`` perform any + # receiver-side per-format work (e.g., marking + # presharded modules). + # + # Hook order: + # 1. ``post_load_apply``: format-specific apply + # work (e.g., MX preshard markers). + # 2. Per-module ``post_load_weights``: creates + # aliases/derived parameter attributes BEFORE + # ``materialize_module`` walks the final module + # tree (including ``draft_model`` for spec dec). + # 3. ``materialize_module``: zero-copy bind GMS + # pool storage onto the model parameters. + # 4. ``post_load_publish``: any receiver-side + # publish (no-op via the receiver guard). + checkpoint_loader.post_load_apply( + model, weights_preloaded=True) + + for module in model.modules(): + if hasattr(module, + 'post_load_weights') and not getattr( + module, '_weights_removed', False): + module.post_load_weights() + + gms_backend.materialize_module(model) + + checkpoint_loader.post_load_publish( + model, + checkpoint_dir=checkpoint_dir, + weights_preloaded=True) + gms_post_load_handled = True + logger.info("LoadFormat.GMS (RO): materialized weights") + else: + raise RuntimeError( + f"GMS backend connected but lock state is unset " + f"(is_rw={gms_backend.is_rw!r}); expected True (RW) " + "or False (RO). This indicates a bug in the GMS " + "adapter or a protocol violation.") + except Exception: + gms_backend.cleanup() + self._gms_backend = None + raise + elif load_format == LoadFormat.DUMMY: self.weight_mapper = checkpoint_loader.get_initialized_weight_mapper( model, config) @@ -480,18 +709,30 @@ def init_meta_tensor(t: torch.Tensor): raise NotImplementedError( f"No load support for load format: {load_format}") - checkpoint_loader.post_load_apply( - model, weights_preloaded=weights_preloaded) - checkpoint_loader.post_load_publish( - model, - checkpoint_dir=checkpoint_dir, - weights_preloaded=weights_preloaded) - - for module in model.modules(): - if hasattr(module, 'post_load_weights') and not getattr( - module, '_weights_removed', False): - module.post_load_weights() - + if not gms_post_load_handled: + checkpoint_loader.post_load_apply( + model, weights_preloaded=weights_preloaded) + checkpoint_loader.post_load_publish( + model, + checkpoint_dir=checkpoint_dir, + weights_preloaded=weights_preloaded) + + for module in model.modules(): + if hasattr(module, 'post_load_weights') and not getattr( + module, '_weights_removed', False): + module.post_load_weights() + + # TODO(GMS-MOE-LB): when the (MoE, GMS) combination is enabled, + # ``register_weight_slots_after_to_cuda`` and ``finalize_model`` + # must run INSIDE ``mem_pool_scope`` and BEFORE ``finalize_write`` + # so MoE allocations become part of the committed layout that + # RO peers receive. Today they run outside the pool and after + # commit, which would silently produce a broken MoE routing + # state on RO peers — that combination is REJECTED at config + # time by ``TorchLlmArgs.validate_gms_moe_compat`` in + # ``tensorrt_llm/llmapi/llm_args.py``. When implementing the + # follow-up, drop the validator gate AFTER moving these calls + # into the pool scope. if isinstance(moe_load_balancer, MoeLoadBalancer): moe_load_balancer.register_weight_slots_after_to_cuda() logger.info("moe_load_balancer finalizing model...") @@ -506,12 +747,37 @@ def reload(self, model: DecoderModelForCausalLM, weights: dict, allow_partial_loading: bool = False): + if self.weight_mapper is None: + raise RuntimeError( + "Cannot reload weights: weight_mapper was not initialized. " + "This can happen when the initial load used GMS, MX P2P, or " + "VISION_ONLY, which bypass the standard weight mapping path.") self._call_load_weights(model.load_weights, weights, self.weight_mapper, allow_partial_loading=allow_partial_loading) torch.cuda.current_stream().synchronize() + def cleanup(self) -> None: + """Release backend resources acquired during :meth:`load`. + + Currently the only backend held by ``ModelLoader`` is the + optional GMS client, established by the ``LoadFormat.GMS`` + branch. Releasing it disconnects from the GMS daemon and evicts + the per-tag client registry entry; weights remain alive + on-device for any other process holding an RO lock on the same + ``tag``. + + Idempotent: a second call after a successful cleanup is a no-op + because the backend handle is dropped. Best-effort: any failure + in the underlying ``GMSBackend.cleanup()`` is swallowed there + and logged, so this method never raises — safe to call from + :meth:`PyTorchModelEngine.cleanup` and ``__del__`` paths. + """ + if self._gms_backend is not None: + self._gms_backend.cleanup() + self._gms_backend = None + def _load_and_validate_config( self, checkpoint_dir: str, checkpoint_loader: BaseCheckpointLoader) -> ModelConfig: diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index a893264d1ef1..b5cc85b2d059 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -964,6 +964,16 @@ def shutdown(self): for manager in self.resource_manager.resource_managers.values(): if manager: manager.shutdown() + # Note: do NOT call engine.cleanup() here. PyExecutor.shutdown() is + # also invoked mid-init by configure_kv_cache_capacity() in + # tensorrt_llm/_torch/pyexecutor/_util.py — the warmup pass calls + # shutdown() and then immediately reads model_engine.model.model_config + # to compute kv_cache_max_memory. cleanup() would set + # model_engine.model = None, breaking that read with + # `'NoneType' object has no attribute 'model_config'`. + # The engine's __del__ still calls cleanup() at terminal teardown + # (when the executor's reference is dropped), which is sufficient for + # the GMS daemon registry eviction the cleanup hook was added for. del self.model_engine if self.draft_model_engine is not None: del self.draft_model_engine diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 4fd4935901e3..c361686370ca 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -261,8 +261,7 @@ class BaseSparseAttentionConfig(StrictBaseModel): ) def supports_backend(self, backend: str) -> bool: - """ - Override if the sparse attention algorithm does not support + """Override if the sparse attention algorithm does not support a subset of the possible backends. """ return True @@ -271,8 +270,7 @@ def get_indices_block_size(self) -> int: return 1 def needs_separate_short_long_cuda_graphs(self) -> bool: - """ - Determines whether to capture a dedicated CUDA graph for batches consisting entirely of short sequences. + """Determines whether to capture a dedicated CUDA graph for batches consisting entirely of short sequences. If True, capture distinct graphs for short-only batches and general cases (e.g., long or mixed batches). If False, capture a single unified CUDA graph for all sequences regardless of length. The seq_len_threshold parameter defines the cutoff boundary between short and long sequences. @@ -388,8 +386,7 @@ def supports_backend(self, backend: str) -> bool: return backend == "pytorch" def needs_separate_short_long_cuda_graphs(self) -> bool: - """ - Whether to capture separate CUDA graphs for short and long sequences. + """Whether to capture separate CUDA graphs for short and long sequences. Use seq_len_threshold to determine the threshold for separating short and long sequences. """ self.seq_len_threshold = self.index_topk @@ -476,8 +473,7 @@ def _compute(phase: str, sparsity: Optional[float]) -> Optional[float]: class MoeLoadBalancerConfig(StrictBaseModel): - """ - Pydantic configuration model for the Mixture of Experts (MoE) load balancer. + """Pydantic configuration model for the Mixture of Experts (MoE) load balancer. This model holds configuration data (`num_slots`, etc.) as well as runtime state (`_ep_rank`, `_ep_size`) which must be set via the @@ -496,8 +492,7 @@ class MoeLoadBalancerConfig(StrictBaseModel): # --- Methods --- def setup(self, ep_rank: int, ep_size: int) -> None: - """ - Initializes the runtime state of the configuration. + """Initializes the runtime state of the configuration. This must be called before accessing properties like `num_local_slots`. """ self._ep_rank = ep_rank @@ -1004,8 +999,7 @@ def validate_max_concurrency_and_draft_len_schedule_mutually_exclusive( return self def supports_backend(self, backend: str) -> bool: - """ - Override if the speculation algorithm does not support + """Override if the speculation algorithm does not support a subset of the possible backends. """ return True @@ -1034,8 +1028,7 @@ def num_capture_layers(self) -> int: class KvCacheConnectorConfig(StrictBaseModel): - """ - Configuration for the KV Cache Connector. + """Configuration for the KV Cache Connector. Can be configured either by specifying a named preset via ``connector`` (e.g. ``"lmcache"``), or by providing explicit ``connector_module``, @@ -1313,8 +1306,7 @@ def spec_dec_mode(self): @functools.cached_property def num_capture_layers(self) -> int: - """ - Returns the number of layers to capture of the target model. + """Returns the number of layers to capture of the target model. If eagle3_layers_to_capture is not None, return the length of the set. Otherwise, assume Eagle3 base set and return 3. """ @@ -1430,8 +1422,7 @@ def spec_dec_mode(self): @functools.cached_property def num_capture_layers(self): - """ - Returns the number of layers to save. + """Returns the number of layers to save. The following hidden states are saved: - If eagle3_layers_to_capture is None, save the eagle3 base set plus the post norm last hidden state. @@ -1806,8 +1797,7 @@ def spec_dec_mode(self): class AutoDecodingConfig(DecodingBaseConfig): - """ - Configuration for auto speculative decoding. + """Configuration for auto speculative decoding. This config will automatically select a good, draft-model free speculation algorithm with some heuristic. @@ -1827,8 +1817,7 @@ def supports_backend(self, backend: str) -> bool: class PrometheusMetricsConfig(StrictBaseModel): - """ - Configuration for Prometheus metrics collection. + """Configuration for Prometheus metrics collection. Groups all Prometheus-related parameters including custom histogram bucket boundaries for latency metrics. @@ -1908,8 +1897,7 @@ def validate_histogram_buckets(cls, v: Optional[List[float]], class RayPlacementConfig(StrictBaseModel): - """ - Configuration for Ray GPU workers placement. + """Configuration for Ray GPU workers placement. Currently, this config is only used with AsyncLLM for RL scenarios. """ defer_workers_init: bool = Field( @@ -1975,8 +1963,8 @@ def validate_ray_placement(self) -> 'RayPlacementConfig': class ExecutorMemoryType(StrEnum): """Types of GPU memory used by executor. - These are used by the sleep/wakeup feature to target specific type of memory. - """ + These are used by the sleep/wakeup feature to target specific type of memory. + """ SAMPLER = "sampler" DRAFTER = "drafter" GUIDED_DECODER = "guided_decoder" @@ -2096,9 +2084,9 @@ def _validate_restore_modes(cls, v): class PybindMirror(ABC): - ''' A class containing the utilities for mirroring Python classes to + """A class containing the utilities for mirroring Python classes to pybind classes. - ''' + """ @abstractmethod def _to_pybind(self): @@ -2114,8 +2102,7 @@ def maybe_to_pybind(ins): @staticmethod def mirror_pybind_fields(pybind_class): - """ - Class decorator that ensures Python class fields mirror those of a C++ class. + """Class decorator that ensures Python class fields mirror those of a C++ class. Args: pybind_class: The C++ class whose fields should be mirrored @@ -2144,7 +2131,7 @@ def decorator(cls): @staticmethod def get_pybind_enum_fields(pybind_class): - ''' Get all the enum fields from the pybind class. ''' + """Get all the enum fields from the pybind class.""" return [ f for f in pybind_class.__members__.keys() if not f.startswith('_') and not callable(getattr(pybind_class, f)) @@ -2152,7 +2139,7 @@ def get_pybind_enum_fields(pybind_class): @staticmethod def mirror_pybind_enum(pybind_class): - ''' Mirror the enum fields from the pybind class to the Python class. ''' + """Mirror the enum fields from the pybind class to the Python class.""" def decorator(cls): assert issubclass(cls, Enum) @@ -2170,7 +2157,7 @@ def decorator(cls): @staticmethod def get_pybind_variable_fields(config_cls): - ''' Get all the variable fields from the pybind class. ''' + """Get all the variable fields from the pybind class.""" return [ f for f in dir(config_cls) if not f.startswith('_') and not callable(getattr(config_cls, f)) @@ -2178,7 +2165,7 @@ def get_pybind_variable_fields(config_cls): @staticmethod def pybind_equals(obj0, obj1): - ''' Check if two pybind objects are equal. ''' + """Check if two pybind objects are equal.""" assert type(obj0) is type(obj1) for field in PybindMirror.get_pybind_variable_fields(type(obj0)): if getattr(obj0, field) != getattr(obj1, field): @@ -2271,7 +2258,7 @@ def _to_pybind(self): @PybindMirror.mirror_pybind_enum(_ContextChunkingPolicy) class ContextChunkingPolicy(StrEnum, metaclass=PybindMirrorEnumMeta): - ''' Context chunking policy. ''' + """Context chunking policy.""" FIRST_COME_FIRST_SERVED = "FIRST_COME_FIRST_SERVED" EQUAL_PROGRESS = "EQUAL_PROGRESS" FORCE_CHUNK = "FORCE_CHUNK" @@ -3560,12 +3547,11 @@ def _load_config_from_ckpt(self, ckpt_dir: Path): @model_validator(mode="after") def validate_model_format_misc(self): - ''' - Load the model format, and do the following: + """Load the model format, and do the following: 1. Load the build_config if got an engine. 2. Load the parallel_config if got a checkpoint. - ''' + """ model_obj = _ModelWrapper(self.model) if model_obj.is_local_model and self.backend not in [ @@ -3662,6 +3648,8 @@ class LoadFormat(Enum): DUMMY = 1 # Only load the multimodal(vision) encoder weights VISION_ONLY = 2 + # Load weights through GPU Memory Service. + GMS = 3 class ModelExpressConfig(StrictBaseModel): @@ -3708,6 +3696,77 @@ def validate_preshard_strategy(self) -> 'ModelExpressConfig': return self +class GmsConfig(StrictBaseModel): + """Prototype configuration for GPU Memory Service (GMS) weight sharing. + + Composes orthogonally with ``checkpoint_format`` on ``TorchLlmArgs``: + GMS controls *where* weights live (a node-shared GPU memory pool + so multiple TRT-LLM instances zero-copy share the same bytes), + while ``checkpoint_format`` controls *how* they get there (HF disk, + MX P2P, etc.). Effective only when ``TorchLlmArgs.load_format == + LoadFormat.GMS``; setting any non-default field here without that + load_format triggers a warning via :meth:`TorchLlmArgs.validate_gms_config`. + + Two roles are decided at connect time by the GMS daemon, not by + config: + + - **RW (writer)**: the first instance loads weights into the pool + via the normal checkpoint-loader pipeline, then commits them. + - **RO (reader)**: subsequent instances zero-copy materialize the + already-committed layout — no disk I/O, no per-instance copies. + + See :class:`tensorrt_llm._torch.memory.gpu_memory_backend.GMSBackend` + for the integration adapter and the ``LoadFormat.GMS`` branch in + ``ModelLoader.load`` for orchestration. + """ + + socket_path: Optional[str] = Field( + default=None, + description="Unix domain socket path of the GPU Memory Service. " + "When unset, the GMS library resolves its default per-device path.", + status="prototype", + ) + + mode: Literal["auto", "rw", "ro"] = Field( + default="auto", + description="GMS operating mode: 'auto' requests RW or RO, 'rw' " + "requires writer mode, and 'ro' requires read-only mode.", + status="prototype", + ) + + tag: str = Field( + default="weights", + description="Tag identifying the model weight set in the GMS memory " + "pool. Defaults to 'weights' to match the GMS library convention. " + "The tag must uniquely identify a compatible model/parallelism " + "layout for a given GMS daemon; reusing a tag for different weights " + "can make RO readers attach to the wrong pool.", + status="prototype", + ) + + @model_validator(mode="after") + def validate_gms_config(self) -> 'GmsConfig': + """Enforce non-empty :attr:`tag`. + + :attr:`mode` is enforced by its ``Literal`` type annotation — + Pydantic rejects out-of-set values at field validation, before + this validator runs, so the JSON schema/docs are self-describing + (enum) without a duplicated whitelist here. This validator only + enforces the non-empty / non-whitespace contract on :attr:`tag`, + which the type system can't express. + + Raises: + ValueError: If ``tag`` is empty / whitespace-only. + + Returns: + ``self`` (Pydantic ``model_validator`` contract). + """ + if not self.tag or not self.tag.strip(): + raise ValueError( + f"gms_config.tag must be a non-empty string, got {self.tag!r}.") + return self + + class SamplerType(StrEnum): """Enum for sampler type options.""" TRTLLMSampler = "TRTLLMSampler" @@ -3960,6 +4019,12 @@ class TorchLlmArgs(BaseLlmArgs): status="prototype", ) + gms_config: GmsConfig = Field( + default_factory=GmsConfig, + description="GPU Memory Service (GMS) weight sharing config.", + status="prototype", + ) + kv_connector_config: Optional[KvCacheConnectorConfig] = Field( default=None, description="The config for KV cache connector.", @@ -4102,6 +4167,16 @@ def quant_config(self, value: QuantConfig): def convert_load_format(cls, v): if isinstance(v, LoadFormat): return v + # ``bool`` is a subclass of ``int`` in Python, so without an + # explicit bool check ``load_format=True/False`` would silently + # coerce to ``LoadFormat(1) == LoadFormat.DUMMY`` / + # ``LoadFormat(0) == LoadFormat.AUTO``. Reject early so callers + # who pass a misread boolean flag get an actionable error + # instead of a silently wrong checkpoint-load mode. + if isinstance(v, bool): + raise ValueError(f"Invalid LoadFormat: {v}") + if isinstance(v, int): + return LoadFormat(v) load_format = v.upper() if load_format not in LoadFormat.__members__: raise ValueError(f"Invalid LoadFormat: {v}") @@ -4285,6 +4360,79 @@ def validate_mx_config(self) -> 'TorchLlmArgs': self.checkpoint_format) return self + @model_validator(mode="after") + def validate_gms_config(self) -> 'TorchLlmArgs': + """Warn when GMS settings are provided without enabling GMS load. + + Catches the most common misconfiguration: a user customizes + :attr:`gms_config` (e.g. sets a custom ``socket_path`` or + ``mode``) but forgets to set ``load_format='GMS'``, in which + case the entire GMS config is silently ignored. We emit a + warning so the user notices at config time instead of + debugging "why are my workers not zero-copy sharing weights?" + afterwards. + + Detection is by deviation from defaults: any of + ``socket_path != None``, ``mode != 'auto'``, or + ``tag != 'weights'`` triggers the warning when + ``load_format != LoadFormat.GMS``. This is intentionally a + warning (not an error) so callers that pre-populate config + objects from templates aren't broken. + + Returns: + ``self`` (Pydantic ``model_validator`` contract). + """ + gms_config_is_non_default = (self.gms_config.socket_path is not None + or self.gms_config.mode != "auto" + or self.gms_config.tag != "weights") + if gms_config_is_non_default and self.load_format != LoadFormat.GMS: + logger.warning( + "gms_config is set but load_format is '%s', not 'GMS'. " + "The GMS config will be ignored. Set load_format='GMS' to " + "enable GPU Memory Service.", self.load_format.name) + return self + + @model_validator(mode="after") + def validate_gms_moe_compat(self) -> 'TorchLlmArgs': + """Reject ``LoadFormat.GMS`` combined with a MoE load balancer. + + The ``MoeLoadBalancer``'s ``register_weight_slots_after_to_cuda`` + and ``finalize_model`` run AFTER the GMS RW pool is closed and + ``finalize_write`` has committed, so any CUDA allocations they + make land in non-GMS memory and are NOT part of the committed + layout that RO peers receive. The result is "wrong inference, + no error" on RO peers (broken MoE routing state). Failing at + config-validation time is strictly better than that silent + miscompute. + + The fix for this gap (running the MoE finalize work INSIDE + ``mem_pool_scope`` and BEFORE ``finalize_write`` so MoE + allocations are part of the committed layout) is tracked as + the (MoE, GMS) follow-up; see ``model_loader.py``'s + ``TODO(GMS-MOE-LB)`` comment. + + Returns: + ``self`` (Pydantic ``model_validator`` contract). + + Raises: + ValueError: When ``load_format == LoadFormat.GMS`` and + ``moe_config.load_balancer`` is set. + """ + if (self.load_format == LoadFormat.GMS and self.moe_config is not None + and self.moe_config.load_balancer is not None): + raise ValueError( + "LoadFormat.GMS is incompatible with moe_config.load_balancer " + "in this PR. The MoE load balancer's " + "register_weight_slots_after_to_cuda and finalize_model run " + "after the GMS pool closes and finalize_write commits, so " + "their allocations land outside the committed layout. RO " + "peers would receive a broken MoE routing state. Either " + "disable moe_config.load_balancer or use LoadFormat.AUTO. " + "Tracked as the (MoE, GMS) follow-up at " + "tensorrt_llm/_torch/pyexecutor/model_loader.py " + "(see TODO(GMS-MOE-LB)).") + return self + @model_validator(mode="after") def validate_load_balancer(self) -> 'TorchLlmArgs': if isinstance(self.moe_config.load_balancer, str): @@ -4372,7 +4520,7 @@ def warn_on_unstable_feature_usage(self) -> 'TorchLlmArgs': def validate_ray_worker_extension_cls(self) -> 'TorchLlmArgs': if self.ray_worker_extension_cls is not None and self.orchestrator_type != "ray": raise ValueError( - f"ray_worker_extension_cls is only supported with orchestrator_type='ray'" + "ray_worker_extension_cls is only supported with orchestrator_type='ray'" ) return self @@ -4507,7 +4655,7 @@ def update_llm_args_with_extra_options(llm_args: Dict, def get_model_format(model_dir: str, trust_remote_code: bool = False) -> _ModelFormatKind: - ''' Get the format of the model. ''' + """Get the format of the model.""" if not (Path(model_dir) / 'config.json').exists(): raise ValueError( f"Failed to infer model format because no config.json exists in {model_dir}" diff --git a/tests/unittest/_torch/memory/test_gms_backend.py b/tests/unittest/_torch/memory/test_gms_backend.py new file mode 100644 index 000000000000..5f01def257c0 --- /dev/null +++ b/tests/unittest/_torch/memory/test_gms_backend.py @@ -0,0 +1,411 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Unit tests for ``GMSBackend`` (``LoadFormat.GMS``). + +These tests intentionally do NOT exercise the upstream +``gpu_memory_service`` library. Tests for the import-failure fallback +path mock ``gpu_memory_service.*`` symbols out of ``sys.modules`` so the +assertion is about *our* fallback behavior, not about the upstream API. + +The only piece of real CUDA touched by ``GMSBackend.__init__`` is a +single ``torch.cuda.current_device()`` call to record the device index. +We monkeypatch that to a fixed integer for the whole module so every +control-path test (construction, pre-connect, connect-failure, RW/RO +gating, protocol conformance) runs on CPU CI without a GPU. + +TODO(GMS-RESTART-E2E): an end-to-end "restart-after-death with weights +resident in GMS" smoke test is the right precursor to shadow-engine +failover work — it validates KV-cache memory accounting on the +restarted worker. Lives in a separate GPU + daemon test suite, not +here. Tracked as a follow-up PR; see PR #13926 review thread +(galletas1712) and §06 of the failover design doc. +""" + +import sys +from contextlib import contextmanager +from unittest.mock import MagicMock + +import pytest +import torch + +from tensorrt_llm._torch.memory.gpu_memory_backend import ( + GMSBackend, + GPUMemoryBackend, + _ptr_in_gms, + _storage_nbytes, +) + + +@pytest.fixture(autouse=True) +def _stub_current_device(monkeypatch): + """Make ``GMSBackend.__init__`` runnable on CPU CI. + + The only real-CUDA dependency in our backend's ``__init__`` is + ``torch.cuda.current_device()`` — there are no kernel launches. + Stubbing it to ``0`` lets every mocked happy/failure connect path + actually execute under CPU CI. + """ + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- + + +class TestConstruction: + """GMSBackend construction-time invariants.""" + + @pytest.mark.parametrize("mode", ["rw", "ro", "auto"], ids=["rw", "ro", "auto"]) + def test_accepts_valid_modes(self, mode): + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock(), mode=mode) + assert backend._mode == mode + assert backend._client is None + assert backend._is_rw is None + + @pytest.mark.parametrize( + "bad_mode", + ["RW", "Auto", "read", "write", "shadow", "", "rw ", "auto-detect"], + ids=[ + "uppercase-rw", + "mixedcase-auto", + "read", + "write", + "shadow", + "empty", + "trailing-space", + "auto-detect", + ], + ) + def test_rejects_invalid_modes(self, bad_mode): + with pytest.raises(ValueError, match="GMS mode must be"): + GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock(), mode=bad_mode) + + def test_default_tag(self): + # IMPORTANT: default tag must be ``weights``, NOT ``model_weights``. + # Matches the GMS library convention (GMS_TAGS upstream). + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock()) + assert backend._tag == "weights" + assert GMSBackend.DEFAULT_TAG == "weights" + + def test_custom_tag(self): + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock(), tag="custom") + assert backend._tag == "custom" + + def test_socket_path_stored(self): + backend = GMSBackend(socket_path="/tmp/custom.sock", mapping=MagicMock()) + assert backend._socket_path == "/tmp/custom.sock" + + def test_socket_path_none_resolved_lazily(self): + # Socket-path resolution to upstream get_socket_path(device, tag) + # happens inside connect(), not __init__. Construction with + # socket_path=None must not raise. + backend = GMSBackend(socket_path=None, mapping=MagicMock()) + assert backend._socket_path is None + assert backend._client is None + + +# --------------------------------------------------------------------------- +# Properties before connect() +# --------------------------------------------------------------------------- + + +class TestPreConnectState: + def _backend(self): + return GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock()) + + def test_is_rw_is_none(self): + assert self._backend().is_rw is None + + def test_has_committed_weights_returns_false(self): + # No committed weights when not connected — return False (not raise). + assert self._backend().has_committed_weights() is False + + def test_cleanup_safe_when_never_connected(self): + # cleanup() must be idempotent and safe to call before connect(). + self._backend().cleanup() # must not raise + + @pytest.mark.parametrize( + "method_name, invoke", + [ + # invoke(backend) -> trigger; we wrap mem_pool_scope so the + # context-manager protocol gets exercised inside pytest.raises. + ("mem_pool_scope", lambda backend: backend.mem_pool_scope().__enter__()), + ("materialize_module", lambda backend: backend.materialize_module(MagicMock())), + ("finalize_write", lambda backend: backend.finalize_write(MagicMock())), + ("move_untracked_params", lambda backend: backend.move_untracked_params(MagicMock())), + ], + ids=["mem-pool-scope", "materialize-module", "finalize-write", "move-untracked-params"], + ) + def test_method_raises_when_not_connected(self, method_name, invoke): + # Every method that uses the GMS client must guard with a clear + # "not connected" error before dereferencing self._client. + with pytest.raises(RuntimeError, match="not connected"): + invoke(self._backend()) + + +# --------------------------------------------------------------------------- +# connect() — graceful failure on missing upstream library +# --------------------------------------------------------------------------- + + +class TestConnectFailure: + def test_returns_false_when_upstream_not_installed(self): + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock()) + with _block_gms(): + assert backend.connect() is False + assert backend._client is None + assert backend._is_rw is None + + def test_returns_false_when_upstream_raises(self): + # If get_or_create_gms_client_memory_manager raises (e.g. socket + # doesn't exist), connect() must surface a False (not propagate). + backend = GMSBackend(socket_path="/tmp/nonexistent.sock", mapping=MagicMock()) + + fake_gms = _build_fake_gms_lock_failure(error=RuntimeError("socket missing")) + with _install_fake_gms(fake_gms): + assert backend.connect() is False + assert backend._client is None + + +# --------------------------------------------------------------------------- +# RW-vs-RO method gating +# --------------------------------------------------------------------------- + + +class TestRwOnlyMethodsGated: + """RW-only methods must raise when the granted lock is RO.""" + + def _ro_backend(self): + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock()) + # Simulate "connected, granted RO" without going through real connect(). + backend._client = MagicMock() + backend._is_rw = False + return backend + + @pytest.mark.parametrize( + "method_name, invoke", + [ + ("mem_pool_scope", lambda backend: backend.mem_pool_scope().__enter__()), + ("finalize_write", lambda backend: backend.finalize_write(MagicMock())), + ], + ids=["mem-pool-scope", "finalize-write"], + ) + def test_method_raises_when_granted_ro(self, method_name, invoke): + with pytest.raises(RuntimeError, match="only valid in RW mode"): + invoke(self._ro_backend()) + + +# --------------------------------------------------------------------------- +# Helper functions: _ptr_in_gms, _storage_nbytes +# --------------------------------------------------------------------------- + + +def _make_gms_client_with_mapping(va: int, size: int): + """Build a fake GMS client that exposes one mapping at (va, size).""" + client = MagicMock() + mapping = MagicMock() + mapping.va = va + mapping.size = size + client._mappings = {"k": mapping} + return client + + +class TestPtrInGms: + """``_ptr_in_gms`` semantics: half-open intervals + defensive guards.""" + + def test_returns_false_when_no_mappings_attr(self): + # Older GMS releases may not have ``_mappings`` at all. + client = MagicMock(spec=[]) + assert _ptr_in_gms(client, 0xABCDEF) is False + + def test_returns_false_when_mappings_empty(self): + client = MagicMock() + client._mappings = {} + assert _ptr_in_gms(client, 0xABCDEF) is False + + @pytest.mark.parametrize( + "label, ptr, expected", + [ + # Reference mapping is at va=0x1000, size=0x100 (range + # [0x1000, 0x1100) — half-open). + ("inside_mapping", 0x1080, True), + ("at_mapping_start", 0x1000, True), + ("just_below_end", 0x10FF, True), + ("at_mapping_end", 0x1100, False), # half-open + ("just_below_start", 0x0FFF, False), + ("far_outside", 0x2000, False), + ], + ids=[ + "inside-mapping", + "at-mapping-start", + "just-below-end", + "at-mapping-end", + "just-below-start", + "far-outside", + ], + ) + def test_half_open_interval(self, label, ptr, expected): + client = _make_gms_client_with_mapping(va=0x1000, size=0x100) + assert _ptr_in_gms(client, ptr) is expected, ( + f"case={label}: ptr=0x{ptr:x} expected {expected}" + ) + + @pytest.mark.parametrize( + "label, va, size", + [ + ("zero_va_zero_size", 0, 0), + ("zero_va_nonzero_size", 0, 0x100), + ("nonzero_va_zero_size", 0x1000, 0), + ], + ids=["zero-va-zero-size", "zero-va-nonzero-size", "nonzero-va-zero-size"], + ) + def test_zero_sentinels_never_match(self, label, va, size): + # Defensive: a mapping with va=0 or size=0 must never match, + # even when the queried pointer is 0. + client = _make_gms_client_with_mapping(va=va, size=size) + assert _ptr_in_gms(client, 0) is False, f"case={label}" + assert _ptr_in_gms(client, va) is False, f"case={label}" + + +class TestStorageNbytes: + def test_cpu_tensor(self): + t = torch.empty(10, dtype=torch.float32) + # 10 * 4 = 40 bytes minimum + assert _storage_nbytes(t) >= 40 + + def test_dtype_dependent(self): + t_f32 = torch.empty(10, dtype=torch.float32) + t_bf16 = torch.empty(10, dtype=torch.bfloat16) + # 10 fp32 occupies more than 10 bf16 + assert _storage_nbytes(t_f32) > _storage_nbytes(t_bf16) + + def test_shape_independent_for_views(self): + # Storage size should reflect the underlying buffer, not the view. + base = torch.empty(100, dtype=torch.float32) + view = base.view(10, 10) + assert _storage_nbytes(view) == _storage_nbytes(base) + + +# --------------------------------------------------------------------------- +# Protocol conformance +# --------------------------------------------------------------------------- + + +class TestProtocolConformance: + def test_gms_backend_implements_protocol(self): + # GMSBackend instances must satisfy the runtime-checkable + # GPUMemoryBackend protocol. + backend = GMSBackend(socket_path="/tmp/gms.sock", mapping=MagicMock()) + assert isinstance(backend, GPUMemoryBackend) + + def test_protocol_method_set(self): + # The protocol must keep the methods that model_loader.py invokes + # so a different backend (e.g. CudaIpcBackend) can be plugged in. + required = { + "connect", + "is_rw", + "has_committed_weights", + "mem_pool_scope", + "materialize_module", + "finalize_write", + "move_untracked_params", + "cleanup", + } + for method in required: + assert hasattr(GPUMemoryBackend, method), ( + f"GPUMemoryBackend protocol missing method: {method}" + ) + + +# --------------------------------------------------------------------------- +# Helpers — fake gpu_memory_service modules and import blockers +# --------------------------------------------------------------------------- + + +_GMS_MODULE_NAMES = [ + "gpu_memory_service", + "gpu_memory_service.client", + "gpu_memory_service.client.torch", + "gpu_memory_service.client.torch.allocator", + "gpu_memory_service.common", + "gpu_memory_service.common.locks", + "gpu_memory_service.common.utils", + "gpu_memory_service.integrations", + "gpu_memory_service.integrations.common", + "gpu_memory_service.integrations.common.patches", +] + + +@contextmanager +def _block_gms(): + """Context manager that makes ``import gpu_memory_service.*`` raise.""" + saved = {name: sys.modules.get(name) for name in _GMS_MODULE_NAMES} + try: + for name in _GMS_MODULE_NAMES: + sys.modules[name] = None # forces ImportError on import + yield + finally: + for name, prior in saved.items(): + if prior is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = prior + + +def _build_fake_gms_lock_failure(*, error): + """Build a fake gpu_memory_service whose factory raises on call.""" + fake_pkg = MagicMock(name="gpu_memory_service") + fake_pkg.client = MagicMock(name="gpu_memory_service.client") + fake_pkg.client.torch = MagicMock(name="gpu_memory_service.client.torch") + fake_pkg.client.torch.allocator = MagicMock(name="gpu_memory_service.client.torch.allocator") + fake_pkg.client.torch.allocator.get_or_create_gms_client_memory_manager = MagicMock( + side_effect=error + ) + + # Lock-type and util modules + fake_pkg.common = MagicMock() + fake_pkg.common.locks = MagicMock() + fake_pkg.common.locks.RequestedLockType = MagicMock(RW="rw", RO="ro", RW_OR_RO="rw_or_ro") + fake_pkg.common.locks.GrantedLockType = MagicMock(RW="rw", RO="ro") + fake_pkg.common.utils = MagicMock() + fake_pkg.common.utils.get_socket_path = MagicMock(return_value="/tmp/fake.sock") + + # patch_empty_cache stub + fake_pkg.integrations = MagicMock() + fake_pkg.integrations.common = MagicMock() + fake_pkg.integrations.common.patches = MagicMock() + fake_pkg.integrations.common.patches.patch_empty_cache = MagicMock() + return fake_pkg + + +@contextmanager +def _install_fake_gms(fake_pkg): + """Install a fake ``gpu_memory_service`` tree into ``sys.modules``.""" + saved = {name: sys.modules.get(name) for name in _GMS_MODULE_NAMES} + try: + sys.modules["gpu_memory_service"] = fake_pkg + sys.modules["gpu_memory_service.client"] = fake_pkg.client + sys.modules["gpu_memory_service.client.torch"] = fake_pkg.client.torch + sys.modules["gpu_memory_service.client.torch.allocator"] = fake_pkg.client.torch.allocator + sys.modules["gpu_memory_service.common"] = fake_pkg.common + sys.modules["gpu_memory_service.common.locks"] = fake_pkg.common.locks + sys.modules["gpu_memory_service.common.utils"] = fake_pkg.common.utils + sys.modules["gpu_memory_service.integrations"] = fake_pkg.integrations + sys.modules["gpu_memory_service.integrations.common"] = fake_pkg.integrations.common + sys.modules["gpu_memory_service.integrations.common.patches"] = ( + fake_pkg.integrations.common.patches + ) + yield + finally: + for name, prior in saved.items(): + if prior is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = prior diff --git a/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py b/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py index 7ffea89b7773..c686a1c664f7 100644 --- a/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py +++ b/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py @@ -43,12 +43,12 @@ class TestConstruction: def test_no_args_constructs(self): loader = MXCheckpointLoader() assert loader.mx_server_url is None - assert loader.p2p_succeeded is False + assert loader.is_weights_preloaded() is False def test_mx_server_url_stored(self): loader = MXCheckpointLoader(mx_server_url="http://mx:8001") assert loader.mx_server_url == "http://mx:8001" - assert loader.p2p_succeeded is False + assert loader.is_weights_preloaded() is False def test_query_timeout_stored(self): loader = MXCheckpointLoader(mx_server_url="http://mx:8001", query_timeout_s=900) @@ -72,9 +72,9 @@ def test_checkpoint_format_backing_attr(self): loader = MXCheckpointLoader() assert loader._checkpoint_format == "MX" - def test_p2p_succeeded_property_initial(self): + def test_is_weights_preloaded_initial(self): loader = MXCheckpointLoader() - assert loader.p2p_succeeded is False + assert loader.is_weights_preloaded() is False # --------------------------------------------------------------------------- @@ -123,7 +123,7 @@ class TestLoadWeightsFallback: """Disk-fallback paths that should not touch the upstream MX library. All four fallback triggers share the same observable contract: - ``p2p_succeeded`` stays False, the parent ``HfCheckpointLoader. + ``is_weights_preloaded()`` stays False, the parent ``HfCheckpointLoader. load_weights`` is invoked exactly once, and its return value is propagated unchanged. We parameterize the trigger to keep that contract in one place. @@ -157,6 +157,12 @@ def _upstream_raises(stack): ("modelexpress_not_installed", _modelexpress_unavailable), ("upstream_raises", _upstream_raises), ], + ids=[ + "no-mx-server-url", + "no-model-kwarg", + "modelexpress-not-installed", + "upstream-raises", + ], ) def test_falls_back_to_disk(self, trigger_id, setup): sentinel = {"disk-load": "result"} @@ -172,8 +178,8 @@ def test_falls_back_to_disk(self, trigger_id, setup): f"trigger={trigger_id}: production code must propagate the " "parent loader's return value unchanged" ) - assert loader.p2p_succeeded is False, ( - f"trigger={trigger_id}: p2p_succeeded must stay False on any fallback path" + assert loader.is_weights_preloaded() is False, ( + f"trigger={trigger_id}: is_weights_preloaded() must stay False on any fallback path" ) mock_super_load.assert_called_once() @@ -186,8 +192,9 @@ def test_falls_back_to_disk(self, trigger_id, setup): class TestLoadWeightsMxPath: def test_p2p_full_success_returns_empty_dict(self): # Empty fallback dict means MX delivered all weights into model - # params; ``ModelLoader`` interprets the empty dict + p2p_succeeded - # flag as "skip the standard weight-mapping pipeline". + # params; ``ModelLoader`` interprets the empty dict + the + # ``is_weights_preloaded()`` signal as "skip the standard + # weight-mapping pipeline". loader = MXCheckpointLoader(mx_server_url="http://mx:8001") fake_mx = _build_fake_modelexpress(load_weights_return={}) mapping = MagicMock(name="mapping") @@ -197,7 +204,7 @@ def test_p2p_full_success_returns_empty_dict(self): result = loader.load_weights("/nonexistent", mapping=mapping, model=model) assert result == {} - assert loader.p2p_succeeded is True + assert loader.is_weights_preloaded() is True # Verify the integration contract with the upstream library: # 1. Constructed MxLiveWeightLoader with our mx_server_url. @@ -224,7 +231,7 @@ def test_mixed_success_returns_fallback_weights(self): ): result = loader.load_weights("/nonexistent", mapping=MagicMock(), model=MagicMock()) - assert loader.p2p_succeeded is True + assert loader.is_weights_preloaded() is True assert result is fallback mock_super_load.assert_not_called() @@ -516,6 +523,16 @@ class TestNormalizeModelIdentity: # Empty / sentinel. ("empty", "", "unknown"), ], + ids=[ + "bare-name", + "hub-id", + "nested-hub-id", + "abs-path-simple", + "abs-path-nested", + "dot-relative", + "home-expansion", + "empty", + ], ) def test_basic_cases(self, label, value, expected): assert _normalize_model_identity(value) == expected, f"case={label}" diff --git a/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py new file mode 100644 index 000000000000..bf692cb61abe --- /dev/null +++ b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py @@ -0,0 +1,382 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for GMS-specific branches in ``ModelLoader``.""" + +from contextlib import contextmanager, nullcontext +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch +from torch import nn + +from tensorrt_llm._torch import memory as memory_mod +from tensorrt_llm._torch.pyexecutor import model_loader as model_loader_mod +from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader +from tensorrt_llm.llmapi.llm_args import LoadFormat + + +class _TinyModel(nn.Module): + def __init__(self, events, *, include_draft=False): + super().__init__() + self._events = events + if include_draft: + self.draft_model = nn.Module() + + def _apply(self, fn): + self._events.append("_apply") + return self + + def to(self, *args, **kwargs): + self._events.append("to") + return self + + def load_weights(self, weights, mapper): + self._events.append("load_weights") + + def post_load_weights(self): + self._events.append("post_load_weights") + + +@contextmanager +def _moe_context(config, mapping): + yield None + + +@contextmanager +def _pool_scope(events): + events.append("pool_enter") + yield + events.append("pool_exit") + + +def _make_loader(monkeypatch, *, events, spec_config=None): + llm_args = SimpleNamespace( + load_format=LoadFormat.GMS, + gms_config=SimpleNamespace(socket_path="/tmp/gms.sock", mode="auto", tag="weights"), + ) + loader = ModelLoader( + llm_args=llm_args, + mapping=MagicMock(name="mapping"), + spec_config=spec_config, + sparse_attention_config=None, + max_num_tokens=128, + max_seq_len=128, + ) + loader._call_load_weights = MagicMock( + side_effect=lambda fn, weights, mapper, **kwargs: fn(weights, mapper) + ) + loader._load_and_validate_config = MagicMock(return_value=SimpleNamespace(name="config")) + + monkeypatch.setattr(model_loader_mod, "timing", lambda *_args, **_kwargs: nullcontext()) + monkeypatch.setattr(model_loader_mod, "maybe_create_moe_load_balancer", _moe_context) + monkeypatch.setattr(model_loader_mod, "MetaInitMode", lambda: nullcontext()) + monkeypatch.setattr( + model_loader_mod.AutoModelForCausalLM, + "from_config", + MagicMock(return_value=_TinyModel(events, include_draft=spec_config is not None)), + ) + monkeypatch.setattr(model_loader_mod, "get_rank_model_storage", lambda _model: 0) + monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr( + torch.cuda, + "current_stream", + lambda: SimpleNamespace(synchronize=lambda: None), + ) + return loader + + +def _build_gms_backend(*, is_rw, events): + backend = MagicMock() + backend.connect.return_value = True + backend.is_rw = is_rw + if is_rw: + backend.mem_pool_scope.side_effect = lambda _device: _pool_scope(events) + else: + + def _materialize(_model): + events.append("materialize") + + backend.materialize_module.side_effect = _materialize + return backend + + +def _install_gms_backend(monkeypatch, backend): + monkeypatch.setattr(memory_mod, "GMSBackend", MagicMock(return_value=backend)) + + +def _spec_config_needing_draft_weights(): + return SimpleNamespace( + spec_dec_mode=SimpleNamespace(need_load_draft_weights=lambda: True), + speculative_model="/draft", + ) + + +@pytest.mark.parametrize( + "is_rw, expected_events", + [ + pytest.param( + True, + [ + "pool_enter", + "_apply", + "to", + "load_weights", + "post_load_weights", + "pool_exit", + ], + id="rw", + ), + pytest.param( + False, + ["post_load_weights", "materialize"], + id="ro", + ), + ], +) +def test_gms_load_branch(monkeypatch, is_rw, expected_events): + """Verify ``ModelLoader.load`` dispatches correctly per GMS lock mode. + + Cases: + rw: the writer opens the GMS pool BEFORE meta-tensor materialization + and ``model.to('cuda')``, runs the entire model bring-up + (``_apply`` for meta materialization, ``to('cuda')``, weight + load, ``post_load_weights``) inside the pool, then commits via + ``finalize_write`` once the scope exits. + ro: the reader runs ``post_load_weights`` to wire module aliases + first, then GMS materializes weights via zero-copy mapping. + """ + events = [] + loader = _make_loader(monkeypatch, events=events) + backend = _build_gms_backend(is_rw=is_rw, events=events) + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + if is_rw: + # RW path: the checkpoint loader's returned weights flow into the + # standard model.load_weights mapping pipeline. + checkpoint_loader.load_weights.return_value = {"weight": MagicMock()} + + model, _ = loader.load("/ckpt", checkpoint_loader) + + assert loader._gms_backend is backend + assert events == expected_events + if is_rw: + # RW: load weights, run them through the mapper, then commit + # everything that landed in the GMS pool for RO consumers. + # ``model=model`` is passed for symmetry with the LoadFormat.AUTO + # path (see model_loader.py); HF ignores it, MX uses it for direct + # P2P writes when MX+GMS composition eventually lands. + checkpoint_loader.load_weights.assert_called_once_with( + "/ckpt", mapping=loader.mapping, model=model + ) + loader._call_load_weights.assert_called_once() + backend.move_untracked_params.assert_called_once_with(model) + backend.finalize_write.assert_called_once_with(model) + else: + # RO: post_load_weights() must run before the GMS materialize + # step so module aliases are wired up before zero-copy mapping. + checkpoint_loader.load_weights.assert_not_called() + loader._call_load_weights.assert_not_called() + backend.materialize_module.assert_called_once_with(model) + + +def test_gms_rw_post_load_runs_inside_pool_before_finalize(monkeypatch): + """Every step that may allocate or rebind tensors must run inside the GMS pool. + + The widened ``mem_pool_scope`` covers meta-tensor materialization, + ``model.to('cuda')``, weight load, and the post_load_* hooks. Only + ``finalize_write`` runs after the pool closes, so the committed + layout reflects the post-post_load model. Asserts the exact ordering + so a future refactor cannot silently re-narrow the scope. + """ + events = [] + loader = _make_loader(monkeypatch, events=events) + backend = _build_gms_backend(is_rw=True, events=events) + + backend.move_untracked_params.side_effect = lambda _model: events.append( + "move_untracked_params" + ) + backend.finalize_write.side_effect = lambda _model: events.append("finalize_write") + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + checkpoint_loader.load_weights.return_value = {"weight": MagicMock()} + checkpoint_loader.post_load_apply.side_effect = lambda *_a, **_kw: events.append( + "post_load_apply" + ) + checkpoint_loader.post_load_publish.side_effect = lambda *_a, **_kw: events.append( + "post_load_publish" + ) + + loader.load("/ckpt", checkpoint_loader) + + assert events == [ + "pool_enter", + "_apply", + "to", + "load_weights", + "post_load_apply", + "post_load_publish", + "post_load_weights", + "move_untracked_params", + "pool_exit", + "finalize_write", + ] + + # Belt-and-braces: inside the events list, every "interesting" step + # must sit between pool_enter and pool_exit, with finalize_write the + # only thing strictly after. + enter_idx = events.index("pool_enter") + exit_idx = events.index("pool_exit") + finalize_idx = events.index("finalize_write") + assert finalize_idx > exit_idx, "finalize_write must follow pool exit" + for inside in ( + "_apply", + "to", + "load_weights", + "post_load_apply", + "post_load_publish", + "post_load_weights", + "move_untracked_params", + ): + idx = events.index(inside) + assert enter_idx < idx < exit_idx, ( + f"{inside!r} (idx={idx}) must run inside the pool (enter={enter_idx}, exit={exit_idx})" + ) + + +def test_gms_rw_loader_preload_skips_mapping_pipeline(monkeypatch): + """RW + empty ``weights`` + ``is_weights_preloaded()=True`` is legitimate. + + Mirrors the MX P2P preload scenario, where the checkpoint loader + writes weights directly into model parameters and signals the + preloaded state. ``ModelLoader`` must skip the standard mapping + pipeline and still commit the populated pool for RO peers. + """ + events = [] + loader = _make_loader(monkeypatch, events=events) + backend = _build_gms_backend(is_rw=True, events=events) + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "MX" + checkpoint_loader.load_weights.return_value = {} + checkpoint_loader.is_weights_preloaded.return_value = True + + model, _ = loader.load("/ckpt", checkpoint_loader) + + loader._call_load_weights.assert_not_called() + backend.move_untracked_params.assert_called_once_with(model) + backend.finalize_write.assert_called_once_with(model) + + +def test_gms_rw_no_load_and_no_preload_raises(monkeypatch): + """RW + empty ``weights`` + ``is_weights_preloaded()=False`` is a bug. + + Without weights and without a loader-driven preload, the model is + not populated. Committing it to GMS would expose an uninitialized + layout to RO peers, so ``ModelLoader`` must raise. + """ + events = [] + loader = _make_loader(monkeypatch, events=events) + backend = _build_gms_backend(is_rw=True, events=events) + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + checkpoint_loader.load_weights.return_value = {} + checkpoint_loader.is_weights_preloaded.return_value = False + + with pytest.raises(RuntimeError, match="Refusing to commit an unpopulated model"): + loader.load("/ckpt", checkpoint_loader) + + # Pool was opened, but nothing must be committed. + backend.finalize_write.assert_not_called() + backend.cleanup.assert_called_once() + assert loader._gms_backend is None + + +def test_gms_rw_exception_during_weight_mapping_cleans_up(monkeypatch): + """A writer error after pool population must release the GMS session.""" + events = [] + loader = _make_loader(monkeypatch, events=events) + loader._call_load_weights.side_effect = RuntimeError("mapping failed") + backend = _build_gms_backend(is_rw=True, events=events) + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + checkpoint_loader.load_weights.return_value = {"weight": MagicMock()} + + with pytest.raises(RuntimeError, match="mapping failed"): + loader.load("/ckpt", checkpoint_loader) + + backend.cleanup.assert_called_once() + backend.finalize_write.assert_not_called() + assert loader._gms_backend is None + + +def test_gms_ro_with_spec_config_materializes_full_model_tree(monkeypatch): + """RO materialization receives the root model, including draft_model.""" + events = [] + loader = _make_loader( + monkeypatch, + events=events, + spec_config=_spec_config_needing_draft_weights(), + ) + backend = _build_gms_backend(is_rw=False, events=events) + _install_gms_backend(monkeypatch, backend) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + + model, _ = loader.load("/ckpt", checkpoint_loader) + + assert hasattr(model, "draft_model") + backend.materialize_module.assert_called_once_with(model) + checkpoint_loader.load_weights.assert_not_called() + + +def test_gms_connect_failure_raises(monkeypatch): + events = [] + loader = _make_loader(monkeypatch, events=events) + + backend = MagicMock() + backend.connect.return_value = False + monkeypatch.setattr(memory_mod, "GMSBackend", MagicMock(return_value=backend)) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + + with pytest.raises(RuntimeError, match="Failed to connect to GMS"): + loader.load("/ckpt", checkpoint_loader) + + +def test_gms_unexpected_lock_state_raises(monkeypatch): + """``is_rw`` must be True or False after a successful connect. + + A None value indicates an adapter bug or protocol violation; the + loader should fail loudly rather than silently take the RO path. + """ + events = [] + loader = _make_loader(monkeypatch, events=events) + + backend = MagicMock() + backend.connect.return_value = True + backend.is_rw = None # Adapter bug: connected but lock state unset. + monkeypatch.setattr(memory_mod, "GMSBackend", MagicMock(return_value=backend)) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + + with pytest.raises(RuntimeError, match="lock state is unset"): + loader.load("/ckpt", checkpoint_loader) + + # Neither branch should have run; nothing was written to the model. + backend.mem_pool_scope.assert_not_called() + backend.finalize_write.assert_not_called() + backend.materialize_module.assert_not_called() + checkpoint_loader.load_weights.assert_not_called() diff --git a/tests/unittest/api_stability/references/llm.yaml b/tests/unittest/api_stability/references/llm.yaml index c59434386a1a..285a7692fdab 100644 --- a/tests/unittest/api_stability/references/llm.yaml +++ b/tests/unittest/api_stability/references/llm.yaml @@ -103,6 +103,10 @@ methods: annotation: tensorrt_llm.llmapi.llm_args.ModelExpressConfig default: null status: prototype + gms_config: + annotation: tensorrt_llm.llmapi.llm_args.GmsConfig + default: null + status: prototype mm_encoder_only: annotation: bool default: False diff --git a/tests/unittest/llmapi/test_gms_args.py b/tests/unittest/llmapi/test_gms_args.py new file mode 100644 index 000000000000..cc51955d270e --- /dev/null +++ b/tests/unittest/llmapi/test_gms_args.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Unit tests for GMS prototype config fields on ``TorchLlmArgs``.""" + +from unittest.mock import patch + +import pytest + +from tensorrt_llm.llmapi.llm_args import LoadFormat, TorchLlmArgs + +_LOGGER_PATH = "tensorrt_llm.llmapi.llm_args.logger" +_DUMMY_MODEL = "/tmp/test-gms-args-nonexistent" + + +def _capture_warnings(call_args_list): + """Flatten captured ``logger.warning(fmt, *args)`` calls into strings.""" + rendered = [] + for call in call_args_list: + args, _kwargs = call + if not args: + continue + fmt, *fmt_args = args + try: + rendered.append(fmt % tuple(fmt_args) if fmt_args else fmt) + except (TypeError, ValueError): + rendered.append(str(args)) + return rendered + + +def _warnings_for(keyword: str, call_args_list) -> list[str]: + """Return only captured warnings whose rendered text mentions keyword.""" + return [m for m in _capture_warnings(call_args_list) if keyword in m] + + +def _make_args(**overrides) -> TorchLlmArgs: + """Construct a ``TorchLlmArgs`` with the given overrides.""" + return TorchLlmArgs(model=_DUMMY_MODEL, **overrides) + + +class TestGmsConfigDefaults: + def test_defaults(self): + args = _make_args() + assert args.gms_config.socket_path is None + assert args.gms_config.mode == "auto" + assert args.gms_config.tag == "weights" + + +class TestGmsConfigValidation: + @pytest.mark.parametrize("mode", ["auto", "rw", "ro"], ids=["auto", "rw", "ro"]) + def test_accepts_known_modes(self, mode): + args = _make_args(load_format=LoadFormat.GMS, gms_config={"mode": mode}) + assert args.gms_config.mode == mode + + @pytest.mark.parametrize( + "bad_mode", + ["", "AUTO", "Rw", "read", "write", "shadow"], + ids=[ + "empty", + "uppercase-auto", + "mixedcase-rw", + "read", + "write", + "shadow", + ], + ) + def test_rejects_unknown_mode(self, bad_mode): + # GmsConfig.mode is typed as Literal["auto", "rw", "ro"], so Pydantic + # rejects bad values at field validation with its stable literal_error + # template. Match on the rendered allowed-values list rather than the + # field path so the assertion is robust across Pydantic 2.x error + # path-formatting differences (dot vs arrow). + with pytest.raises(ValueError, match=r"Input should be 'auto', 'rw' or 'ro'"): + _make_args(gms_config={"mode": bad_mode}) + + @pytest.mark.parametrize( + "bad_tag", + ["", " ", "\t", "\n", " \t "], + ids=["empty", "space", "tab", "newline", "mixed-whitespace"], + ) + def test_rejects_empty_or_whitespace_tag(self, bad_tag): + with pytest.raises(ValueError, match="gms_config.tag must be a non-empty"): + _make_args(gms_config={"tag": bad_tag}) + + def test_accepts_custom_non_empty_tag(self): + args = _make_args(load_format=LoadFormat.GMS, gms_config={"tag": "custom"}) + assert args.gms_config.tag == "custom" + + +class TestGmsCrossFieldWarning: + @pytest.mark.parametrize( + "load_format, gms_config, expect_warning", + [ + pytest.param( + LoadFormat.GMS, {"socket_path": "/tmp/gms.sock"}, False, id="gms-active-no-warning" + ), + pytest.param( + LoadFormat.AUTO, + {"socket_path": "/tmp/gms.sock"}, + True, + id="socket-path-without-gms", + ), + pytest.param(LoadFormat.AUTO, {"mode": "rw"}, True, id="mode-without-gms"), + pytest.param(LoadFormat.AUTO, {"tag": "custom"}, True, id="tag-without-gms"), + ], + ) + def test_non_default_config_warns_when_gms_inactive( + self, load_format, gms_config, expect_warning + ): + with patch(_LOGGER_PATH) as mock_logger: + args = _make_args(load_format=load_format, gms_config=gms_config) + + for key, value in gms_config.items(): + assert getattr(args.gms_config, key) == value + relevant = _warnings_for("gms_config", mock_logger.warning.call_args_list) + if expect_warning: + assert relevant, "expected a warning about ignored gms_config" + else: + assert relevant == [] + + +class TestLoadFormatGms: + def test_gms_enum_present(self): + assert LoadFormat.GMS.name == "GMS" + assert LoadFormat.GMS.value == 3 + + @pytest.mark.parametrize( + "raw, expected", + [ + pytest.param("GMS", LoadFormat.GMS, id="uppercase-string"), + pytest.param("gms", LoadFormat.GMS, id="lowercase-string"), + pytest.param(3, LoadFormat.GMS, id="enum-int"), + ], + ) + def test_gms_load_format_conversion(self, raw, expected): + args = _make_args(load_format=raw) + assert args.load_format == expected + + def test_gms_only_two_axis_defaults(self): + args = _make_args(load_format=LoadFormat.GMS) + assert args.checkpoint_format == "HF" + assert args.load_format == LoadFormat.GMS From 32eda52a447d20177ab7b3957a0ddd3c5b5d59d2 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 Date: Sat, 30 May 2026 14:48:55 +0800 Subject: [PATCH 141/308] [None][test] Unwaive some Perf Tests (#14664) Signed-off-by: Chenfei Zhang --- tests/integration/test_lists/waives.txt | 7 ------- ...k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 8 +++++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 015efc6e97e6..c0869905e48e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -215,7 +215,6 @@ full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161 full:B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) -full:GB200/perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6194788) full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1] SKIP (https://nvbugs/4731514) full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) @@ -267,10 +266,6 @@ perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20-gpus:2] SK perf/test_perf.py::test_perf[t5-bench-float16-maxbs:1-input_output_len:128,20-gpus:2] SKIP perf/test_perf.py::test_perf[t5_base-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20] SKIP -perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215810) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k] SKIP (https://nvbugs/6167060) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k] SKIP (https://nvbugs/6190071) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6189928) perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k] SKIP (https://nvbugs/6227472) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236108) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6236094) @@ -281,7 +276,6 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6179661) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6016528) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6221024) @@ -289,7 +283,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6200257) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6221022) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] SKIP (https://nvbugs/6215678) stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] SKIP (https://nvbugs/6215678) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 686fb268738a..f01db9b09770 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '256' @@ -48,7 +48,7 @@ accuracy: worker_config: gen: print_iter_log: true - max_batch_size: 16 + max_batch_size: 32 max_num_tokens: 64 tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -58,7 +58,7 @@ worker_config: enable_lm_head_tp_in_adp: true cuda_graph_config: enable_padding: true - max_batch_size: 16 + max_batch_size: 32 kv_cache_config: enable_block_reuse: false free_gpu_memory_fraction: 0.8 @@ -69,6 +69,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -95,5 +96,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 From f20858cc05c514be4e38708266afba5a89395438 Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Sat, 30 May 2026 02:31:15 -0700 Subject: [PATCH 142/308] [https://nvbugs/6204488][fix] Replace fixed disagg fill throttle with slow-start ramp (#14475) Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 18 ++- .../_torch/executor/test_benchmark_disagg.py | 147 +++++++++++++++--- 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index b5cc85b2d059..deabcb756e1c 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -623,6 +623,10 @@ def on_detected(): # normal dummy add-forward-terminate lifecycle handles taper-down. # Only relevant in benchmark disagg mode; False otherwise. self._benchmark_fill_phase_active = self.is_benchmark_disagg + # Slow-start admission cap for benchmark disagg fill (see + # _pop_from_waiting_queue). 0 = uninitialised; first throttled iter + # seeds it to tp_size and each subsequent iter doubles it. + self._fill_admit_cap: int = 0 # Initialize disagg PP termination handler if needed self._disagg_pp_termination_handler = None @@ -2616,6 +2620,7 @@ def _check_benchmark_disagg_gate(self, scheduled_batch: ScheduledRequests, scheduled_batch) if can_forward: self._benchmark_fill_phase_active = False + self._fill_admit_cap = 0 else: time.sleep(0.1) return can_forward, True @@ -3352,15 +3357,14 @@ def _pop_from_waiting_queue( max_new_requests = total_max - total_num_active_requests - # Benchmark disagg fill-phase admission throttle: a preloaded queue - # (concurrency == total_max) would otherwise pop all requests into - # DISAGG_GENERATION_INIT in one iter, tripping PR #12206's fail-fast - # under ADP-router imbalance and growing the recv-buffer fallback - # faster than transfers drain. Cap at `tp_size` (≈ pre-#12208 - # blocking fill rate). Verified on Kimi-K2 8k1k ctx8/gen1 con=8192. + # Benchmark disagg fill-phase admission throttle (slow-start ramp). if (self.is_benchmark_disagg and self._benchmark_fill_phase_active and not self.is_warmup): - max_new_requests = min(max_new_requests, self.dist.tp_size) + if self._fill_admit_cap == 0: + self._fill_admit_cap = self.dist.tp_size + else: + self._fill_admit_cap = min(self._fill_admit_cap * 2, total_max) + max_new_requests = min(max_new_requests, self._fill_admit_cap) return get_from_waiting_queue( waiting_queue, diff --git a/tests/unittest/_torch/executor/test_benchmark_disagg.py b/tests/unittest/_torch/executor/test_benchmark_disagg.py index 76bc3cff7628..f801e8e0e162 100644 --- a/tests/unittest/_torch/executor/test_benchmark_disagg.py +++ b/tests/unittest/_torch/executor/test_benchmark_disagg.py @@ -17,11 +17,11 @@ In benchmark disagg mode the GEN executor must defer the forward pass until all benchmark requests have completed KV transfer. These tests cover: -- State-based ``_is_benchmark_disagg_fill_complete`` predicate -- ``can_forward`` gating initialisation and transitions +- State-based `_is_benchmark_disagg_fill_complete` predicate +- `can_forward` gating initialisation and transitions - ADP dummy suppression during fill vs taper-down - ADP router imbalance regression (nvbug 6071070) -- Non-blocking behaviour of ``_prepare_and_schedule_batch`` +- Non-blocking behaviour of `_prepare_and_schedule_batch` """ from unittest.mock import Mock, patch @@ -56,8 +56,8 @@ def _make_transceiver(transfer_complete: bool = True) -> Mock: class MockBenchmarkExecutor: """Minimal stub mirroring the PyExecutor attributes used by - ``_is_benchmark_disagg_fill_complete``, ``_check_benchmark_disagg_gate``, - and the ``can_forward`` gate. + `_is_benchmark_disagg_fill_complete`, `_check_benchmark_disagg_gate`, + and the `can_forward` gate. Binds the real production methods so tests exercise actual logic without needing a fully-initialised executor. @@ -80,6 +80,7 @@ def __init__( benchmark_req_queues_size > 0 and kv_cache_transceiver is not None ) self._benchmark_fill_phase_active = self.is_benchmark_disagg + self._fill_admit_cap = 0 self.enable_attention_dp = enable_attention_dp self.num_fetch_requests = num_fetch_requests self.is_warmup = is_warmup @@ -361,7 +362,7 @@ def test_gate_no_op(self, mock_time, is_warmup, can_forward_in): class MockPadDummyExecutor: """Stub mirroring the PyExecutor attributes used by - ``_pad_attention_dp_dummy_request``. + `_pad_attention_dp_dummy_request`. """ def __init__( @@ -487,7 +488,7 @@ def test_skips_when_adp_disabled(self): class TestPrepareAndScheduleBatchNoBlock: """_prepare_and_schedule_batch must not block on request fetching. - NOTE: This test uses ``object.__new__(PyExecutor)`` to bypass __init__ + NOTE: This test uses `object.__new__(PyExecutor)` to bypass __init__ and manually sets internal attributes. Keep the attribute list in sync with the method's implementation. """ @@ -500,6 +501,7 @@ def test_fetch_called_once_even_in_benchmark_disagg(self): ex.kv_cache_transceiver = Mock() ex.is_benchmark_disagg = True ex._benchmark_fill_phase_active = True + ex._fill_admit_cap = 0 ex.enable_attention_dp = False ex.num_fetch_requests = 0 ex.dist = Mock(rank=0, tp_size=1) @@ -535,16 +537,7 @@ def test_fetch_called_once_even_in_benchmark_disagg(self): class TestBenchmarkFillAdmissionFlowControl: - """Verify benchmark disagg fill admits requests gradually. - - The failing wide-EP Kimi case has ``benchmark_req_queues_size`` equal to - ``tp_size * max_batch_size``. Without an explicit fill-phase cap, GEN can - admit the entire benchmark queue in one iteration, prepare too many KV - receives before the first forward pass, and hit process-level memory - pressure. The desired invariant is non-blocking flow control: each - executor iteration should still return to the outer loop, but it should - only admit a small bounded number of new requests during the fill phase. - """ + """Verify benchmark disagg fill admits requests via a slow-start ramp.""" @staticmethod def _make_executor(tp_size: int = 4, fill_phase_active: bool = True): @@ -554,6 +547,7 @@ def _make_executor(tp_size: int = 4, fill_phase_active: bool = True): ex.enable_attention_dp = True ex.is_benchmark_disagg = True ex._benchmark_fill_phase_active = fill_phase_active + ex._fill_admit_cap = 0 ex.benchmark_req_queues_size = 32 ex.num_fetch_requests = 0 ex.max_num_active_requests = 8 @@ -562,9 +556,23 @@ def _make_executor(tp_size: int = 4, fill_phase_active: bool = True): ex.dist.tp_size = tp_size return ex + @staticmethod + def _expected_ramp(tp_size: int, total_max: int) -> list: + """Build the expected admission-cap sequence: tp_size, 2*tp_size, ..., + clamped to total_max, then steady at total_max.""" + seq = [] + cap = tp_size + while cap < total_max: + seq.append(cap) + cap = min(cap * 2, total_max) + seq.append(total_max) + return seq + @pytest.mark.parametrize("tp_size", [1, 4, 32]) @patch("tensorrt_llm._torch.pyexecutor.py_executor.get_from_waiting_queue") - def test_fill_phase_caps_admission_to_tp_size(self, mock_get_from_waiting_queue, tp_size): + def test_first_iter_caps_at_tp_size(self, mock_get_from_waiting_queue, tp_size): + """Iter 0 of the ramp must still cap at tp_size — this is the + load-bearing iter-0 burst protection (PR #12206 fail-fast).""" ex = self._make_executor(tp_size=tp_size) waiting_queue = Mock() all_ranks_num_active_requests = [0] * tp_size @@ -578,12 +586,45 @@ def test_fill_phase_caps_admission_to_tp_size(self, mock_get_from_waiting_queue, mock_get_from_waiting_queue.assert_called_once() _, max_new_requests = mock_get_from_waiting_queue.call_args.args[:2] - assert max_new_requests == ex.dist.tp_size, ( - "Benchmark disagg fill should admit at most tp_size requests per " - "executor iteration. Using full global capacity would admit " - "tp_size * max_batch_size requests at once and recreate the " - "fill-phase memory-pressure failure." + assert max_new_requests == tp_size, ( + "Iter 0 of the benchmark-fill admission ramp must cap at tp_size " + "to protect against the simultaneous-DISAGG_GENERATION_INIT burst " + "that trips PR #12206's fail-fast under ADP router imbalance." + ) + assert ex._fill_admit_cap == tp_size + + @pytest.mark.parametrize("tp_size", [1, 4, 8, 32]) + @patch("tensorrt_llm._torch.pyexecutor.py_executor.get_from_waiting_queue") + def test_ramp_doubles_each_iter_and_saturates_at_total_max( + self, mock_get_from_waiting_queue, tp_size + ): + """Each subsequent fill-phase iter doubles the admission cap, and the + cap saturates at `total_max` (never exceeds the global capacity).""" + ex = self._make_executor(tp_size=tp_size) + total_max = ex.dist.tp_size * ex.max_num_active_requests + expected = self._expected_ramp(tp_size, total_max) + + observed = [] + # Walk a few iters past saturation to confirm the cap stays put. + for _ in range(len(expected) + 3): + ex._pop_from_waiting_queue( + Mock(), + total_num_active_requests=0, + all_ranks_num_active_requests=[0] * tp_size, + ) + _, max_new_requests = mock_get_from_waiting_queue.call_args.args[:2] + observed.append(max_new_requests) + + # The ramp matches tp_size, 2*tp_size, ..., total_max. + assert observed[: len(expected)] == expected, ( + f"ramp mismatch: expected {expected}, observed {observed}" ) + # Cap stays at total_max thereafter. + assert all(c == total_max for c in observed[len(expected) - 1 :]) + # Convergence is logarithmic in total_max / tp_size. + # (Allow +1 for the saturating final entry.) + max_iters = (total_max // tp_size).bit_length() + 1 + assert len(expected) <= max_iters @patch("tensorrt_llm._torch.pyexecutor.py_executor.get_from_waiting_queue") def test_no_fill_phase_uses_full_available_capacity(self, mock_get_from_waiting_queue): @@ -600,6 +641,62 @@ def test_no_fill_phase_uses_full_available_capacity(self, mock_get_from_waiting_ _, max_new_requests = mock_get_from_waiting_queue.call_args.args[:2] assert max_new_requests == ex.dist.tp_size * ex.max_num_active_requests + assert ex._fill_admit_cap == 0, ( + "When fill phase is inactive, the slow-start cap must remain " + "untouched so a future fill phase starts the ramp from tp_size." + ) + + @patch("tensorrt_llm._torch.pyexecutor.py_executor.get_from_waiting_queue") + def test_ramp_state_resets_when_fill_phase_ends(self, mock_get_from_waiting_queue): + """After the fill gate opens (`_fill_admit_cap` reset to 0 by + `_check_benchmark_disagg_gate`), a *subsequent* fill phase starts + the ramp from tp_size again rather than wherever the previous ramp + ended. Matters for back-to-back benchmarks (e.g. after warmup).""" + ex = self._make_executor(tp_size=4) + + # Walk a couple of iters into the ramp. + for _ in range(3): + ex._pop_from_waiting_queue( + Mock(), + total_num_active_requests=0, + all_ranks_num_active_requests=[0] * 4, + ) + assert ex._fill_admit_cap > 4 + + # Simulate the gate-open path that `_check_benchmark_disagg_gate` + # takes when fill completes. + ex._benchmark_fill_phase_active = False + ex._fill_admit_cap = 0 + + # New fill phase starts; the next throttled iter must reseed at tp_size. + ex._benchmark_fill_phase_active = True + ex._pop_from_waiting_queue( + Mock(), + total_num_active_requests=0, + all_ranks_num_active_requests=[0] * 4, + ) + _, max_new_requests = mock_get_from_waiting_queue.call_args.args[:2] + assert max_new_requests == 4 + assert ex._fill_admit_cap == 4 + + @patch("tensorrt_llm._torch.pyexecutor.py_executor.get_from_waiting_queue") + def test_warmup_skips_throttle(self, mock_get_from_waiting_queue): + """Warmup must bypass the slow-start ramp so warmup iters are not + artificially throttled and `_fill_admit_cap` stays unchanged.""" + ex = self._make_executor(tp_size=4) + # Bypass the is_warmup property setter (which also pokes model_engine, + # not present on this stub). + ex._is_warmup = True + + ex._pop_from_waiting_queue( + Mock(), + total_num_active_requests=0, + all_ranks_num_active_requests=[0] * 4, + ) + + _, max_new_requests = mock_get_from_waiting_queue.call_args.args[:2] + assert max_new_requests == ex.dist.tp_size * ex.max_num_active_requests + assert ex._fill_admit_cap == 0 # --------------------------------------------------------------------------- @@ -774,7 +871,7 @@ class TestFailFastDuringBenchmarkFill: This covers the CI regression where the fill-phase guard suppressed the fail-fast forever and - ``test_disaggregated_benchmark_gen_only_insufficient_kv`` timed out. + `test_disaggregated_benchmark_gen_only_insufficient_kv` timed out. """ def _make_executor( @@ -793,6 +890,7 @@ def _make_executor( ex.kv_cache_transceiver = Mock() ex.is_benchmark_disagg = True ex._benchmark_fill_phase_active = fill_phase_active + ex._fill_admit_cap = 0 ex.enable_attention_dp = False ex.num_fetch_requests = num_fetch_requests ex.dist = Mock(rank=0, tp_size=1) @@ -937,6 +1035,7 @@ def _make_executor(self): ex.kv_cache_transceiver = _make_transceiver(transfer_complete=False) ex.is_benchmark_disagg = True ex._benchmark_fill_phase_active = True + ex._fill_admit_cap = 0 ex.enable_attention_dp = True ex.num_fetch_requests = 0 ex.max_num_active_requests = self.MAX_BATCH_SIZE From 255a54bb8ba60fcd517e060aba8296bac49f2833 Mon Sep 17 00:00:00 2001 From: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> Date: Sat, 30 May 2026 13:36:06 -0700 Subject: [PATCH 143/308] [None][chore] Waive failing multi-gpu test (#14788) Signed-off-by: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c0869905e48e..1ffe62ada48a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -15,6 +15,7 @@ accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbu accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True] SKIP (https://nvbugs/6240561) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] SKIP (https://nvbugs/6245279) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) From 5f106dfabd768489304223caec99829604cdf905 Mon Sep 17 00:00:00 2001 From: belgarten-nv Date: Sat, 30 May 2026 14:08:07 -0700 Subject: [PATCH 144/308] [TRTLLM-11408][feat] Add VisualGen TP Support (#13614) Signed-off-by: Brenden Elgarten Signed-off-by: belgarten-nv --- docs/source/models/visual-generation.md | 19 +- examples/visual_gen/README.md | 45 ++ examples/visual_gen/visual_gen_flux.py | 16 +- examples/visual_gen/visual_gen_ltx2.py | 19 +- examples/visual_gen/visual_gen_wan_i2v.py | 17 +- examples/visual_gen/visual_gen_wan_t2v.py | 7 + tensorrt_llm/_torch/device_mesh.py | 1 + tensorrt_llm/_torch/visual_gen/config.py | 2 +- tensorrt_llm/_torch/visual_gen/executor.py | 2 +- tensorrt_llm/_torch/visual_gen/mapping.py | 49 ++ .../visual_gen/models/flux/attention.py | 98 ++- .../visual_gen/models/flux/joint_proj.py | 303 ++++++++ .../models/flux/transformer_flux.py | 48 +- .../models/flux/transformer_flux2.py | 26 +- .../models/ltx2/transformer_ltx2.py | 3 + .../visual_gen/models/wan/transformer_wan.py | 43 +- .../_torch/visual_gen/modules/attention.py | 103 ++- .../_torch/visual_gen/modules/rms_norm.py | 105 +++ .../_torch/visual_gen/pipeline_loader.py | 1 + tensorrt_llm/visual_gen/args.py | 8 +- .../visual_gen/multi_gpu/test_flux_tp.py | 686 ++++++++++++++++++ .../visual_gen/multi_gpu/test_tp_attention.py | 522 +++++++++++++ .../visual_gen/multi_gpu/test_wan_tp.py | 594 +++++++++++++++ 23 files changed, 2616 insertions(+), 101 deletions(-) create mode 100644 tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py create mode 100644 tensorrt_llm/_torch/visual_gen/modules/rms_norm.py create mode 100644 tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py create mode 100644 tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py create mode 100644 tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py diff --git a/docs/source/models/visual-generation.md b/docs/source/models/visual-generation.md index c1260f91de22..f528bfc7cc32 100644 --- a/docs/source/models/visual-generation.md +++ b/docs/source/models/visual-generation.md @@ -15,7 +15,7 @@ TensorRT-LLM **VisualGen** provides a unified inference stack for diffusion mode - Pluggable attention backends: PyTorch SDPA (`VANILLA`), TRT-LLM kernels (`TRTLLM`), TRT-LLM CuTe DSL kernels (`CUTEDSL`, Blackwell-class GPUs), and Flash Attention 4 (`FA4`). - Quantization support (dynamic and static) using the [ModelOpt](https://github.com/NVIDIA/TensorRT-Model-Optimizer) configuration format. - Quantized attention support: `QK16PV8` to quantize Bmm2 on `CUTEDSL`, `SAGE` to run SageAttention on `TRTLLM` (requires Blackwell SM100). -- Multi-GPU parallelism (CFG parallel, Ulysses sequence parallel). +- Multi-GPU parallelism (CFG parallel, Ulysses sequence parallel, Tensor parallelism). - **TeaCache** — a runtime caching optimization that skips transformer steps when timestep embeddings change slowly. - `trtllm-serve` integration with OpenAI-compatible API endpoints for image and video generation. @@ -38,13 +38,13 @@ Models are auto-detected from the checkpoint directory. Diffusers-format models ### Feature Matrix -| Model | FP8 blockwise | NVFP4 | TeaCache | CFG Parallelism | Ulysses Parallelism | Parallel VAE | CUDA Graph | torch.compile | trtllm-serve | Attention2D | Ring Attention | -|---|---|---|---|---|---|---|---|---|---|--|--| -| **FLUX.1** | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | -| **FLUX.2** | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | -| **Wan 2.1** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | -| **Wan 2.2** | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | -| **LTX-2** | Yes | Yes | No | Yes | Yes | No | No | Yes | Yes | Yes | Yes | +| Model | FP8 blockwise | NVFP4 | TeaCache | CFG Parallelism | Ulysses Parallelism | Parallel VAE | CUDA Graph | torch.compile | trtllm-serve | Attention2D | Ring Attention | Tensor Parallelism | +|---|---|---|---|---|---|---|---|---|---|--|--|--| +| **FLUX.1** | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| **FLUX.2** | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| **Wan 2.1** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +| **Wan 2.2** | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +| **LTX-2** | Yes | Yes | No | Yes | Yes | No | No | Yes | Yes | Yes | Yes | No | [^1]: FLUX models use embedded guidance and do not have a separate negative prompt path, so CFG parallelism is not applicable. @@ -168,7 +168,7 @@ The `teacache_thresh` parameter controls the similarity threshold. Cache-DiT is ### Multi-GPU Parallelism -5 parallelism modes can be combined: +6 parallelism modes can be combined: - **CFG Parallelism** (`--cfg_size 2`): Splits positive/negative guidance prompts across GPUs. - **Ulysses Parallelism** (`--ulysses_size N`): Splits the sequence dimension across GPUs for longer sequences. @@ -176,6 +176,7 @@ The `teacache_thresh` parameter controls the similarity threshold. Cache-DiT is - **Attention Parallel**: There are 2 methods supported to run attention parallel. Both of these methods require the attention backend to support LSE (`FA4` and `CUTEDSL`) - - **Attention2D Parallelism** (`--attn2d_row_size N`, `--attn2d_col_size M`): Shards the sequence axis across a 2D `N x M` device mesh, all-gathering Q along rows and K/V along columns so each rank computes a sub-block of the attention matrix (total CP degree = `N * M`; not currently combinable with Ulysses). - **Ring Attention Parallelism** (`--ring_size N`): Shards the sequence axis across a 1D ring of `N` ranks and streams K/V blocks around the ring so each rank computes its attention output without materializing the full K/V (mutually exclusive with Attention2D). +- **Tensor Parallelism** (`--tp_size N`): Splits attention heads and transformer MLPs across GPUs for faster compute and reduced memory usage. ## Developer Guide ### Architecture Overview diff --git a/examples/visual_gen/README.md b/examples/visual_gen/README.md index b9a27d3e5fbf..c08cd3dd871c 100644 --- a/examples/visual_gen/README.md +++ b/examples/visual_gen/README.md @@ -116,6 +116,9 @@ WAN supports two parallelism modes that can be combined: - *Ulysses*: Split sequence along head dimension across GPUs; requires `ulysses_size` to divide the model's head count - *Attention2D*: 2D mesh sequence parallelism; no head-count constraint; requires `--attention_backend FA4` - Combining Ulysses and Attention2D is not yet supported +- **Tensor Parallelism** + - Splits attention heads and transformer MLPs across GPUs; requires `tp_size` to divide the model's head count and MLP up dimension. + - Can be combined with Ulysses and CFG **Ulysses Only (2 GPUs):** @@ -155,6 +158,18 @@ python visual_gen_wan_t2v.py \ ``` GPU Layout: Sequence equally split among GPU 0-3 +**TP Only (2 GPUs):** +```bash +python visual_gen_wan_t2v.py \ + --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "A cute cat playing piano" \ + --height 480 --width 832 --num_frames 33 \ + --attention_backend FA4 \ + --cfg_size 1 --tp_size 2\ + --output_path output.mp4 +``` +GPU Layout: Attention heads and MLP projections equally split among GPU 0-1 + **CFG + Ulysses (4 GPUs):** ```bash python visual_gen_wan_t2v.py \ @@ -180,6 +195,31 @@ python visual_gen_wan_t2v.py \ ``` GPU Layout: GPU 0-3 (positive, Attention2D) | GPU 4-7 (negative, Attention2D) +**CFG + TP (4 GPUs):** +```bash +python visual_gen_wan_t2v.py \ + --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "A cute cat playing piano" \ + --height 480 --width 832 --num_frames 33 \ + --attention_backend TRTLLM \ + --cfg_size 2 --tp_size 2 \ + --output_path output.mp4 +``` +GPU Layout: GPU 0-1 (positive, TP) | GPU 2-3 (negative, TP) + +**CFG + Ulysses + TP (8 GPUs):** +```bash +python visual_gen_wan_t2v.py \ + --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --prompt "A cute cat playing piano" \ + --height 480 --width 832 --num_frames 33 \ + --attention_backend TRTLLM \ + --cfg_size 2 --ulysses_size 2 \ + --tp_size 2 \ + --output_path output.mp4 +``` +GPU Layout: GPU 0-3 (positive, Ulysses + TP) | GPU 4-7 (negative, Ulysses + TP) + **Large-Scale (64 GPUs):** ```bash python visual_gen_wan_t2v.py \ @@ -291,6 +331,7 @@ python visual_gen_ltx2.py \ | `--parallel_vae_size` | - | ✓ | — | 1 | Parallelism used for VAE | | `--attn2d_row_size` | ✓ | ✓ | ✓ | 1 | Attention2D mesh row size | | `--attn2d_col_size` | ✓ | ✓ | ✓ | 1 | Attention2D mesh column size | +| `--tp_size` | ✓ | ✓ | ✓ | 1 | Tensor parallelism | | `--linear_type` | ✓ | ✓ | — | default | Quantization type | | `--enhance_prompt` | — | ✓ | False | Gemma3 prompt enhancement | | `--stg_scale` | — | ✓ | 0.0 | Spatiotemporal guidance scale | @@ -327,6 +368,10 @@ python visual_gen_ltx2.py \ - Total GPUs = `cfg_size × attn2d_row_size × attn2d_col_size` - Sequence length must be divisible by `attn2d_row_size × attn2d_col_size` +**TP Errors:** +- `tp_size × ulysses_size` must divide the model's head count (12 for WAN) +- Total GPUs = `cfg_size × ulysses_size × tp_size` + ## Output Formats - **FLUX**: `.png` (image) diff --git a/examples/visual_gen/visual_gen_flux.py b/examples/visual_gen/visual_gen_flux.py index 00ccbb0dd8bb..f931292210b0 100755 --- a/examples/visual_gen/visual_gen_flux.py +++ b/examples/visual_gen/visual_gen_flux.py @@ -228,6 +228,12 @@ def parse_args(): help="Ulysses (head-sharding) parallel size within each CFG group. " "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor Parallel size", + ) parser.add_argument( "--attn2d_row_size", type=int, @@ -358,6 +364,7 @@ def build_visual_gen_args(args) -> VisualGenArgs: parallel_config={ "ulysses_size": args.ulysses_size, "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), + "tp_size": args.tp_size, }, torch_compile_config={ "enable": not args.disable_torch_compile, @@ -384,14 +391,17 @@ def main(): visual_gen_args = build_visual_gen_args(args) + parallel_str = "" + if args.tp_size > 1: + parallel_str = f"TP(size={args.tp_size})" if args.ulysses_size > 1: - parallel_str = f"Ulysses(size={args.ulysses_size})" + parallel_str += f"Ulysses(size={args.ulysses_size})" elif attn2d_size > 1: - parallel_str = ( + parallel_str += ( f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " f"total={attn2d_size})" ) - else: + elif not parallel_str: parallel_str = "None" logger.info(f"Initializing VisualGen: parallelism={parallel_str}") visual_gen = VisualGen( diff --git a/examples/visual_gen/visual_gen_ltx2.py b/examples/visual_gen/visual_gen_ltx2.py index 83e1e0c55374..cc1e8fe4f509 100755 --- a/examples/visual_gen/visual_gen_ltx2.py +++ b/examples/visual_gen/visual_gen_ltx2.py @@ -237,6 +237,12 @@ def parse_args(): help="Ulysses (head-sharding) parallel size within each CFG group. " "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor Parallel size", + ) parser.add_argument( "--attn2d_row_size", type=int, @@ -359,6 +365,7 @@ def _build_visual_gen_args(args) -> VisualGenArgs: "cfg_size": args.cfg_size, "ulysses_size": args.ulysses_size, "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), + "tp_size": args.tp_size, }, torch_compile_config={ "enable": not args.disable_torch_compile, @@ -383,16 +390,22 @@ def main(): "Combining --ulysses_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." ) + if args.tp_size > 1: + raise ValueError("LTX2 does not currently support TP.") + visual_gen_args = _build_visual_gen_args(args) + parallel_str = "" + if args.tp_size > 1: + parallel_str = f"TP(size={args.tp_size})" if args.ulysses_size > 1: - parallel_str = f"Ulysses(size={args.ulysses_size})" + parallel_str += f"Ulysses(size={args.ulysses_size})" elif attn2d_size > 1: - parallel_str = ( + parallel_str += ( f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " f"total={attn2d_size})" ) - else: + elif not parallel_str: parallel_str = "None" logger.info( f"Initializing VisualGen (LTX2): cfg_size={args.cfg_size}, parallelism={parallel_str}" diff --git a/examples/visual_gen/visual_gen_wan_i2v.py b/examples/visual_gen/visual_gen_wan_i2v.py index 671b02250779..3be72ed43939 100644 --- a/examples/visual_gen/visual_gen_wan_i2v.py +++ b/examples/visual_gen/visual_gen_wan_i2v.py @@ -246,6 +246,12 @@ def parse_args(): default=1, help="Ring Attention parallel size. Cannot be combined with --attn2d_row_size / --attn2d_col_size.", ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="TP group size", + ) parser.add_argument( "--parallel_vae_size", type=int, @@ -361,14 +367,18 @@ def main(): "Combining --ring_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." ) + parallel_str = "" + if args.tp_size > 1: + parallel_str = f"TP(size={args.tp_size})" + if args.ulysses_size > 1 or args.ring_size > 1: - parallel_str = f"Ulysses(size={args.ulysses_size}), Ring(size={args.ring_size})" + parallel_str += f"Ulysses(size={args.ulysses_size}), Ring(size={args.ring_size})" elif attn2d_size > 1: - parallel_str = ( + parallel_str += ( f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " f"total={attn2d_size})" ) - else: + elif not parallel_str: parallel_str = "None" kwargs = dict( @@ -379,6 +389,7 @@ def main(): "ulysses_size": args.ulysses_size, "ring_size": args.ring_size, "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), + "tp_size": args.tp_size, "parallel_vae_size": args.parallel_vae_size, }, torch_compile_config={ diff --git a/examples/visual_gen/visual_gen_wan_t2v.py b/examples/visual_gen/visual_gen_wan_t2v.py index ce4a32e766d6..af77bab8c133 100755 --- a/examples/visual_gen/visual_gen_wan_t2v.py +++ b/examples/visual_gen/visual_gen_wan_t2v.py @@ -245,6 +245,12 @@ def parse_args(): default=1, help="Ring Attention parallel size. Cannot be combined with --attn2d_row_size / --attn2d_col_size.", ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="TP group size", + ) parser.add_argument( "--parallel_vae_size", type=int, @@ -386,6 +392,7 @@ def main(): "ulysses_size": args.ulysses_size, "ring_size": args.ring_size, "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), + "tp_size": args.tp_size, "parallel_vae_size": args.parallel_vae_size, }, torch_compile_config={ diff --git a/tensorrt_llm/_torch/device_mesh.py b/tensorrt_llm/_torch/device_mesh.py index bdc9d94f8f55..be7ca9366e7b 100644 --- a/tensorrt_llm/_torch/device_mesh.py +++ b/tensorrt_llm/_torch/device_mesh.py @@ -146,6 +146,7 @@ def build_mesh(self): logger.debug(f"DeviceMeshTopology.tp_mesh: {cls.tp_mesh}") @require_device_mesh + @torch.compiler.disable def _get_mesh_dim_by_name(self, name: str) -> dist.DeviceMesh: cls = DeviceMeshTopologyImpl diff --git a/tensorrt_llm/_torch/visual_gen/config.py b/tensorrt_llm/_torch/visual_gen/config.py index 3f7009bef569..34c7b6665621 100644 --- a/tensorrt_llm/_torch/visual_gen/config.py +++ b/tensorrt_llm/_torch/visual_gen/config.py @@ -107,7 +107,7 @@ class DiffusionModelConfig(BaseModel): mapping: Mapping = PydanticField(default_factory=Mapping) skip_create_weights_in_init: bool = False force_dynamic_quantization: bool = False - allreduce_strategy: AllReduceStrategy = PydanticField(default=AllReduceStrategy.AUTO) + allreduce_strategy: AllReduceStrategy = PydanticField(default=AllReduceStrategy.NCCL) extra_attrs: Dict = PydanticField(default_factory=dict) # Unified parallelism mapping (populated by setup_visual_gen_mapping) diff --git a/tensorrt_llm/_torch/visual_gen/executor.py b/tensorrt_llm/_torch/visual_gen/executor.py index 7c9459761257..8b4f2dd876d2 100644 --- a/tensorrt_llm/_torch/visual_gen/executor.py +++ b/tensorrt_llm/_torch/visual_gen/executor.py @@ -388,7 +388,7 @@ def run_diffusion_worker( torch.cuda.set_device(device_id) dist.init_process_group( - backend="nccl" if torch.cuda.is_available() else "gloo", + backend="cuda:nccl,cpu:gloo" if torch.cuda.is_available() else "gloo", init_method="env://", world_size=world_size, rank=rank, diff --git a/tensorrt_llm/_torch/visual_gen/mapping.py b/tensorrt_llm/_torch/visual_gen/mapping.py index 2253cc8af61e..a6e09ad5eacf 100644 --- a/tensorrt_llm/_torch/visual_gen/mapping.py +++ b/tensorrt_llm/_torch/visual_gen/mapping.py @@ -10,6 +10,7 @@ from __future__ import annotations +import os from typing import Optional import torch.distributed as dist @@ -142,9 +143,56 @@ def __init__( } if dist.is_initialized() and world_size > 1: + if self.tp_size > 1: + self.setup_communicators() self.build_mesh() self._build_vae_group() + def _get_host_id(self): + """Resolve node rank from whichever launcher is active. + + torchrun sets GROUP_RANK. + SLURM srun sets SLURM_NODEID. + Plain mp.Process (single-node): no env var → 0. + """ + for var in ("GROUP_RANK", "SLURM_NODEID"): + val = os.environ.get(var) + if val is not None: + logger.debug(f"[Rank {self._rank}] node id from env: {var} = {val}") + return int(val) + + logger.debug(f"[Rank {self._rank}] node id from env: {var} = {val}") + return 0 # single-node mp.Process: all ranks are co-located + + def setup_communicators(self): + host = self._get_host_id() + + all_hosts = [None for _ in range(self.world_size)] + dist.all_gather_object(all_hosts, (self._rank, host)) + + host_to_ranks = {} + for rank, host in all_hosts: + host_to_ranks.setdefault(host, []).append(rank) + + self.local_comm = None + for host in sorted(host_to_ranks): + ranks = sorted(host_to_ranks[host]) + # All global ranks from the default process group to participate in the call, + # even if some ranks are not part of the new process group being created + pg = dist.new_group(ranks=ranks, backend="cuda:nccl,cpu:gloo") + if int(self._rank) in ranks: + logger.debug( + f"[Rank {self._rank}] Done setting local comm. ip_to_ranks: {host_to_ranks}" + ) + self.local_comm = pg + + assert self.local_comm is not None + + from tensorrt_llm._utils import torch_pybind11_abi + from tensorrt_llm.bindings.internal.process_group import init_pg + + init_pg(dist.group.WORLD, self.local_comm, torch_pybind11_abi()) + # ------------------------------------------------------------------ # Mesh construction # ------------------------------------------------------------------ @@ -165,6 +213,7 @@ def build_mesh(self): return shape = tuple(self._dim_sizes[d] for d in self._dim_names) + cls.device_mesh = init_device_mesh( "cuda", mesh_shape=shape, diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py index 6652b0b0af95..93f08d662f3a 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py @@ -13,10 +13,19 @@ import torch import torch.nn.functional as F -from tensorrt_llm._torch.modules.linear import Linear, WeightMode, WeightsLoadingConfig +from tensorrt_llm._torch.modules.linear import ( + Linear, + TensorParallelMode, + WeightMode, + WeightsLoadingConfig, +) from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.modules.swiglu import swiglu from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig +from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import ( + FluxJointAttnMLPProj, + FluxJointQKVMLPProj, +) from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode, apply_rotary_emb # ============================================================================= @@ -66,7 +75,6 @@ def __init__( self.pre_only = pre_only self.added_kv_proj_dim = added_kv_proj_dim - # Delete output projection for single-stream blocks if self.pre_only: del self.to_out @@ -74,7 +82,7 @@ def __init__( if added_kv_proj_dim is not None: self.add_qkv_proj = Linear( added_kv_proj_dim, - 3 * self.q_dim, + self.q_dim + 2 * self.kv_dim, bias=self.bias, dtype=self.dtype, quant_config=self.quant_config, @@ -84,27 +92,42 @@ def __init__( weight_mode=WeightMode.FUSED_QKV_LINEAR ), fused_weight_shard_indices_mapping={ - "q": (0, self.q_dim), - "k": (self.q_dim, self.q_dim), - "v": (2 * self.q_dim, self.q_dim), + "q": (0, self.local_q_dim), + "k": (self.local_q_dim, self.local_kv_dim), + "v": (self.local_q_dim + self.local_kv_dim, self.local_kv_dim), }, + mapping=config.mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + reduce_output=False, ) + # Need not pass any mapping info since this is intra-head normalization + # Hence it is unaffected by TP which only changes cross-head work self.norm_added_q = RMSNorm( - hidden_size=head_dim, eps=eps, dtype=self.dtype, has_weights=True + hidden_size=head_dim, + eps=eps, + dtype=self.dtype, + has_weights=True, ) self.norm_added_k = RMSNorm( - hidden_size=head_dim, eps=eps, dtype=self.dtype, has_weights=True + hidden_size=head_dim, + eps=eps, + dtype=self.dtype, + has_weights=True, ) self.to_add_out = Linear( - self.q_dim, + self.kv_dim, added_kv_proj_dim, bias=self.bias, dtype=self.dtype, quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + mapping=config.mapping, + allreduce_strategy=config.allreduce_strategy, + tensor_parallel_mode=TensorParallelMode.ROW, + reduce_output=True, ) def apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: @@ -142,18 +165,18 @@ def _prepare_qkv_unfused( is_dual_stream = encoder_hidden_states is not None and self.added_kv_proj_dim is not None query, key, value = self.get_qkv(hidden_states) - query = query.view(batch_size, -1, self.num_attention_heads, self.head_dim) - key = key.view(batch_size, -1, self.num_attention_heads, self.head_dim) - value = value.view(batch_size, -1, self.num_attention_heads, self.head_dim) + query = query.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + key = key.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + value = value.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) query, key = self.apply_qk_norm(query, key) if is_dual_stream: encoder_qkv = self.add_qkv_proj(encoder_hidden_states) enc_q, enc_k, enc_v = encoder_qkv.chunk(3, dim=-1) - enc_q = enc_q.view(batch_size, -1, self.num_attention_heads, self.head_dim) - enc_k = enc_k.view(batch_size, -1, self.num_attention_heads, self.head_dim) - enc_v = enc_v.view(batch_size, -1, self.num_attention_heads, self.head_dim) + enc_q = enc_q.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + enc_k = enc_k.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + enc_v = enc_v.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) enc_q, enc_k = self.apply_qk_added_norm(enc_q, enc_k) @@ -201,7 +224,9 @@ def _prepare_qkv_fused( k_add_weight=k_add, ) - query, key, value = qkv.split([self.q_dim, self.kv_dim, self.kv_dim], dim=-1) + query, key, value = qkv.split( + [self.local_q_dim, self.local_kv_dim, self.local_kv_dim], dim=-1 + ) return query, key, value def _prepare_qkv( @@ -251,12 +276,14 @@ def forward( if not self.pre_only: hidden_states = self.to_out[0](hidden_states) + encoder_hidden_states_out = self.to_add_out(encoder_hidden_states_out) return hidden_states, encoder_hidden_states_out else: if not self.pre_only: hidden_states = self.to_out[0](hidden_states) + return hidden_states @@ -287,8 +314,12 @@ def __init__( config: Optional[DiffusionModelConfig] = None, layer_idx: int = 0, ): + # self.tp_size is set in super().__init__ + tp_size = config.mapping.tp_size if config and config.mapping else 1 + # Set MLP dims BEFORE super().__init__() — _init_qkv_proj() needs them self.mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.local_mlp_hidden_dim = self.mlp_hidden_dim // tp_size self.mlp_mult_factor = 2 # SwiGLU doubles input super().__init__( @@ -303,29 +334,33 @@ def __init__( layer_idx=layer_idx, ) - # Combined output: [q_dim + mlp_hidden_dim] -> [hidden_size] - self.to_out = Linear( - self.q_dim + self.mlp_hidden_dim, - hidden_size, + # Output projection needs FULL dims (ROW parallel divides internally) + self.to_out = FluxJointAttnMLPProj( + attn_dim=self.q_dim, + mlp_dim=self.mlp_hidden_dim, + out_dim=hidden_size, bias=bias, dtype=self.dtype, quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + config=config, ) def _init_qkv_proj(self): """Override: fused QKV+MLP projection instead of standard QKV.""" - qkv_dim = 3 * self.q_dim mlp_in_dim = self.mlp_hidden_dim * self.mlp_mult_factor - self.to_qkv_mlp_proj = Linear( - self.hidden_size, - qkv_dim + mlp_in_dim, + self.to_qkv_mlp_proj = FluxJointQKVMLPProj( + in_dim=self.hidden_size, + q_dim=self.q_dim, + kv_dim=self.kv_dim, + mlp_dim=mlp_in_dim, bias=self.bias, dtype=self.dtype, quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + mapping=self.mapping, ) def _apply_norm_rope_unfused( @@ -336,9 +371,9 @@ def _apply_norm_rope_unfused( """Apply separate norm + RoPE to packed QKV (unfused path).""" batch_size = qkv.shape[0] q, k, v = qkv.chunk(3, dim=-1) - q = q.view(batch_size, -1, self.num_attention_heads, self.head_dim) - k = k.view(batch_size, -1, self.num_attention_heads, self.head_dim) - v = v.view(batch_size, -1, self.num_attention_heads, self.head_dim) + q = q.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + k = k.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + v = v.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) q, k = self.apply_qk_norm(q, k) @@ -364,7 +399,7 @@ def _apply_norm_rope_fused( self.apply_packed_qk_norm_rope(qkv, freqs_cos, freqs_sin, num_txt_tokens=-1) - q, k, v = qkv.split([self.q_dim, self.kv_dim, self.kv_dim], dim=-1) + q, k, v = qkv.split([self.local_q_dim, self.local_kv_dim, self.local_kv_dim], dim=-1) return q, k, v def _apply_norm_rope( @@ -393,10 +428,7 @@ def forward( hidden_states [batch, seq, dim] """ # Fused QKV + MLP projection - proj_out = self.to_qkv_mlp_proj(hidden_states) - qkv, mlp_hidden = torch.split( - proj_out, [3 * self.q_dim, self.mlp_hidden_dim * self.mlp_mult_factor], dim=-1 - ) + qkv, mlp_hidden = self.to_qkv_mlp_proj(hidden_states) q, k, v = self._apply_norm_rope(qkv, image_rotary_emb) @@ -408,4 +440,4 @@ def forward( mlp_out = swiglu(mlp_hidden.reshape(-1, shape[-1])).reshape(*shape[:-1], -1) # Concatenate + project - return self.to_out(torch.cat([attn_out, mlp_out], dim=-1)) + return self.to_out(attn_out, mlp_out) diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py new file mode 100644 index 000000000000..edb908c2d80a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn + +from tensorrt_llm._torch.distributed.ops import AllReduce +from tensorrt_llm._torch.modules.linear import ( + Linear, + TensorParallelMode, + WeightMode, + WeightsLoadingConfig, +) +from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig +from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader +from tensorrt_llm.mapping import Mapping + + +class FluxJointAttnMLPProj(nn.Module): + """Output projection that accepts split attn + mlp inputs. + + At TP=1, equivalent to a single Linear over cat([attn, mlp]). + At TP>1, splits into two ROW-parallel Linears (one per input) so the + preceding attention and MLP branches can stay sharded — avoiding an + allgather before the projection. Bias is added after the allreduce. + + Named ``proj_out`` on the parent block so that ``filter_weights`` finds + the checkpoint keys ``proj_out.weight`` / ``proj_out.bias`` automatically. + """ + + def __init__( + self, + attn_dim: int, + mlp_dim: int, + out_dim: int, + bias: bool = True, + dtype: torch.dtype = None, + quant_config=None, + skip_create_weights_in_init: bool = False, + force_dynamic_quantization: bool = False, + config: Optional[DiffusionModelConfig] = None, + ): + super().__init__() + mapping = config.mapping if config else None + self.tp_size = getattr(mapping, "tp_size", 1) + self.tp_rank = getattr(mapping, "tp_rank", 0) + self.attn_dim = attn_dim + self.has_bias = bias + + if self.tp_size == 1: + self.proj = Linear( + attn_dim + mlp_dim, + out_dim, + bias=bias, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + reduce_output=False, + ) + else: + self.attn_proj = Linear( + attn_dim, + out_dim, + bias=False, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + mapping=config.mapping, + tensor_parallel_mode=TensorParallelMode.ROW, + reduce_output=False, + ) + self.mlp_proj = Linear( + mlp_dim, + out_dim, + bias=False, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + mapping=mapping, + tensor_parallel_mode=TensorParallelMode.ROW, + reduce_output=False, + ) + self.allreduce = AllReduce(mapping, strategy=config.allreduce_strategy) + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim, dtype=dtype)) + + def forward(self, attn_out: torch.Tensor, mlp_out: torch.Tensor) -> torch.Tensor: + if self.tp_size <= 1: + return self.proj(torch.cat([attn_out, mlp_out], dim=-1)) + out = self.allreduce(self.attn_proj(attn_out) + self.mlp_proj(mlp_out)) + if self.has_bias: + out = out + self.bias + return out + + def load_weights(self, weight_dict: Dict[str, torch.Tensor], loader: DynamicLinearWeightLoader): + """Load from checkpoint proj_out.weight / proj_out.bias. + + At TP=1, passes through to the inner Linear. + At TP>1, splits the weight along the input dim (columns) into + attn and mlp portions, then loads each as a vanilla ROW Linear. + """ + for sub in self.modules(): + if callable(getattr(sub, "create_weights", None)): + sub.create_weights() + + if self.tp_size == 1: + loader.load_linear_weights(self.proj, "proj_out", [weight_dict]) + else: + W = weight_dict["weight"] # [out_dim, attn_dim + mlp_dim] + W_attn = W[:, : self.attn_dim] + W_mlp = W[:, self.attn_dim :] + + for sub_module, sub_weight in [(self.attn_proj, W_attn), (self.mlp_proj, W_mlp)]: + loader.load_linear_weights(sub_module, "proj_out", [{"weight": sub_weight}]) + + if self.has_bias and "bias" in weight_dict: + self.bias.data.copy_(weight_dict["bias"].to(self.bias.dtype)) + + +class FluxJointQKVMLPProj(nn.Module): + """Input projection producing QKV + MLP gate/up from a single input. + + At TP=1: single Linear, output split into [qkv, mlp_gate_up]. + At TP>1: two column-parallel Linears (qkv_proj + mlp_proj) so each + can shard independently — QKV shards by heads, MLP shards by + intermediate dim. + + HF checkpoint stores a single fused weight ``to_qkv_mlp_proj.weight`` + with layout [Q | K | V | gate | up] along the output dimension. + + All dimension arguments are FULL (pre-TP). Column-parallel Linears + handle the TP sharding internally. The ``forward`` method returns + tensors with local (post-TP) sizes. + """ + + def __init__( + self, + in_dim: int, + q_dim: int, + kv_dim: int, + mlp_dim: int, + bias: bool = False, + dtype: torch.dtype = None, + quant_config=None, + skip_create_weights_in_init: bool = False, + force_dynamic_quantization: bool = False, + mapping: Optional[Mapping] = None, + ): + super().__init__() + + self.tp_size = mapping.tp_size if mapping else 1 + + # Store full (pre-TP) dims for weight loading (splitting checkpoint weight) + self.full_q_dim = q_dim + self.full_kv_dim = kv_dim + self.full_qkv_dim = q_dim + 2 * kv_dim + self.full_mlp_dim = mlp_dim + self.mlp_hidden_dim = mlp_dim // 2 # single gate or up dim + + if self.tp_size == 1: + self.proj = Linear( + in_dim, + q_dim + 2 * kv_dim + mlp_dim, + bias=bias, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + reduce_output=False, + ) + self.local_qkv_dim = q_dim + 2 * kv_dim + self.local_mlp_dim = mlp_dim + else: + local_q_dim = q_dim // self.tp_size + local_kv_dim = kv_dim // self.tp_size + shard_mlp_hidden_dim = self.mlp_hidden_dim // self.tp_size + # QKV: column-parallel with fused Q/K/V sharding + self.qkv_proj = Linear( + in_dim, + q_dim + 2 * kv_dim, + bias=bias, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + weights_loading_config=WeightsLoadingConfig( + weight_mode=WeightMode.FUSED_QKV_LINEAR, + ), + fused_weight_shard_indices_mapping={ + "q": (0, local_q_dim), + "k": (local_q_dim, local_kv_dim), + "v": (local_q_dim + local_kv_dim, local_kv_dim), + }, + mapping=mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + reduce_output=False, + ) + # MLP gate+up: column-parallel with fused gate/up sharding + self.mlp_proj = Linear( + in_dim, + mlp_dim, + bias=bias, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights_in_init, + force_dynamic_quantization=force_dynamic_quantization, + weights_loading_config=WeightsLoadingConfig( + weight_mode=WeightMode.FUSED_GATE_UP_LINEAR, + ), + fused_weight_shard_indices_mapping={ + "gate": (0, shard_mlp_hidden_dim), + "up": (shard_mlp_hidden_dim, shard_mlp_hidden_dim), + }, + mapping=mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + reduce_output=False, + ) + self.local_qkv_dim = (q_dim + 2 * kv_dim) // self.tp_size + self.local_mlp_dim = mlp_dim // self.tp_size + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Returns (qkv, mlp_gate_up) with local (post-TP) sizes.""" + if self.tp_size == 1: + out = self.proj(x) + return out.split([self.local_qkv_dim, self.local_mlp_dim], dim=-1) + return self.qkv_proj(x), self.mlp_proj(x) + + def load_weights(self, weight_dict: Dict[str, torch.Tensor], loader: DynamicLinearWeightLoader): + """Load from checkpoint to_qkv_mlp_proj.weight (and optional .bias). + + The checkpoint stores a single fused weight with layout: + [Q: q_dim | K: kv_dim | V: kv_dim | gate: mlp_hid | up: mlp_hid] + + At TP=1: loads directly into self.proj. + At TP>1: splits into QKV and MLP portions, then loads into + column-parallel sub-Linears which handle per-rank sharding. + """ + for sub in self.modules(): + if callable(getattr(sub, "create_weights", None)): + sub.create_weights() + + if self.tp_size == 1: + loader.load_linear_weights(self.proj, "to_qkv_mlp_proj", [weight_dict]) + return + + W = weight_dict["weight"] # [full_qkv_dim + full_mlp_dim, in_dim] + W_qkv = W[: self.full_qkv_dim] + W_mlp = W[self.full_qkv_dim :] + + # Split QKV into Q, K, V for FUSED_QKV_LINEAR loader + W_q = W_qkv[: self.full_q_dim] + W_k = W_qkv[self.full_q_dim : self.full_q_dim + self.full_kv_dim] + W_v = W_qkv[self.full_q_dim + self.full_kv_dim :] + + # Split MLP into gate, up for FUSED_GATE_UP_LINEAR loader + W_gate = W_mlp[: self.mlp_hidden_dim] + W_up = W_mlp[self.mlp_hidden_dim :] + + # Build weight dict lists for fused loaders + if "bias" in weight_dict: + B = weight_dict["bias"] + B_qkv = B[: self.full_qkv_dim] + B_mlp = B[self.full_qkv_dim :] + + B_q = B_qkv[: self.full_q_dim] + B_k = B_qkv[self.full_q_dim : self.full_q_dim + self.full_kv_dim] + B_v = B_qkv[self.full_q_dim + self.full_kv_dim :] + + B_gate = B_mlp[: self.mlp_hidden_dim] + B_up = B_mlp[self.mlp_hidden_dim :] + + qkv_dicts = [ + {"weight": W_q, "bias": B_q}, + {"weight": W_k, "bias": B_k}, + {"weight": W_v, "bias": B_v}, + ] + mlp_dicts = [ + {"weight": W_gate, "bias": B_gate}, + {"weight": W_up, "bias": B_up}, + ] + else: + qkv_dicts = [{"weight": W_q}, {"weight": W_k}, {"weight": W_v}] + mlp_dicts = [{"weight": W_gate}, {"weight": W_up}] + + loader.load_linear_weights(self.qkv_proj, "to_qkv_mlp_proj", qkv_dicts) + loader.load_linear_weights(self.mlp_proj, "to_qkv_mlp_proj", mlp_dicts) diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py index 94c3dcf2ffd5..e87ace476d74 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py @@ -27,11 +27,12 @@ from tqdm import tqdm from tensorrt_llm._torch.modules.layer_norm import LayerNorm -from tensorrt_llm._torch.modules.linear import Linear +from tensorrt_llm._torch.modules.linear import Linear, TensorParallelMode from tensorrt_llm._torch.modules.mlp import MLP from tensorrt_llm._torch.utils import maybe_compile from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.models.flux.attention import FluxJointAttention +from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import FluxJointAttnMLPProj from tensorrt_llm._torch.visual_gen.models.flux.pos_embed_flux import FluxPosEmbed from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader from tensorrt_llm._torch.visual_gen.utils import SequenceSharder @@ -94,6 +95,7 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) self.norm = LayerNorm( hidden_size=embedding_dim, @@ -274,6 +276,8 @@ def __init__( self.config = config self.layer_idx = layer_idx + tp_size = config.mapping.tp_size if config and config.mapping else 1 + # AdaLN for image and text self.norm1 = AdaLayerNormZero( dim, @@ -322,7 +326,7 @@ def __init__( dtype=dtype, config=config, layer_idx=layer_idx, - reduce_output=False, + reduce_output=(tp_size != 1), ) self.ff_context = MLP( hidden_size=dim, @@ -332,7 +336,7 @@ def __init__( dtype=dtype, config=config, layer_idx=layer_idx, - reduce_output=False, + reduce_output=(tp_size != 1), ) def forward( @@ -455,18 +459,25 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + mapping=config.mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN if config.mapping.tp_size > 1 else None, + reduce_output=False, ) self.act_mlp = _gelu_tanh_eager - # Output projection (concat of attn + mlp) - TRT-LLM Linear - self.proj_out = Linear( - dim + self.mlp_hidden_dim, - dim, + kv_dim = num_attention_heads * attention_head_dim + + # MLP + Attn Output projection, requires special handling for TP + self.proj_out = FluxJointAttnMLPProj( + attn_dim=kv_dim, + mlp_dim=self.mlp_hidden_dim, + out_dim=dim, bias=True, dtype=dtype, quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + config=config, ) # Attention (no added_kv_proj_dim since tokens are already concatenated) @@ -523,9 +534,9 @@ def forward( ) # Concat and project - hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + hidden_states = self.proj_out(attn_output, mlp_hidden_states) gate = gate.unsqueeze(1) - hidden_states = gate * self.proj_out(hidden_states) + hidden_states = gate * hidden_states # Residual hidden_states = residual + hidden_states @@ -650,6 +661,7 @@ def __init__(self, model_config: DiffusionModelConfig): quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) # NOTE: x_embedder quantization is excluded when in_channels < 128. # FLUX.1 has in_channels=64, which is below the 128-block size required by @@ -668,6 +680,7 @@ def __init__(self, model_config: DiffusionModelConfig): quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) # Dual-stream transformer blocks @@ -726,6 +739,7 @@ def __init__(self, model_config: DiffusionModelConfig): quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) self.__post_init__() @@ -864,12 +878,28 @@ def load_weights(self, weights: dict) -> None: loader = DynamicLinearWeightLoader(self.model_config, params_map=params_map) + # Track prefixes of wrapper projectors whose sub-Linears are loaded + # by the parent's load_weights — the generic Linear loader must skip + # them (their FUSED weight modes would look for nonexistent checkpoint + # keys via params_map and error). + managed_prefixes = set() + for name, module in tqdm(self.named_modules(), desc="Loading weights"): + if any(name.startswith(p) for p in managed_prefixes): + continue + # Create weights for modules with skip_create_weights_in_init=True # This must be done before loading weights (following Wan pattern) if callable(getattr(module, "create_weights", None)): module.create_weights() + # Wrapper modules have no direct _parameters; handle before the guard. + if isinstance(module, FluxJointAttnMLPProj): + managed_prefixes.add(name + ".") + module_weights = loader.filter_weights(name, weights) + module.load_weights(module_weights, loader) + continue + if len(module._parameters) == 0: continue diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py index d276089a23ba..15dce09d9565 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py @@ -32,6 +32,10 @@ Flux2ParallelSelfAttention, FluxJointAttention, ) +from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import ( + FluxJointAttnMLPProj, + FluxJointQKVMLPProj, +) from tensorrt_llm._torch.visual_gen.models.flux.pos_embed_flux import FluxPosEmbed from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux import ( AdaLayerNormContinuous, @@ -222,6 +226,8 @@ def __init__( self.num_attention_heads = num_attention_heads self.attention_head_dim = attention_head_dim + tp_size = config.mapping.tp_size if config and config.mapping else 1 + # Layer norms (TRT-LLM - without elementwise affine, modulation provides scale/shift) self.norm1 = LayerNorm(hidden_size=dim, eps=eps, has_weights=False, has_bias=False) self.norm1_context = LayerNorm(hidden_size=dim, eps=eps, has_weights=False, has_bias=False) @@ -247,7 +253,7 @@ def __init__( dtype=dtype, config=config, layer_idx=layer_idx, - reduce_output=False, + reduce_output=(tp_size != 1), ) # FFN for text stream self.ff_context = GatedMLP( @@ -257,7 +263,7 @@ def __init__( dtype=dtype, config=config, layer_idx=layer_idx, - reduce_output=False, + reduce_output=(tp_size != 1), ) def forward( @@ -807,11 +813,27 @@ def load_weights(self, weights: dict) -> None: loader = DynamicLinearWeightLoader(self.model_config, params_map=params_map) + # Track prefixes of wrapper projectors whose sub-Linears are loaded + # by the parent's load_weights — the generic Linear loader must skip + # them (their FUSED weight modes would look for nonexistent checkpoint + # keys via params_map and error). + managed_prefixes = set() + for name, module in tqdm(self.named_modules(), desc="Loading FLUX.2 weights"): + # Skip sub-modules of wrapper projectors (loaded by parent above) + if any(name.startswith(p) for p in managed_prefixes): + continue + # Create weights for modules with skip_create_weights_in_init=True if callable(getattr(module, "create_weights", None)): module.create_weights() + if isinstance(module, (FluxJointAttnMLPProj, FluxJointQKVMLPProj)): + managed_prefixes.add(name + ".") + module_weights = loader.filter_weights(name, weights) + module.load_weights(module_weights, loader) + continue + if len(module._parameters) == 0: continue diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py index d16ab44b3b16..8d68f907e861 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py @@ -1019,6 +1019,9 @@ def __init__( f"must be divisible by ulysses_size ({vgm.ulysses_size})" ) + if self.model_config.mapping.tp_size > 1: + raise ValueError("LTX2 does not currently support TP.") + self._audio_is_sharded = False self._audio_pad = 0 # set by configure_audio_ulysses self._cache_dit_video_args: Optional[TransformerArgs] = None diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py index 1f6e300c3b51..c0c4907e0d88 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py @@ -9,11 +9,11 @@ from tensorrt_llm._torch.models.hf_parameter_utils import get_parameter_device from tensorrt_llm._torch.modules.layer_norm import LayerNorm -from tensorrt_llm._torch.modules.linear import Linear +from tensorrt_llm._torch.modules.linear import Linear, TensorParallelMode from tensorrt_llm._torch.modules.mlp import MLP -from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode +from tensorrt_llm._torch.visual_gen.modules.rms_norm import RMSNormTPAware from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader from tensorrt_llm._torch.visual_gen.utils import SequenceSharder from tensorrt_llm.logger import logger @@ -129,6 +129,7 @@ def __init__( force_dynamic_quantization=model_config.force_dynamic_quantization if model_config else False, + reduce_output=False, ) self.ff_out = Linear( in_features, @@ -143,6 +144,7 @@ def __init__( force_dynamic_quantization=model_config.force_dynamic_quantization if model_config else False, + reduce_output=False, ) self.norm2 = LayerNorm( @@ -201,6 +203,7 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) self.text_embedder = PixArtAlphaTextProjection(text_embed_dim, dim, act_fn="gelu_tanh") @@ -281,9 +284,13 @@ def __init__( hidden_size=hidden_size, eps=eps, dtype=torch.float32, has_weights=False, has_bias=False ) - # Self-attention with fused QKV. All WAN variants (1.3B 12h, 5B 24h, 14B 40h) - # fit the default fused_dit_qk_norm_rope op's full-dim template now that - # the num_heads cap is 64 (post-survey 2026-05). + # Self-attention with fused QKV. All WAN variants (1.3B 12h, 5B 24h, + # 14B 40h) fit the default fused_dit_qk_norm_rope op's full-dim + # template now that the num_heads cap is 64 (post-survey 2026-05). + # However, this kernel does not support TP due to the cross-head + # normalization being a collective op. Thus, we must disable it if + # using TP. + tp_size = model_config.mapping.tp_size if model_config.mapping else 1 self.attn1 = Attention( hidden_size=hidden_size, num_attention_heads=num_heads, @@ -291,7 +298,7 @@ def __init__( qkv_mode=QKVMode.FUSE_QKV, qk_norm=True, eps=eps, - fuse_qk_norm_rope=True, + fuse_qk_norm_rope=(tp_size == 1), config=model_config, layer_idx=_layer_idx, ) @@ -331,13 +338,14 @@ def __init__( dtype=dtype, config=model_config, layer_idx=_layer_idx, - reduce_output=False, + reduce_output=(tp_size != 1), ) - # I2V: Additional K/V projections for image embeddings + # I2V: Additional K/V projections for image embeddings. self.add_k_proj = self.add_v_proj = None self.norm_added_k = None if added_kv_proj_dim is not None: + tp_mode = TensorParallelMode.COLUMN if tp_size > 1 else None self.add_k_proj = Linear( added_kv_proj_dim, hidden_size, @@ -346,6 +354,8 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) self.add_v_proj = Linear( added_kv_proj_dim, @@ -355,9 +365,16 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) - self.norm_added_k = RMSNorm( - hidden_size=hidden_size, eps=eps, dtype=dtype, has_weights=True + self.norm_added_k = RMSNormTPAware( + hidden_size=hidden_size, + eps=eps, + dtype=dtype, + has_weights=True, + enable_tp=(tp_size > 1), + mapping=model_config.mapping, ) # Use torch.empty().normal_(std=...) instead of torch.randn()/scale for MetaInitMode compatibility @@ -469,8 +486,6 @@ def __init__( self.model_config = model_config vgm = model_config.visual_gen_mapping - if vgm is not None and vgm.tp_size > 1: - raise ValueError(f"WAN does not support tensor parallelism. Got tp_size={vgm.tp_size}") num_heads = getattr(model_config.pretrained_config, "num_attention_heads", 12) self.sharder = SequenceSharder.from_vgm(vgm, num_attention_heads=num_heads) @@ -571,6 +586,7 @@ def __init__( quant_config=quant_config, skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, + reduce_output=False, ) # Use torch.empty().normal_(std=...) instead of torch.randn()/scale for MetaInitMode compatibility self.scale_shift_table = nn.Parameter( @@ -772,6 +788,9 @@ def load_weights(self, weights: dict) -> None: loader.load_linear_weights(module, name, weight_dicts) elif "add_k_proj" in name or "add_v_proj" in name: logger.info(f"[Weight Loading] No weights found for I2V module: {name}") + elif isinstance(module, RMSNormTPAware): + module_weights = loader.filter_weights(name, weights) + module.load_weights(module_weights) else: module_weights = loader.filter_weights(name, weights) for param_name, param in module._parameters.items(): diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index 77f9a377a1ae..74bb62a1e21d 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -6,11 +6,11 @@ from tensorrt_llm.llmapi.llm_args import SkipSoftmaxAttentionConfig -from ...modules.linear import Linear, WeightMode, WeightsLoadingConfig -from ...modules.rms_norm import RMSNorm +from ...modules.linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig from ..attention_backend.interface import AttentionTensorLayout from ..attention_backend.utils import create_attention from ..config import DiffusionModelConfig, SkipSoftmaxConfig +from ..modules.rms_norm import RMSNormTPAware class QKVMode(str, Enum): @@ -60,7 +60,8 @@ def __init__( self.quant_config = config.quant_config self.skip_create_weights_in_init = config.skip_create_weights_in_init self.force_dynamic_quantization = config.force_dynamic_quantization - self.mapping = getattr(config, "mapping", None) + self.mapping = config.mapping + self.allreduce_strategy = config.allreduce_strategy self.hidden_size = hidden_size self.num_attention_heads = num_attention_heads @@ -69,12 +70,22 @@ def __init__( self.qkv_mode = QKVMode(qkv_mode) if isinstance(qkv_mode, str) else qkv_mode self.bias = bias + self.tp_size = self.mapping.tp_size if self.mapping else 1 + assert ( + self.num_attention_heads % self.tp_size == 0 + and self.num_key_value_heads % self.tp_size == 0 + ), "TP size must divide the number of Query and KV Heads" + # Fused QK Norm + RoPE: each model class opts in via fuse_qk_norm_rope. # Backed by torch.ops.trtllm.fused_dit_qk_norm_rope which auto-dispatches: # - per-head template (FLUX/Cosmos): q/k_weight.shape == [head_dim] # - full-dim template (LTX-2, WAN): q/k_weight.shape == [num_heads * head_dim] # Full-dim template envelope: num_heads <= 64, head_dim in {64, 128}. self.fuse_qk_norm_rope = fuse_qk_norm_rope if fuse_qk_norm_rope is not None else False + assert not (self.fuse_qk_norm_rope and self.tp_size > 1 and qk_norm_mode == "full"), ( + "fuse_qk_norm_rope + qk_norm_mode='full' + TP>1: fused kernel lacks cross-rank " + "all-reduce for cross-head RMSNorm variance. Disable fuse_qk_norm_rope for TP>1." + ) self.interleave = interleave # Select compute backend (orthogonal to parallelism) @@ -97,6 +108,11 @@ def __init__( self.q_dim = self.num_attention_heads * self.head_dim self.kv_dim = self.num_key_value_heads * self.head_dim + self.local_num_attention_heads = self.num_attention_heads // self.tp_size + self.local_num_key_value_heads = self.num_key_value_heads // self.tp_size + self.local_q_dim = self.local_num_attention_heads * self.head_dim + self.local_kv_dim = self.local_num_key_value_heads * self.head_dim + self._init_qkv_proj() attention_metadata_state = getattr(config, "attention_metadata_state", None) @@ -104,13 +120,25 @@ def __init__( if self.qk_norm: # "full": norm over all heads combined (e.g. WAN, dim=q_dim) # "per_head": norm over each head independently (e.g. FLUX, dim=head_dim) + q_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.q_dim k_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.kv_dim - self.norm_q = RMSNorm( - hidden_size=q_norm_dim, eps=self.eps, dtype=self.dtype, has_weights=True + enable_tp_rms = self.tp_size > 1 and qk_norm_mode == "full" + self.norm_q = RMSNormTPAware( + hidden_size=q_norm_dim, + eps=self.eps, + dtype=self.dtype, + has_weights=True, + enable_tp=enable_tp_rms, + mapping=self.mapping, ) - self.norm_k = RMSNorm( - hidden_size=k_norm_dim, eps=self.eps, dtype=self.dtype, has_weights=True + self.norm_k = RMSNormTPAware( + hidden_size=k_norm_dim, + eps=self.eps, + dtype=self.dtype, + has_weights=True, + enable_tp=enable_tp_rms, + mapping=self.mapping, ) # TODO: Use weight mapper to create just a Linear module @@ -125,6 +153,9 @@ def __init__( quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=TensorParallelMode.ROW if self.tp_size > 1 else None, + reduce_output=(self.tp_size > 1), + allreduce_strategy=self.allreduce_strategy, ) ] ) @@ -139,11 +170,11 @@ def __init__( # Ulysses shards heads across workers; inner backend sees sharded count # Attention2D gathers sequence (not heads); inner backend sees full count if use_ulysses: - backend_num_heads = self.num_attention_heads // ulysses_size - backend_num_kv_heads = self.num_key_value_heads // ulysses_size + backend_num_heads = self.local_num_attention_heads // ulysses_size + backend_num_kv_heads = self.local_num_key_value_heads // ulysses_size else: - backend_num_heads = self.num_attention_heads - backend_num_kv_heads = self.num_key_value_heads + backend_num_heads = self.local_num_attention_heads + backend_num_kv_heads = self.local_num_key_value_heads # Resolve sparse attention config for TRTLLM backend sparse_attention_config = None @@ -201,8 +232,13 @@ def __init__( self.attn = UlyssesAttention(self.attn, process_group=vgm.ulysses_group) def _init_qkv_proj(self) -> None: + tp_mode = TensorParallelMode.COLUMN if self.tp_size > 1 else None + if self.qkv_mode == QKVMode.FUSE_QKV: qkv_out_dim = self.q_dim + 2 * self.kv_dim + + # Input / Output dims are the full tensor sizes + # fused_weight_shard_indices_mapping want indexes for just _this_ shard self.qkv_proj = Linear( self.hidden_size, qkv_out_dim, @@ -216,10 +252,12 @@ def _init_qkv_proj(self) -> None: weight_mode=WeightMode.FUSED_QKV_LINEAR ), fused_weight_shard_indices_mapping={ - "q": (0, self.q_dim), - "k": (self.q_dim, self.kv_dim), - "v": (self.q_dim + self.kv_dim, self.kv_dim), + "q": (0, self.local_q_dim), + "k": (self.local_q_dim, self.local_kv_dim), + "v": (self.local_q_dim + self.local_kv_dim, self.local_kv_dim), }, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) else: self.to_q = Linear( @@ -231,6 +269,8 @@ def _init_qkv_proj(self) -> None: quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) self.to_k = Linear( self.hidden_size, @@ -241,6 +281,8 @@ def _init_qkv_proj(self) -> None: quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) self.to_v = Linear( self.hidden_size, @@ -251,6 +293,8 @@ def _init_qkv_proj(self) -> None: quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) def get_qkv( @@ -260,7 +304,7 @@ def get_qkv( ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if self.qkv_mode == QKVMode.FUSE_QKV: qkv = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_dim, self.kv_dim, self.kv_dim], dim=-1) + q, k, v = qkv.split([self.local_q_dim, self.local_kv_dim, self.local_kv_dim], dim=-1) else: kv_source = ( encoder_hidden_states if encoder_hidden_states is not None else hidden_states @@ -298,6 +342,7 @@ def apply_packed_qk_norm_rope( """ B, S, D = qkv.shape tokens_per_batch = S if num_txt_tokens > 0 else 0 + assert self.tp_size == 1, "fused_dit_split_norm_rope does not support TP" torch.ops.trtllm.fused_dit_qk_norm_rope( qkv.view(B * S, D), self.num_attention_heads, @@ -334,6 +379,7 @@ def apply_split_norm_rope( Caller passes raw cos/sin (any rank); the op reshapes internally. """ B, T, _ = tensor.shape + assert self.tp_size == 1, "fused_dit_split_norm_rope does not support TP" torch.ops.trtllm.fused_dit_split_norm_rope( tensor.view(B * T, -1), num_heads, @@ -357,6 +403,7 @@ def apply_split_norm( no RoPE -- e.g. LTX-2 text cross-attn (Q-norm with pe=None). """ B, T, _ = tensor.shape + assert self.tp_size == 1, "fused_dit_split_norm does not support TP" torch.ops.trtllm.fused_dit_split_norm( tensor.view(B * T, -1), num_heads, @@ -408,13 +455,19 @@ def _attn_impl( # Reshape inputs: [B, S, H*D] -> backend's preferred 4D layout if backend_layout == AttentionTensorLayout.HND: - q = q.view(batch_size, -1, self.num_attention_heads, self.head_dim).transpose(1, 2) - k = k.view(batch_size, -1, self.num_key_value_heads, self.head_dim).transpose(1, 2) - v = v.view(batch_size, -1, self.num_key_value_heads, self.head_dim).transpose(1, 2) + q = q.view(batch_size, -1, self.local_num_attention_heads, self.head_dim).transpose( + 1, 2 + ) + k = k.view(batch_size, -1, self.local_num_key_value_heads, self.head_dim).transpose( + 1, 2 + ) + v = v.view(batch_size, -1, self.local_num_key_value_heads, self.head_dim).transpose( + 1, 2 + ) else: - q = q.view(batch_size, -1, self.num_attention_heads, self.head_dim) - k = k.view(batch_size, -1, self.num_key_value_heads, self.head_dim) - v = v.view(batch_size, -1, self.num_key_value_heads, self.head_dim) + q = q.view(batch_size, -1, self.local_num_attention_heads, self.head_dim) + k = k.view(batch_size, -1, self.local_num_key_value_heads, self.head_dim) + v = v.view(batch_size, -1, self.local_num_key_value_heads, self.head_dim) kwargs.update( { @@ -459,7 +512,7 @@ def forward( qkv = self.qkv_proj(hidden_states) freqs_cos, freqs_sin = freqs self.apply_packed_qk_norm_rope(qkv, freqs_cos, freqs_sin) - q, k, v = qkv.split([self.q_dim, self.kv_dim, self.kv_dim], dim=-1) + q, k, v = qkv.split([self.local_q_dim, self.local_kv_dim, self.local_kv_dim], dim=-1) out = self._attn_impl(q, k, v) return self.to_out[0](out) @@ -470,8 +523,10 @@ def forward( # Apply RoPE if provided (model handles RoPE, not attention backend) if freqs is not None: freqs_cos, freqs_sin = freqs - q = q.view(batch_size, seq_len, self.num_attention_heads, self.head_dim) # [B, S, H, D] - k = k.view(batch_size, kv_seq_len, self.num_key_value_heads, self.head_dim) + q = q.view( + batch_size, seq_len, self.local_num_attention_heads, self.head_dim + ) # [B, S, H, D] + k = k.view(batch_size, kv_seq_len, self.local_num_key_value_heads, self.head_dim) q = apply_rotary_emb(q, freqs_cos, freqs_sin) k = apply_rotary_emb(k, freqs_cos, freqs_sin) q = q.flatten(2) diff --git a/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py new file mode 100644 index 000000000000..b58239ca8b9b --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from torch import nn + +from tensorrt_llm._torch.distributed import AllReduce +from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.mapping import Mapping + + +class RMSNormTPAware(nn.Module): + def __init__( + self, + *, + hidden_size: int, + eps: float, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + has_weights: bool = True, + use_gemma: bool = False, + enable_tp: bool = False, + mapping: Optional[Mapping] = None, + allreduce_strategy: AllReduceStrategy = AllReduceStrategy.NCCL, + ): + super().__init__() + + self.variance_epsilon = eps + self.use_gemma = use_gemma + + self.mapping = mapping + self.enable_tp = enable_tp + + if enable_tp: + assert mapping is not None + self.full_size = hidden_size + shard = hidden_size // mapping.tp_size + start = shard * mapping.tp_rank + end = min(shard * (mapping.tp_rank + 1), hidden_size) + hidden_size = end - start + + self.allreduce = AllReduce( + mapping=mapping, strategy=allreduce_strategy, dtype=torch.float32 + ) + else: + self.allreduce = None + + if use_gemma and not has_weights: + raise ValueError("has_weights must be True if use_gemma is True") + if has_weights: + if not use_gemma: + self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype, device=device)) + else: + self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype, device=device)) + else: + self.register_buffer( + "weight", torch.ones(hidden_size, dtype=dtype, device=device), persistent=False + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + + x2 = hidden_states.pow(2) + if self.allreduce: + x2_sum = x2.sum(-1, keepdim=True) + variance = self.allreduce(x2_sum) / self.full_size + else: + variance = x2.mean(-1, keepdim=True) + + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + if not self.use_gemma: + hidden_states = self.weight * hidden_states.to(input_dtype) + else: + hidden_states = (self.weight + 1) * hidden_states.to(input_dtype) + + return hidden_states + + def load_weights(self, weights: torch.Tensor): + for param_name, param in self._parameters.items(): + if param is None or param_name not in weights: + continue + if param_name == "weight" and self.enable_tp: + shard = self.full_size // self.mapping.tp_size + start = shard * self.mapping.tp_rank + end = min(shard * (self.mapping.tp_rank + 1), self.full_size) + data = weights[param_name][..., start:end] + else: + data = weights[param_name] + + param.data.copy_(data.to(param.dtype)) diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py index cb015195d357..ac61ce729fbc 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py @@ -152,6 +152,7 @@ def _setup_visual_gen_mapping(self, config: DiffusionModelConfig) -> None: ring_size=self.args.parallel_config.ring_size, attn2d_row_size=attn2d_row, attn2d_col_size=attn2d_col, + tp_size=self.args.parallel_config.tp_size, parallel_vae_size=self.args.parallel_config.parallel_vae_size, ) config.visual_gen_mapping = vgm diff --git a/tensorrt_llm/visual_gen/args.py b/tensorrt_llm/visual_gen/args.py index c4a324779f0c..8daefd5d7bb6 100644 --- a/tensorrt_llm/visual_gen/args.py +++ b/tensorrt_llm/visual_gen/args.py @@ -226,6 +226,12 @@ class ParallelConfig(StrictBaseModel): "column group. Mutually exclusive with ring_size > 1." ), ) + tp_size: int = Field( + 1, + ge=1, + status="prototype", + description=("Tensor parallel group size. Heads are sharded across tp_size GPUs."), + ) @property def seq_parallel_size(self) -> int: @@ -246,7 +252,7 @@ def seq_parallel_size(self) -> int: @property def n_workers(self) -> int: - return self.cfg_size * self.seq_parallel_size + return self.cfg_size * self.seq_parallel_size * self.tp_size @property def total_parallel_size(self) -> int: diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py new file mode 100644 index 000000000000..54b462a4cc62 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py @@ -0,0 +1,686 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-GPU tests for FLUX Tensor Parallelism (TP). + +Tests that FLUX.1 and FLUX.2 transformers produce correct outputs when using +TP (sharding weights across GPUs), and combined TP + Ulysses. + +Run with: + pytest tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py -v +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" + +from types import SimpleNamespace +from typing import Callable + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +try: + from tensorrt_llm._torch.visual_gen.config import ( + AttentionConfig, + DiffusionModelConfig, + TorchCompileConfig, + create_attention_metadata_state, + ) + from tensorrt_llm._torch.visual_gen.mapping import VisualGenMapping + from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import ( + FluxJointAttnMLPProj, + FluxJointQKVMLPProj, + ) + from tensorrt_llm._utils import get_free_port + from tensorrt_llm.models.modeling_utils import QuantConfig + + MODULES_AVAILABLE = True +except ImportError: + MODULES_AVAILABLE = False + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + """Clean up TLLM_DISABLE_MPI env var after tests complete.""" + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +# ============================================================================= +# Distributed helpers (same pattern as test_flux_ulysses.py) +# ============================================================================= + + +def init_distributed_worker(rank: int, world_size: int, backend: str = "nccl", port: int = 29500): + """Initialize distributed environment for a worker process.""" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + + torch.cuda.set_device(rank % torch.cuda.device_count()) + dist.init_process_group(backend=backend, rank=rank, world_size=world_size) + + +def cleanup_distributed(): + """Clean up distributed environment.""" + if dist.is_initialized(): + dist.destroy_process_group() + + +def _distributed_worker(rank, world_size, backend, test_fn, port): + """Worker function that runs in each process. Module-level for pickling.""" + try: + init_distributed_worker(rank, world_size, backend, port) + test_fn(rank, world_size) + except Exception as e: + print(f"Rank {rank} failed with error: {e}") + raise + finally: + cleanup_distributed() + + +def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = True): + """Run a test function in a distributed environment.""" + if not MODULES_AVAILABLE: + pytest.skip("Required modules not available") + + if use_cuda and torch.cuda.device_count() < world_size: + pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") + + backend = "nccl" if use_cuda else "gloo" + port = get_free_port() + + mp.spawn( + _distributed_worker, args=(world_size, backend, test_fn, port), nprocs=world_size, join=True + ) + + +# ============================================================================= +# Model config helpers +# ============================================================================= + +# Small FLUX.1 config for testing (reduced layers, 8 heads, 64 head_dim = 512 inner_dim) +_FLUX1_TEST_CONFIG = dict( + num_attention_heads=8, + attention_head_dim=64, + in_channels=64, + out_channels=64, + num_layers=2, + num_single_layers=4, + joint_attention_dim=256, + pooled_projection_dim=128, + guidance_embeds=False, + patch_size=1, + axes_dims_rope=[16, 24, 24], + theta=10000, +) + +# Small FLUX.2 config for testing +_FLUX2_TEST_CONFIG = dict( + num_attention_heads=8, + attention_head_dim=64, + in_channels=128, + out_channels=128, + num_layers=2, + num_single_layers=4, + joint_attention_dim=256, + pooled_projection_dim=128, + guidance_embeds=False, + patch_size=1, + mlp_ratio=3.0, + axes_dims_rope=[16, 16, 16, 16], + rope_theta=2000.0, + eps=1e-6, + timestep_guidance_channels=256, +) + + +def _make_model_config(pretrained_dict, tp_size=1, ulysses_size=1, backend="VANILLA"): + """Create DiffusionModelConfig for testing with TP and/or Ulysses.""" + pretrained_config = SimpleNamespace(**pretrained_dict) + ws = tp_size * ulysses_size + if ws > 1 and dist.is_initialized(): + ws = dist.get_world_size() + rk = dist.get_rank() + else: + rk = 0 + vgm = VisualGenMapping(world_size=ws, rank=rk, tp_size=tp_size, ulysses_size=ulysses_size) + + config = DiffusionModelConfig( + pretrained_config=pretrained_config, + quant_config=QuantConfig(), + torch_compile=TorchCompileConfig(enable=False), + attention=AttentionConfig(backend=backend), + visual_gen_mapping=vgm, + cache=None, + attention_metadata_state=( + create_attention_metadata_state() if backend.upper() == "TRTLLM" else None + ), + skip_create_weights_in_init=False, + ) + config.mapping = vgm.to_llm_mapping() + return config + + +def _stabilize_model_weights(model): + """Reinitialize model weights for stable BF16 forward pass. + + Random default init (std~1.0) causes BF16 overflow through multiple + transformer blocks. Use small uniform init that keeps activations bounded. + """ + with torch.no_grad(): + for name, p in model.named_parameters(): + if p.ndim >= 2: + fan_in = p.shape[1] if p.ndim >= 2 else p.shape[0] + std = 0.02 / max(1.0, fan_in**0.5) + p.data.uniform_(-std, std) + else: + p.data.uniform_(-0.01, 0.01) + + +# ============================================================================= +# TP weight sharding helpers +# ============================================================================= + + +def _shard_dim0(tensor, tp_rank, tp_size): + """Shard a tensor along dim 0.""" + chunk = tensor.shape[0] // tp_size + return tensor[tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() + + +def _shard_dim1(tensor, tp_rank, tp_size): + """Shard a tensor along dim 1.""" + chunk = tensor.shape[1] // tp_size + return tensor[:, tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() + + +def _shard_fused_qkv(tensor, tp_rank, tp_size, q_dim, kv_dim): + """Shard a fused QKV weight [q_dim + 2*kv_dim, ...] preserving Q/K/V structure.""" + q, k, v = tensor.split([q_dim, kv_dim, kv_dim], dim=0) + return torch.cat( + [ + _shard_dim0(q, tp_rank, tp_size), + _shard_dim0(k, tp_rank, tp_size), + _shard_dim0(v, tp_rank, tp_size), + ], + dim=0, + ) + + +def _shard_fused_gate_up(tensor, tp_rank, tp_size): + """Shard a fused gate_up weight [2*intermediate, ...] preserving gate/up structure.""" + half = tensor.shape[0] // 2 + gate, up = tensor.split([half, half], dim=0) + return torch.cat( + [ + _shard_dim0(gate, tp_rank, tp_size), + _shard_dim0(up, tp_rank, tp_size), + ], + dim=0, + ) + + +def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): + """Copy weights from a TP=1 reference model into a TP model with correct sharding. + + Handles column-parallel (QKV, MLP up/gate), row-parallel (output projs), + fused QKV/gate_up weights, and wrapper projectors (FluxJointAttnMLPProj, + FluxJointQKVMLPProj) that have different sub-module structure at TP>1. + """ + ref_params = dict(ref_model.named_parameters()) + + # First handle wrapper projectors whose sub-module names differ between TP=1 and TP>1. + # At TP=1: single .proj Linear. At TP>1: split into sub-Linears. + handled_tp_params = set() + + for tp_name, tp_module in tp_model.named_modules(): + if isinstance(tp_module, FluxJointAttnMLPProj) and tp_module.tp_size > 1: + # TP model has .attn_proj + .mlp_proj; ref has .proj + ref_w = ref_params[f"{tp_name}.proj.weight"] # [out, attn_dim + mlp_dim] + w_attn = ref_w[:, : tp_module.attn_dim] + w_mlp = ref_w[:, tp_module.attn_dim :] + tp_model.get_parameter(f"{tp_name}.attn_proj.weight").data.copy_( + _shard_dim1(w_attn, tp_rank, tp_size) + ) + tp_model.get_parameter(f"{tp_name}.mlp_proj.weight").data.copy_( + _shard_dim1(w_mlp, tp_rank, tp_size) + ) + handled_tp_params.update( + [ + f"{tp_name}.attn_proj.weight", + f"{tp_name}.mlp_proj.weight", + ] + ) + if tp_module.has_bias: + ref_b = ref_params[f"{tp_name}.proj.bias"] + tp_model.get_parameter(f"{tp_name}.bias").data.copy_(ref_b) + handled_tp_params.add(f"{tp_name}.bias") + + elif isinstance(tp_module, FluxJointQKVMLPProj) and tp_module.tp_size > 1: + # TP model has .qkv_proj + .mlp_proj; ref has .proj + ref_w = ref_params[f"{tp_name}.proj.weight"] # [qkv+mlp, hidden] + w_qkv = ref_w[: tp_module.full_qkv_dim] + w_mlp = ref_w[tp_module.full_qkv_dim :] + + # QKV: split into Q/K/V, shard each, re-fuse + tp_model.get_parameter(f"{tp_name}.qkv_proj.weight").data.copy_( + _shard_fused_qkv( + w_qkv, tp_rank, tp_size, tp_module.full_q_dim, tp_module.full_kv_dim + ) + ) + # MLP: split gate/up, shard each, re-fuse + tp_model.get_parameter(f"{tp_name}.mlp_proj.weight").data.copy_( + _shard_fused_gate_up(w_mlp, tp_rank, tp_size) + ) + handled_tp_params.update( + [ + f"{tp_name}.qkv_proj.weight", + f"{tp_name}.mlp_proj.weight", + ] + ) + # Handle bias if present + if f"{tp_name}.proj.bias" in ref_params: + ref_b = ref_params[f"{tp_name}.proj.bias"] + b_qkv = ref_b[: tp_module.full_qkv_dim] + b_mlp = ref_b[tp_module.full_qkv_dim :] + tp_model.get_parameter(f"{tp_name}.qkv_proj.bias").data.copy_( + _shard_fused_qkv( + b_qkv, tp_rank, tp_size, tp_module.full_q_dim, tp_module.full_kv_dim + ) + ) + tp_model.get_parameter(f"{tp_name}.mlp_proj.bias").data.copy_( + _shard_fused_gate_up(b_mlp, tp_rank, tp_size) + ) + handled_tp_params.update( + [ + f"{tp_name}.qkv_proj.bias", + f"{tp_name}.mlp_proj.bias", + ] + ) + + # Now handle all remaining parameters by shape comparison. + with torch.no_grad(): + for tp_name, tp_param in tp_model.named_parameters(): + if tp_name in handled_tp_params: + continue + if tp_name not in ref_params: + continue + + ref_param = ref_params[tp_name] + + if tp_param.shape == ref_param.shape: + # Replicated parameter (norms, embeddings, etc.) + tp_param.data.copy_(ref_param.data) + elif tp_param.ndim >= 2 and tp_param.shape[1] == ref_param.shape[1]: + # Column parallel: dim 0 is smaller (output dim sharded) + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + # Fused QKV: figure out q_dim from total (q=k=v for FLUX) + q_dim = ref_param.shape[0] // 3 + tp_param.data.copy_( + _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim >= 2 and tp_param.shape[0] == ref_param.shape[0]: + # Row parallel: dim 1 is smaller (input dim sharded) + tp_param.data.copy_(_shard_dim1(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim == 1 and tp_param.shape[0] < ref_param.shape[0]: + # 1D bias for column parallel + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + q_dim = ref_param.shape[0] // 3 + tp_param.data.copy_( + _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) + else: + raise ValueError( + f"Cannot shard {tp_name}: ref={ref_param.shape}, tp={tp_param.shape}" + ) + + +# ============================================================================= +# FLUX.1 test logic functions (module-level for pickling) +# ============================================================================= + + +def _logic_flux1_tp_forward(rank, world_size): + """FLUX.1 transformer forward with TP — verify shape and no NaN/Inf.""" + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux import FluxTransformer2DModel + + device = torch.device(f"cuda:{rank}") + torch.manual_seed(42) + + model_config = _make_model_config(_FLUX1_TEST_CONFIG, tp_size=world_size) + model = FluxTransformer2DModel(model_config).to(device).to(torch.bfloat16) + _stabilize_model_weights(model) + + batch = 1 + img_seq = 16 + txt_seq = 8 + + torch.manual_seed(100) + hidden_states = torch.randn(batch, img_seq, 64, device=device, dtype=torch.bfloat16) * 0.1 + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 256, device=device, dtype=torch.bfloat16) * 0.1 + ) + pooled_projections = torch.randn(batch, 128, device=device, dtype=torch.bfloat16) * 0.1 + timestep = torch.tensor([0.5], device=device, dtype=torch.bfloat16) + img_ids = torch.randn(img_seq, 3, device=device) + txt_ids = torch.randn(txt_seq, 3, device=device) + + with torch.no_grad(): + output = model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + pooled_projections=pooled_projections, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + + sample = output["sample"] + assert sample.shape == (batch, img_seq, 64), ( + f"Rank {rank}: Expected shape {(batch, img_seq, 64)}, got {sample.shape}" + ) + assert not torch.isnan(sample).any(), f"Rank {rank}: NaN in output" + assert not torch.isinf(sample).any(), f"Rank {rank}: Inf in output" + + +def _logic_flux1_tp_vs_single_gpu(rank, world_size): + """FLUX.1: TP 2-GPU output matches single-GPU reference.""" + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux import FluxTransformer2DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + + batch = 1 + img_seq = 16 + txt_seq = 8 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_FLUX1_TEST_CONFIG, tp_size=1) + ref_model = FluxTransformer2DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create TP model and copy sharded weights from ref + torch.manual_seed(123) + tp_config = _make_model_config(_FLUX1_TEST_CONFIG, tp_size=world_size) + tp_model = FluxTransformer2DModel(tp_config).to(device).to(compute_dtype) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + + # Same inputs on all ranks + torch.manual_seed(456) + hidden_states = torch.randn(batch, img_seq, 64, device=device, dtype=compute_dtype) * 0.1 + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 256, device=device, dtype=compute_dtype) * 0.1 + ) + pooled_projections = torch.randn(batch, 128, device=device, dtype=compute_dtype) * 0.1 + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + img_ids = torch.randn(img_seq, 3, device=device) + txt_ids = torch.randn(txt_seq, 3, device=device) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + pooled_projections=pooled_projections, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + tp_output = tp_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + pooled_projections=pooled_projections, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + + torch.testing.assert_close( + tp_output["sample"], + ref_output["sample"], + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: FLUX.1 TP output differs from single-GPU reference", + ) + + +# ============================================================================= +# FLUX.2 test logic functions (module-level for pickling) +# ============================================================================= + + +def _logic_flux2_tp_forward(rank, world_size): + """FLUX.2 transformer forward with TP — verify shape and no NaN/Inf.""" + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import Flux2Transformer2DModel + + device = torch.device(f"cuda:{rank}") + torch.manual_seed(42) + + model_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=world_size) + model = Flux2Transformer2DModel(model_config).to(device).to(torch.bfloat16) + _stabilize_model_weights(model) + + batch = 1 + img_seq = 16 + txt_seq = 8 + + torch.manual_seed(100) + hidden_states = torch.randn(batch, img_seq, 128, device=device, dtype=torch.bfloat16) * 0.1 + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 256, device=device, dtype=torch.bfloat16) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=torch.bfloat16) + img_ids = torch.randn(img_seq, 4, device=device) + txt_ids = torch.randn(txt_seq, 4, device=device) + + with torch.no_grad(): + output = model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + + sample = output["sample"] + assert sample.shape == (batch, img_seq, 128), ( + f"Rank {rank}: Expected shape {(batch, img_seq, 128)}, got {sample.shape}" + ) + assert not torch.isnan(sample).any(), f"Rank {rank}: NaN in output" + assert not torch.isinf(sample).any(), f"Rank {rank}: Inf in output" + + +def _logic_flux2_tp_vs_single_gpu(rank, world_size): + """FLUX.2: TP 2-GPU output matches single-GPU reference.""" + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import Flux2Transformer2DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + + batch = 1 + img_seq = 16 + txt_seq = 8 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=1) + ref_model = Flux2Transformer2DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create TP model and copy sharded weights from ref + torch.manual_seed(123) + tp_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=world_size) + tp_model = Flux2Transformer2DModel(tp_config).to(device).to(compute_dtype) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + + # Same inputs on all ranks + torch.manual_seed(456) + hidden_states = torch.randn(batch, img_seq, 128, device=device, dtype=compute_dtype) * 0.1 + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 256, device=device, dtype=compute_dtype) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + img_ids = torch.randn(img_seq, 4, device=device) + txt_ids = torch.randn(txt_seq, 4, device=device) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + tp_output = tp_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + + torch.testing.assert_close( + tp_output["sample"], + ref_output["sample"], + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: FLUX.2 TP output differs from single-GPU reference", + ) + + +# ============================================================================= +# Combined TP + Ulysses test logic (module-level for pickling) +# ============================================================================= + + +def _logic_flux2_tp_ulysses_vs_single_gpu(rank, world_size): + """FLUX.2: TP=2 + Ulysses=2 (4 GPUs) output matches single-GPU reference. + + Each rank has a shard of both the weights (TP) and sequence (Ulysses). + The output should match the single-GPU model after gathering. + """ + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import Flux2Transformer2DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + tp_size = 2 + ulysses_size = 2 + + batch = 1 + img_seq = 16 # Must be divisible by ulysses_size + txt_seq = 8 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=1, ulysses_size=1) + ref_model = Flux2Transformer2DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create combined TP + Ulysses model and copy sharded weights + torch.manual_seed(123) + combined_config = _make_model_config( + _FLUX2_TEST_CONFIG, tp_size=tp_size, ulysses_size=ulysses_size + ) + combined_model = Flux2Transformer2DModel(combined_config).to(device).to(compute_dtype) + vgm = combined_config.visual_gen_mapping + _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size) + + # Same inputs on all ranks (Ulysses shards at runtime) + torch.manual_seed(456) + hidden_states = torch.randn(batch, img_seq, 128, device=device, dtype=compute_dtype) * 0.1 + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 256, device=device, dtype=compute_dtype) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + img_ids = torch.randn(img_seq, 4, device=device) + txt_ids = torch.randn(txt_seq, 4, device=device) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + combined_output = combined_model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + img_ids=img_ids, + txt_ids=txt_ids, + ) + + torch.testing.assert_close( + combined_output["sample"], + ref_output["sample"], + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: FLUX.2 TP+Ulysses output differs from single-GPU reference", + ) + + +# ============================================================================= +# Test classes +# ============================================================================= + + +class TestFlux1TP: + """Tensor parallelism tests for FLUX.1 transformer.""" + + def test_flux1_tp_forward(self): + """FLUX.1 TP forward: correct output shape and no NaN/Inf.""" + run_test_in_distributed(world_size=2, test_fn=_logic_flux1_tp_forward) + + def test_flux1_tp_vs_single_gpu(self): + """FLUX.1 TP 2-GPU output matches single-GPU reference.""" + run_test_in_distributed(world_size=2, test_fn=_logic_flux1_tp_vs_single_gpu) + + +class TestFlux2TP: + """Tensor parallelism tests for FLUX.2 transformer.""" + + def test_flux2_tp_forward(self): + """FLUX.2 TP forward: correct output shape and no NaN/Inf.""" + run_test_in_distributed(world_size=2, test_fn=_logic_flux2_tp_forward) + + def test_flux2_tp_vs_single_gpu(self): + """FLUX.2 TP 2-GPU output matches single-GPU reference.""" + run_test_in_distributed(world_size=2, test_fn=_logic_flux2_tp_vs_single_gpu) + + +class TestFlux2TPUlyssesCombined: + """Combined TP + Ulysses tests for FLUX.2 transformer.""" + + def test_flux2_tp_ulysses_vs_single_gpu(self): + """FLUX.2 TP=2 + Ulysses=2 (4 GPUs) matches single-GPU reference.""" + run_test_in_distributed(world_size=4, test_fn=_logic_flux2_tp_ulysses_vs_single_gpu) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py new file mode 100644 index 000000000000..2d232915569d --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py @@ -0,0 +1,522 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-GPU tests for Tensor Parallel (TP) Attention. + +Validates that TP-sharded attention (column-parallel QKV, row-parallel output +with all-reduce) produces the same result as F.scaled_dot_product_attention +with full (unsharded) weights. + +Also includes a combined TP + Ulysses test to verify both parallelisms compose. + +Run with: + pytest tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py -v +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" + +import math +from typing import Callable + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +import torch.nn.functional as F + +try: + from tensorrt_llm._torch.device_mesh import DeviceMeshTopologyImpl + from tensorrt_llm._torch.visual_gen.config import AttentionConfig, DiffusionModelConfig + from tensorrt_llm._torch.visual_gen.mapping import VisualGenMapping + from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode + from tensorrt_llm._utils import get_free_port + from tensorrt_llm.mapping import Mapping + + MODULES_AVAILABLE = True +except ImportError: + MODULES_AVAILABLE = False + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +# ============================================================================= +# Distributed helpers (same pattern as test_ulysses_attention.py) +# ============================================================================= + + +def _init_worker(rank: int, world_size: int, port: int): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + torch.cuda.set_device(rank % torch.cuda.device_count()) + dist.init_process_group(backend="nccl", rank=rank, world_size=world_size) + + +def _cleanup(): + if dist.is_initialized(): + dist.destroy_process_group() + + +def _distributed_worker(rank, world_size, test_fn, port, *args): + try: + _init_worker(rank, world_size, port) + test_fn(rank, world_size, *args) + except Exception as e: + print(f"Rank {rank} failed: {e}") + raise + finally: + _cleanup() + + +def _run(world_size: int, test_fn: Callable, *args): + if not MODULES_AVAILABLE: + pytest.skip("Required modules not available") + if torch.cuda.device_count() < world_size: + pytest.skip(f"Need {world_size} GPUs, have {torch.cuda.device_count()}") + port = get_free_port() + mp.spawn( + _distributed_worker, args=(world_size, test_fn, port, *args), nprocs=world_size, join=True + ) + + +# ============================================================================= +# Helpers for building configs and sharding weights +# ============================================================================= + + +def _setup_vgm(rank, world_size, tp_size=1, ulysses_size=1): + """Create a VisualGenMapping which calls init_pg (required by the C++ all-reduce backend).""" + DeviceMeshTopologyImpl.device_mesh = None + if tp_size == 1 and ulysses_size == 1: + return None + return VisualGenMapping( + world_size=world_size, + rank=rank, + tp_size=tp_size, + ulysses_size=ulysses_size, + ) + + +def _make_config(vgm=None, backend="VANILLA", **mapping_kwargs): + """Build a DiffusionModelConfig for testing. + + When a VisualGenMapping is provided, its to_llm_mapping() supplies the + Mapping backed by the device mesh. Otherwise a plain Mapping is created + from ``mapping_kwargs`` (tp_size, rank, etc.). + """ + if vgm is not None and (vgm.tp_size > 1 or vgm.ulysses_size > 1): + mapping = vgm.to_llm_mapping() + else: + mapping = Mapping(**mapping_kwargs) + return DiffusionModelConfig( + mapping=mapping, + visual_gen_mapping=vgm, + attention=AttentionConfig(backend=backend), + skip_create_weights_in_init=False, + ) + + +def _broadcast_params(module): + """Initialize weights with small random values and broadcast from rank 0. + + TRT-LLM Linear uses torch.empty (uninitialized memory), so we must + explicitly fill with valid data before use. + """ + for p in module.parameters(): + torch.nn.init.normal_(p.data, mean=0.0, std=0.02) + dist.broadcast(p.data, src=0) + + +def _shard_tp_weights(ref_attn, tp_attn, tp_rank, tp_size, qkv_mode=QKVMode.FUSE_QKV): + """Copy the appropriate TP shard of ref_attn's weights into tp_attn. + + Column-parallel (QKV): split output dim (dim 0) + Row-parallel (output): split input dim (dim 1) + RMSNorm (TP-enabled): split weight + """ + with torch.no_grad(): + if qkv_mode == QKVMode.FUSE_QKV: + # Fused QKV: weight is [q_dim + 2*kv_dim, hidden_size] + full_w = ref_attn.qkv_proj.weight.data + q_dim = ref_attn.q_dim + kv_dim = ref_attn.kv_dim + q_w, k_w, v_w = full_w.split([q_dim, kv_dim, kv_dim], dim=0) + + q_shard = _shard_dim0(q_w, tp_rank, tp_size) + k_shard = _shard_dim0(k_w, tp_rank, tp_size) + v_shard = _shard_dim0(v_w, tp_rank, tp_size) + tp_attn.qkv_proj.weight.data.copy_(torch.cat([q_shard, k_shard, v_shard], dim=0)) + + if ref_attn.qkv_proj.bias is not None: + full_b = ref_attn.qkv_proj.bias.data + q_b, k_b, v_b = full_b.split([q_dim, kv_dim, kv_dim], dim=0) + tp_attn.qkv_proj.bias.data.copy_( + torch.cat( + [ + _shard_dim0(q_b, tp_rank, tp_size), + _shard_dim0(k_b, tp_rank, tp_size), + _shard_dim0(v_b, tp_rank, tp_size), + ], + dim=0, + ) + ) + else: + for name in ("to_q", "to_k", "to_v"): + ref_proj = getattr(ref_attn, name) + tp_proj = getattr(tp_attn, name) + tp_proj.weight.data.copy_(_shard_dim0(ref_proj.weight.data, tp_rank, tp_size)) + if ref_proj.bias is not None: + tp_proj.bias.data.copy_(_shard_dim0(ref_proj.bias.data, tp_rank, tp_size)) + + # Output projection: row-parallel (split input dim = dim 1) + ref_out = ref_attn.to_out[0] + tp_out = tp_attn.to_out[0] + shard_size = math.ceil(ref_out.weight.shape[1] / tp_size) + start = tp_rank * shard_size + end = min(start + shard_size, ref_out.weight.shape[1]) + tp_out.weight.data.copy_(ref_out.weight.data[:, start:end].contiguous()) + if ref_out.bias is not None: + tp_out.bias.data.copy_(ref_out.bias.data) + + # QK norm weights (if TP-enabled, they're sharded) + if hasattr(ref_attn, "norm_q") and hasattr(tp_attn, "norm_q"): + if tp_attn.norm_q.enable_tp: + shard_size = ref_attn.norm_q.weight.shape[0] // tp_size + start = tp_rank * shard_size + end = start + shard_size + tp_attn.norm_q.weight.data.copy_(ref_attn.norm_q.weight.data[start:end]) + tp_attn.norm_k.weight.data.copy_(ref_attn.norm_k.weight.data[start:end]) + else: + tp_attn.norm_q.weight.data.copy_(ref_attn.norm_q.weight.data) + tp_attn.norm_k.weight.data.copy_(ref_attn.norm_k.weight.data) + + +def _shard_dim0(tensor, tp_rank, tp_size): + """Shard a tensor along dim 0 (works for both 1D bias and 2D weight).""" + shard_size = math.ceil(tensor.shape[0] / tp_size) + start = tp_rank * shard_size + end = min(start + shard_size, tensor.shape[0]) + return tensor[start:end].contiguous() + + +# ============================================================================= +# Manual F.sdpa reference +# ============================================================================= + + +def _wb(module): + """Return (weight, bias_or_None) for a Linear module.""" + return module.weight.data, (module.bias.data if module.bias is not None else None) + + +def _sdpa_ref(q, k, v, out_weight, out_bias, num_heads, head_dim): + """Reshape Q/K/V [B,S,H*D] → run F.sdpa → output projection.""" + B, S, _ = q.shape + q = q.view(B, S, num_heads, head_dim).transpose(1, 2) + k = k.view(B, S, num_heads, head_dim).transpose(1, 2) + v = v.view(B, S, num_heads, head_dim).transpose(1, 2) + + out = F.scaled_dot_product_attention(q, k, v, scale=1.0 / math.sqrt(head_dim)) + out = out.transpose(1, 2).reshape(B, S, num_heads * head_dim) + + return F.linear(out, out_weight, out_bias) + + +def _manual_attention(x, qkv_weight, qkv_bias, out_weight, out_bias, num_heads, head_dim): + """Manual F.sdpa reference with fused QKV projection.""" + q_dim = num_heads * head_dim + qkv = F.linear(x, qkv_weight, qkv_bias) + q, k, v = qkv.split([q_dim, q_dim, q_dim], dim=-1) + return _sdpa_ref(q, k, v, out_weight, out_bias, num_heads, head_dim) + + +def _manual_attention_separate(x, q_w, q_b, k_w, k_b, v_w, v_b, out_w, out_b, num_heads, head_dim): + """Manual F.sdpa reference with separate Q/K/V projections.""" + q = F.linear(x, q_w, q_b) + k = F.linear(x, k_w, k_b) + v = F.linear(x, v_w, v_b) + return _sdpa_ref(q, k, v, out_w, out_b, num_heads, head_dim) + + +def _manual_attention_with_norm( + x, qkv_weight, qkv_bias, out_weight, out_bias, norm_q_w, norm_k_w, eps, num_heads, head_dim +): + """Manual F.sdpa reference with fused QKV + RMSNorm on Q/K.""" + q_dim = num_heads * head_dim + qkv = F.linear(x, qkv_weight, qkv_bias) + q, k, v = qkv.split([q_dim, q_dim, q_dim], dim=-1) + + def _rms_norm(t, weight): + t_f32 = t.to(torch.float32) + var = t_f32.pow(2).mean(-1, keepdim=True) + return weight * (t_f32 * torch.rsqrt(var + eps)).to(t.dtype) + + q = _rms_norm(q, norm_q_w) + k = _rms_norm(k, norm_k_w) + return _sdpa_ref(q, k, v, out_weight, out_bias, num_heads, head_dim) + + +# ============================================================================= +# Reusable test body +# ============================================================================= + + +def _run_tp_with_params( + rank, world_size, batch, seq, hidden_size, num_heads, qkv_mode=QKVMode.FUSE_QKV, qk_norm=False +): + """Run a single TP-vs-F.sdpa comparison with the given dimensions.""" + tp_size = world_size + device = torch.device(f"cuda:{rank}") + head_dim = hidden_size // num_heads + + vgm = _setup_vgm(rank, world_size, tp_size=tp_size) + + # Build ref module for weight extraction + config_ref = _make_config(world_size=1, rank=0, tp_size=1) + attn_ref = Attention( + hidden_size, + num_heads, + qkv_mode=qkv_mode, + qk_norm=qk_norm, + qk_norm_mode="full", + config=config_ref, + ).to(device) + _broadcast_params(attn_ref) + + # Build TP module, shard weights from ref + config_tp = _make_config(vgm=vgm) + attn_tp = Attention( + hidden_size, + num_heads, + qkv_mode=qkv_mode, + qk_norm=qk_norm, + qk_norm_mode="full", + config=config_tp, + ).to(device) + _shard_tp_weights(attn_ref, attn_tp, rank, tp_size, qkv_mode=qkv_mode) + + # Same input on all ranks + torch.manual_seed(42) + x = torch.randn(batch, seq, hidden_size, device=device, dtype=torch.bfloat16) + + # Build manual F.sdpa reference + out_w, out_b = _wb(attn_ref.to_out[0]) + + if qkv_mode == QKVMode.FUSE_QKV: + qkv_w, qkv_b = _wb(attn_ref.qkv_proj) + if qk_norm: + ref_out = _manual_attention_with_norm( + x, + qkv_w, + qkv_b, + out_w, + out_b, + attn_ref.norm_q.weight.data, + attn_ref.norm_k.weight.data, + attn_ref.norm_q.variance_epsilon, + num_heads, + head_dim, + ) + else: + ref_out = _manual_attention(x, qkv_w, qkv_b, out_w, out_b, num_heads, head_dim) + else: + ref_out = _manual_attention_separate( + x, + *_wb(attn_ref.to_q), + *_wb(attn_ref.to_k), + *_wb(attn_ref.to_v), + out_w, + out_b, + num_heads, + head_dim, + ) + + tp_out = attn_tp(x) + + torch.testing.assert_close(tp_out, ref_out, rtol=1e-2, atol=1e-2) + + +# ============================================================================= +# Test logic functions (module-level for pickling by mp.spawn) +# ============================================================================= + + +def _logic_tp_fused_qkv(rank, world_size): + """TP with fused QKV matches F.sdpa reference.""" + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=256, num_heads=8) + + +def _logic_tp_separate_qkv(rank, world_size): + """TP with separate Q/K/V projections matches F.sdpa reference.""" + _run_tp_with_params( + rank, + world_size, + batch=2, + seq=16, + hidden_size=256, + num_heads=8, + qkv_mode=QKVMode.SEPARATE_QKV, + ) + + +def _logic_tp_with_qk_norm(rank, world_size): + """TP with QK norm (full mode) matches F.sdpa reference.""" + _run_tp_with_params( + rank, world_size, batch=2, seq=16, hidden_size=256, num_heads=8, qk_norm=True + ) + + +def _logic_tp_batch_size_1(rank, world_size): + _run_tp_with_params(rank, world_size, batch=1, seq=16, hidden_size=256, num_heads=8) + + +def _logic_tp_hidden_128(rank, world_size): + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=128, num_heads=4) + + +def _logic_tp_hidden_512(rank, world_size): + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=512, num_heads=4) + + +def _logic_tp_heads_not_divisible(rank, world_size): + """TP when num_heads % tp_size != 0. Expected to fail until uneven sharding is implemented.""" + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=320, num_heads=5) + + +def _logic_tp_world_size_4(rank, world_size): + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=512, num_heads=16) + + +def _logic_tp_ulysses_combined(rank, world_size, ulysses_size, tp_size): + """TP + Ulysses combined matches F.sdpa reference on the full sequence. + + 4 GPUs: tp_size=2, ulysses_size=2. + Ulysses shards the sequence; TP shards the heads/weights. + """ + assert tp_size * ulysses_size == world_size + device = torch.device(f"cuda:{rank}") + + hidden_size = 512 + num_heads = 16 + head_dim = hidden_size // num_heads + batch = 2 + seq_per_rank = 8 + seq_full = seq_per_rank * ulysses_size + + vgm = _setup_vgm(rank, world_size, tp_size=tp_size, ulysses_size=ulysses_size) + + # Build ref module for weight extraction + config_ref = _make_config(world_size=1, rank=0, tp_size=1) + attn_ref = Attention(hidden_size, num_heads, qk_norm=False, config=config_ref).to(device) + _broadcast_params(attn_ref) + + qkv_w, qkv_b = _wb(attn_ref.qkv_proj) + out_w, out_b = _wb(attn_ref.to_out[0]) + + # Build combined TP + Ulysses module + config_combined = _make_config(vgm=vgm) + attn_combined = Attention(hidden_size, num_heads, qk_norm=False, config=config_combined).to( + device + ) + _shard_tp_weights(attn_ref, attn_combined, vgm.tp_rank, tp_size) + + # Same full input on all ranks + torch.manual_seed(42) + x_full = torch.randn(batch, seq_full, hidden_size, device=device, dtype=torch.bfloat16) + + # F.sdpa reference on full sequence + ref_out = _manual_attention(x_full, qkv_w, qkv_b, out_w, out_b, num_heads, head_dim) + + # Each Ulysses rank takes its sequence shard + ulysses_rank = vgm.ulysses_rank + x_shard = x_full[ + :, ulysses_rank * seq_per_rank : (ulysses_rank + 1) * seq_per_rank + ].contiguous() + expected_shard = ref_out[:, ulysses_rank * seq_per_rank : (ulysses_rank + 1) * seq_per_rank] + + combined_out = attn_combined(x_shard) + + torch.testing.assert_close(combined_out, expected_shard, rtol=1e-2, atol=1e-2) + + +# ============================================================================= +# Test classes +# ============================================================================= + + +class TestTPAttention: + """Core correctness tests for TP-sharded Attention.""" + + def test_tp_fused_qkv(self): + _run(2, _logic_tp_fused_qkv) + + def test_tp_separate_qkv(self): + _run(2, _logic_tp_separate_qkv) + + def test_tp_with_qk_norm(self): + _run(2, _logic_tp_with_qk_norm) + + +class TestTPAttentionEdgeCases: + """Edge case tests for TP attention.""" + + def test_batch_size_1(self): + _run(2, _logic_tp_batch_size_1) + + def test_hidden_128(self): + _run(2, _logic_tp_hidden_128) + + def test_hidden_512(self): + _run(2, _logic_tp_hidden_512) + + @pytest.mark.xfail(reason="Uneven head sharding not yet implemented", raises=Exception) + def test_tp_heads_not_divisible(self): + _run(2, _logic_tp_heads_not_divisible) + + def test_tp_world_size_4(self): + _run(4, _logic_tp_world_size_4) + + +class TestTPUlyssesCombined: + """Tests for combined TP + Ulysses parallelism.""" + + def test_tp_ulysses_combined(self): + tp_size = 2 + ulysses_size = 2 + world = ulysses_size * tp_size + _run(world, _logic_tp_ulysses_combined, ulysses_size, tp_size) + + def test_tp_2_ulysses_4(self): + tp_size = 2 + ulysses_size = 4 + world = ulysses_size * tp_size + _run(world, _logic_tp_ulysses_combined, ulysses_size, tp_size) + + def test_tp_4_ulysses_2(self): + tp_size = 4 + ulysses_size = 2 + world = ulysses_size * tp_size + _run(world, _logic_tp_ulysses_combined, ulysses_size, tp_size) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py new file mode 100644 index 000000000000..ecd98ba203ff --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py @@ -0,0 +1,594 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-GPU tests for WAN Tensor Parallelism (TP). + +Tests that WAN T2V and I2V transformers produce correct outputs when using +TP (sharding weights across GPUs), and combined TP + Ulysses. + +Run with: + pytest tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py -v +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" + +from types import SimpleNamespace +from typing import Callable + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +try: + from tensorrt_llm._torch.visual_gen.config import ( + AttentionConfig, + DiffusionModelConfig, + TorchCompileConfig, + create_attention_metadata_state, + ) + from tensorrt_llm._torch.visual_gen.mapping import VisualGenMapping + from tensorrt_llm._utils import get_free_port + from tensorrt_llm.models.modeling_utils import QuantConfig + + MODULES_AVAILABLE = True +except ImportError: + MODULES_AVAILABLE = False + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + """Clean up TLLM_DISABLE_MPI env var after tests complete.""" + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +# ============================================================================= +# Distributed helpers (same pattern as test_flux_tp.py) +# ============================================================================= + + +def init_distributed_worker(rank: int, world_size: int, backend: str = "nccl", port: int = 29500): + """Initialize distributed environment for a worker process.""" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + + torch.cuda.set_device(rank % torch.cuda.device_count()) + dist.init_process_group(backend=backend, rank=rank, world_size=world_size) + + +def cleanup_distributed(): + """Clean up distributed environment.""" + # Reset the DeviceMesh singleton before destroying the process group + # to prevent NCCL process group destructor segfaults during interpreter exit. + try: + from tensorrt_llm._torch.device_mesh import DeviceMeshTopologyImpl + + DeviceMeshTopologyImpl.device_mesh = None + DeviceMeshTopologyImpl.tp_mesh = None + except ImportError: + pass + if dist.is_initialized(): + dist.barrier() + dist.destroy_process_group() + + +def _distributed_worker(rank, world_size, backend, test_fn, port): + """Worker function that runs in each process. Module-level for pickling.""" + try: + init_distributed_worker(rank, world_size, backend, port) + test_fn(rank, world_size) + except Exception as e: + print(f"Rank {rank} failed with error: {e}") + raise + finally: + cleanup_distributed() + + +def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = True): + """Run a test function in a distributed environment.""" + if not MODULES_AVAILABLE: + pytest.skip("Required modules not available") + + if use_cuda and torch.cuda.device_count() < world_size: + pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") + + backend = "nccl" if use_cuda else "gloo" + port = get_free_port() + + mp.spawn( + _distributed_worker, args=(world_size, backend, test_fn, port), nprocs=world_size, join=True + ) + + +# ============================================================================= +# Model configs +# ============================================================================= + +# Small WAN T2V config (4 heads, 64 head_dim = 256 hidden) +_WAN_T2V_TEST_CONFIG = dict( + num_attention_heads=4, + attention_head_dim=64, + num_layers=2, + in_channels=16, + out_channels=16, + text_dim=128, + freq_dim=64, + patch_size=[1, 2, 2], + ffn_dim=512, + eps=1e-6, + cross_attn_norm=True, +) + +# Small WAN I2V config (same base + image embedding and added KV projections) +_WAN_I2V_TEST_CONFIG = dict( + **_WAN_T2V_TEST_CONFIG, + image_dim=64, # image embedder: image_dim -> hidden_size + added_kv_proj_dim=256, # add_k_proj input dim = hidden_size (image embeds projected to hidden_size before blocks) +) + + +# ============================================================================= +# Model config + weight helpers +# ============================================================================= + + +def _make_model_config(pretrained_dict, tp_size=1, ulysses_size=1, backend="VANILLA"): + """Create DiffusionModelConfig for testing with TP and/or Ulysses.""" + pretrained_config = SimpleNamespace(**pretrained_dict) + ws = tp_size * ulysses_size + if ws > 1 and dist.is_initialized(): + ws = dist.get_world_size() + rk = dist.get_rank() + else: + rk = 0 + vgm = VisualGenMapping(world_size=ws, rank=rk, tp_size=tp_size, ulysses_size=ulysses_size) + + config = DiffusionModelConfig( + pretrained_config=pretrained_config, + quant_config=QuantConfig(), + torch_compile=TorchCompileConfig(enable=False), + attention=AttentionConfig(backend=backend), + visual_gen_mapping=vgm, + cache=None, + attention_metadata_state=( + create_attention_metadata_state() if backend.upper() == "TRTLLM" else None + ), + skip_create_weights_in_init=False, + ) + config.mapping = vgm.to_llm_mapping() + return config + + +def _stabilize_model_weights(model): + """Reinitialize model weights for stable BF16 forward pass. + + Random default init (std~1.0) causes BF16 overflow through multiple + transformer blocks. Use small uniform init that keeps activations bounded. + """ + with torch.no_grad(): + for name, p in model.named_parameters(): + if p.ndim >= 2: + fan_in = p.shape[1] if p.ndim >= 2 else p.shape[0] + std = 0.02 / max(1.0, fan_in**0.5) + p.data.uniform_(-std, std) + else: + p.data.uniform_(-0.01, 0.01) + + +# ============================================================================= +# TP weight sharding helpers +# ============================================================================= + + +def _shard_dim0(tensor, tp_rank, tp_size): + """Shard a tensor along dim 0.""" + chunk = tensor.shape[0] // tp_size + return tensor[tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() + + +def _shard_dim1(tensor, tp_rank, tp_size): + """Shard a tensor along dim 1.""" + chunk = tensor.shape[1] // tp_size + return tensor[:, tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() + + +def _shard_fused_qkv(tensor, tp_rank, tp_size, q_dim, kv_dim): + """Shard a fused QKV weight [q_dim + 2*kv_dim, ...] preserving Q/K/V structure.""" + q, k, v = tensor.split([q_dim, kv_dim, kv_dim], dim=0) + return torch.cat( + [ + _shard_dim0(q, tp_rank, tp_size), + _shard_dim0(k, tp_rank, tp_size), + _shard_dim0(v, tp_rank, tp_size), + ], + dim=0, + ) + + +def _shard_fused_gate_up(tensor, tp_rank, tp_size): + """Shard a fused gate_up weight [2*intermediate, ...] preserving gate/up structure.""" + half = tensor.shape[0] // 2 + gate, up = tensor.split([half, half], dim=0) + return torch.cat( + [ + _shard_dim0(gate, tp_rank, tp_size), + _shard_dim0(up, tp_rank, tp_size), + ], + dim=0, + ) + + +def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): + """Copy weights from a TP=1 reference model into a TP model with correct sharding. + + Handles column-parallel (QKV, MLP up/gate), row-parallel (output projs), + fused QKV/gate_up weights, and replicated parameters (norms, embeddings). + """ + ref_params = dict(ref_model.named_parameters()) + + with torch.no_grad(): + for tp_name, tp_param in tp_model.named_parameters(): + if tp_name not in ref_params: + continue + + ref_param = ref_params[tp_name] + + if tp_param.shape == ref_param.shape: + # Replicated parameter (norms, embeddings, etc.) + tp_param.data.copy_(ref_param.data) + elif tp_param.ndim >= 2 and tp_param.shape[1] == ref_param.shape[1]: + # Column parallel: dim 0 is smaller (output dim sharded) + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + q_dim = ref_param.shape[0] // 3 + tp_param.data.copy_( + _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim >= 2 and tp_param.shape[0] == ref_param.shape[0]: + # Row parallel: dim 1 is smaller (input dim sharded) + tp_param.data.copy_(_shard_dim1(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim == 1 and tp_param.shape[0] < ref_param.shape[0]: + # 1D bias for column parallel + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + q_dim = ref_param.shape[0] // 3 + tp_param.data.copy_( + _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) + else: + raise ValueError( + f"Cannot shard {tp_name}: ref={ref_param.shape}, tp={tp_param.shape}" + ) + + +# ============================================================================= +# WAN T2V test logic functions (module-level for pickling) +# ============================================================================= + + +def _logic_wan_t2v_tp_forward(rank, world_size): + """WAN T2V transformer forward with TP — verify shape and no NaN/Inf.""" + from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel + + device = torch.device(f"cuda:{rank}") + torch.manual_seed(42) + + model_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=world_size) + model = WanTransformer3DModel(model_config).to(device).to(torch.bfloat16) + _stabilize_model_weights(model) + + batch = 1 + T, H, W = 2, 4, 4 # Small spatial dims (must be divisible by patch_size [1,2,2]) + in_channels = 16 + txt_seq = 8 + + torch.manual_seed(100) + hidden_states = ( + torch.randn(batch, in_channels, T, H, W, device=device, dtype=torch.bfloat16) * 0.1 + ) + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 128, device=device, dtype=torch.bfloat16) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=torch.bfloat16) + + with torch.no_grad(): + output = model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + ) + + assert output.shape == (batch, in_channels, T, H, W), ( + f"Rank {rank}: Expected shape {(batch, in_channels, T, H, W)}, got {output.shape}" + ) + assert not torch.isnan(output).any(), f"Rank {rank}: NaN in output" + assert not torch.isinf(output).any(), f"Rank {rank}: Inf in output" + + +def _logic_wan_t2v_tp_vs_single_gpu(rank, world_size): + """WAN T2V: TP 2-GPU output matches single-GPU reference.""" + from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + + batch = 1 + T, H, W = 2, 4, 4 + in_channels = 16 + txt_seq = 8 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=1) + ref_model = WanTransformer3DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create TP model and copy sharded weights from ref + torch.manual_seed(123) + tp_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=world_size) + tp_model = WanTransformer3DModel(tp_config).to(device).to(compute_dtype) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + + # Same inputs on all ranks + torch.manual_seed(456) + hidden_states = ( + torch.randn(batch, in_channels, T, H, W, device=device, dtype=compute_dtype) * 0.1 + ) + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 128, device=device, dtype=compute_dtype) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + ) + tp_output = tp_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + ) + + torch.testing.assert_close( + tp_output, + ref_output, + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: WAN T2V TP output differs from single-GPU reference", + ) + + +def _logic_wan_t2v_tp_ulysses_vs_single_gpu(rank, world_size): + """WAN T2V: TP=2 + Ulysses=2 (4 GPUs) output matches single-GPU reference.""" + from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + tp_size = 2 + ulysses_size = 2 + + batch = 1 + T, H, W = 2, 4, 4 # seq_len = T * (H/2) * (W/2) = 2*2*2 = 8, divisible by ulysses_size=2 + in_channels = 16 + txt_seq = 8 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=1, ulysses_size=1) + ref_model = WanTransformer3DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create combined TP + Ulysses model and copy sharded weights + torch.manual_seed(123) + combined_config = _make_model_config( + _WAN_T2V_TEST_CONFIG, tp_size=tp_size, ulysses_size=ulysses_size + ) + combined_model = WanTransformer3DModel(combined_config).to(device).to(compute_dtype) + vgm = combined_config.visual_gen_mapping + _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size) + + # Same inputs on all ranks (Ulysses shards at runtime) + torch.manual_seed(456) + hidden_states = ( + torch.randn(batch, in_channels, T, H, W, device=device, dtype=compute_dtype) * 0.1 + ) + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 128, device=device, dtype=compute_dtype) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + ) + combined_output = combined_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + ) + + torch.testing.assert_close( + combined_output, + ref_output, + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: WAN T2V TP+Ulysses output differs from single-GPU reference", + ) + + +# ============================================================================= +# WAN I2V test logic functions (module-level for pickling) +# ============================================================================= + + +def _logic_wan_i2v_tp_forward(rank, world_size): + """WAN I2V transformer forward with TP — verify shape and no NaN/Inf.""" + from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel + + device = torch.device(f"cuda:{rank}") + torch.manual_seed(42) + + model_config = _make_model_config(_WAN_I2V_TEST_CONFIG, tp_size=world_size) + model = WanTransformer3DModel(model_config).to(device).to(torch.bfloat16) + _stabilize_model_weights(model) + + batch = 1 + T, H, W = 2, 4, 4 + in_channels = 16 + txt_seq = 8 + img_seq = 4 + + torch.manual_seed(100) + hidden_states = ( + torch.randn(batch, in_channels, T, H, W, device=device, dtype=torch.bfloat16) * 0.1 + ) + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 128, device=device, dtype=torch.bfloat16) * 0.1 + ) + encoder_hidden_states_image = ( + torch.randn(batch, img_seq, 64, device=device, dtype=torch.bfloat16) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=torch.bfloat16) + + with torch.no_grad(): + output = model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_image=encoder_hidden_states_image, + ) + + assert output.shape == (batch, in_channels, T, H, W), ( + f"Rank {rank}: Expected shape {(batch, in_channels, T, H, W)}, got {output.shape}" + ) + assert not torch.isnan(output).any(), f"Rank {rank}: NaN in output" + assert not torch.isinf(output).any(), f"Rank {rank}: Inf in output" + + +def _logic_wan_i2v_tp_vs_single_gpu(rank, world_size): + """WAN I2V: TP 2-GPU output matches single-GPU reference.""" + from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel + + device = torch.device(f"cuda:{rank}") + compute_dtype = torch.bfloat16 + + batch = 1 + T, H, W = 2, 4, 4 + in_channels = 16 + txt_seq = 8 + img_seq = 4 + + # Create single-GPU reference model + torch.manual_seed(123) + ref_config = _make_model_config(_WAN_I2V_TEST_CONFIG, tp_size=1) + ref_model = WanTransformer3DModel(ref_config).to(device).to(compute_dtype) + _stabilize_model_weights(ref_model) + + # Create TP model and copy sharded weights from ref + torch.manual_seed(123) + tp_config = _make_model_config(_WAN_I2V_TEST_CONFIG, tp_size=world_size) + tp_model = WanTransformer3DModel(tp_config).to(device).to(compute_dtype) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + + # Same inputs on all ranks + torch.manual_seed(456) + hidden_states = ( + torch.randn(batch, in_channels, T, H, W, device=device, dtype=compute_dtype) * 0.1 + ) + encoder_hidden_states = ( + torch.randn(batch, txt_seq, 128, device=device, dtype=compute_dtype) * 0.1 + ) + encoder_hidden_states_image = ( + torch.randn(batch, img_seq, 64, device=device, dtype=compute_dtype) * 0.1 + ) + timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) + + with torch.no_grad(): + ref_output = ref_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_image=encoder_hidden_states_image, + ) + tp_output = tp_model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_image=encoder_hidden_states_image, + ) + + torch.testing.assert_close( + tp_output, + ref_output, + rtol=1e-2, + atol=1e-2, + msg=f"Rank {rank}: WAN I2V TP output differs from single-GPU reference", + ) + + +# ============================================================================= +# Test classes +# ============================================================================= + + +class TestWanT2VTP: + """Tensor parallelism tests for WAN T2V transformer.""" + + def test_wan_t2v_tp_forward(self): + """WAN T2V TP forward: correct output shape and no NaN/Inf.""" + run_test_in_distributed(world_size=2, test_fn=_logic_wan_t2v_tp_forward) + + def test_wan_t2v_tp_vs_single_gpu(self): + """WAN T2V TP 2-GPU output matches single-GPU reference.""" + run_test_in_distributed(world_size=2, test_fn=_logic_wan_t2v_tp_vs_single_gpu) + + +class TestWanT2VTPUlyssesCombined: + """Combined TP + Ulysses tests for WAN T2V transformer.""" + + def test_wan_t2v_tp_ulysses_vs_single_gpu(self): + """WAN T2V TP=2 + Ulysses=2 (4 GPUs) matches single-GPU reference.""" + run_test_in_distributed(world_size=4, test_fn=_logic_wan_t2v_tp_ulysses_vs_single_gpu) + + +class TestWanI2VTP: + """Tensor parallelism tests for WAN I2V transformer.""" + + def test_wan_i2v_tp_forward(self): + """WAN I2V TP forward: correct output shape and no NaN/Inf.""" + run_test_in_distributed(world_size=2, test_fn=_logic_wan_i2v_tp_forward) + + def test_wan_i2v_tp_vs_single_gpu(self): + """WAN I2V TP 2-GPU output matches single-GPU reference.""" + run_test_in_distributed(world_size=2, test_fn=_logic_wan_i2v_tp_vs_single_gpu) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 26c099f52ddac75a00dcaa6f172eff334f14be5f Mon Sep 17 00:00:00 2001 From: chenfeiz0326 Date: Sun, 31 May 2026 09:49:42 +0800 Subject: [PATCH 145/308] [None][test] Add TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS in Spec Decoding Perf Test (#14438) Signed-off-by: Chenfei Zhang --- jenkins/L0_Test.groovy | 19 +- .../perf/aggregated/slurm_launch_draft.sh | 7 +- .../perf/disaggregated/slurm_launch_draft.sh | 9 +- jenkins/scripts/perf/disaggregated/submit.py | 466 ------------ jenkins/scripts/perf/local/submit.py | 77 +- jenkins/scripts/perf/submit.py | 614 ++++++++++++++++ .../defs/perf/README_test_perf_sanity.md | 30 +- .../integration/defs/perf/test_perf_sanity.py | 665 ++++++++++++++---- .../aggregated/config_database_b200_nvl.yaml | 20 + .../aggregated/config_database_h200_sxm.yaml | 9 + .../deepseek_r1_fp4_v2_2_nodes_blackwell.yaml | 3 + ...eek_r1_fp4_v2_2_nodes_grace_blackwell.yaml | 3 + .../deepseek_r1_fp4_v2_blackwell.yaml | 4 + .../deepseek_r1_fp4_v2_grace_blackwell.yaml | 9 + .../aggregated/deepseek_r1_fp8_blackwell.yaml | 4 + .../deepseek_v32_fp4_blackwell.yaml | 2 + .../deepseek_v32_fp4_grace_blackwell.yaml | 4 + ...eek_r1_fp4_v2_2_nodes_grace_blackwell.yaml | 3 + .../gpt_oss_120b_fp4_grace_blackwell.yaml | 1 + .../host_perf_llama8b_spec_decode.yaml | 2 + ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...tx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...tx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...tx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...tx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...tx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...tx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 7 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml | 5 - ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ...1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 5 - ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 7 +- ..._ctx1_tp2_gen1_tp2_eplb0_mtp0_ccb-UCX.yaml | 2 - ..._tp2_gen1_tep4_eplb0_mtp0_ccb-DEFAULT.yaml | 2 - ...1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT.yaml | 2 - ...x1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...tx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...tx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml | 2 - ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...n4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml | 4 +- ...p16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...n3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml | 2 - ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 14 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml | 14 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml | 12 +- ..._dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml | 12 +- ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 - ..._ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml | 2 - ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...tx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...tx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...x1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...x1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...n4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml | 2 - ...p16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...tx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...tx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...tx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...x1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 2 +- ...en1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml | 2 +- ...en1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml | 2 +- ...32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml | 2 - ...plb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml | 2 - ...6_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml | 4 +- ...8_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml | 2 - ...2_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml | 4 +- ...32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml | 2 - ...6_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...8_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml | 2 - ...2_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml | 4 +- ...en1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml | 2 +- ...en1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml | 2 +- ...tx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...tx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...x1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 14 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 14 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml | 14 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 2 - ...dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml | 14 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml | 12 +- ..._dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml | 12 +- ...1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...x1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...tx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...x1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 12 +- ...x1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml | 12 +- ...x1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml | 14 +- ...p16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...6_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml | 4 +- ...2_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml | 4 +- ...6_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml | 4 +- ...2_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml | 4 +- 209 files changed, 2140 insertions(+), 1404 deletions(-) delete mode 100644 jenkins/scripts/perf/disaggregated/submit.py create mode 100755 jenkins/scripts/perf/submit.py diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 4c8297debbf6..26ca41050ff8 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -954,7 +954,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // Create a unique suffix for the job name String customSuffix = "${env.BUILD_TAG}-${UUID.randomUUID().toString().replaceAll("-", "").substring(0, 6)}".toLowerCase() def jobUID = "${cluster.host}-multi_node_test-${customSuffix}" - def disaggMode = stageName.contains("Disagg-PerfSanity") + def disaggMultiNodeMode = stageName.contains("Disagg-PerfSanity") + def aggMultiNodeMode = !disaggMultiNodeMode && nodeCount > 1 && stageName.contains("PerfSanity") Utils.exec(pipeline, script: "env | sort && pwd && ls -alh") @@ -1096,7 +1097,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // Generate Job Launch Script def container = LLM_DOCKER_IMAGE.replace("urm.nvidia.com/", "urm.nvidia.com#") def mounts = getMountListForSlurmTest(cluster, true).join(",") - String[] taskArgs = getNodeArgs(nodeCount, gpuCount, disaggMode) + String[] taskArgs = getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode) if (taskArgs == null) { error "Invalid Slurm test stage name is set" } @@ -1216,24 +1217,24 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG ${srunPrologue} """.replaceAll("(?m)^\\s*", "") - if (disaggMode) { + if (disaggMultiNodeMode || aggMultiNodeMode) { def scriptLaunchPrefixPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch_prefix.sh") def scriptLaunchSrunArgsPathLocal = Utils.createTempLocation(pipeline, "./slurm_srun_args.txt") - def scriptLaunchDraftPathLocal = "${llmSrcLocal}/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh" - def scriptSubmitLocalPath = "${llmSrcLocal}/jenkins/scripts/perf/disaggregated/submit.py" + // The unified submit.py handles both agg and disagg; only the + // draft launch script differs between the two paths. + def scriptLaunchDraftPathLocal = disaggMultiNodeMode + ? "${llmSrcLocal}/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh" + : "${llmSrcLocal}/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh" + def scriptSubmitLocalPath = "${llmSrcLocal}/jenkins/scripts/perf/submit.py" - // Remove unrelated mpi envs before passing to submit.py - // because mpi env will be set in slurm_launch.sh for disagg. srunArgs.removeAll { it == "--mpi=pmi2" || it == "--mpi=pmix" } pipeline.writeFile(file: scriptLaunchPrefixPathLocal, text: scriptLaunchPrefix) pipeline.writeFile(file: scriptLaunchSrunArgsPathLocal, text: srunArgs.join(" ")) - // Output is the corresponding scriptLaunchPathLocal script under the disaggMode sh """ pip3 install pyyaml && \\ python3 ${scriptSubmitLocalPath} \\ - --run-ci \\ --llm-src ${llmSrcLocal} \\ --test-list ${testListPathLocal} \\ --draft-launch-sh ${scriptLaunchDraftPathLocal} \\ diff --git a/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh b/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh index 278d6ec7dfd5..0421439d8236 100644 --- a/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh +++ b/jenkins/scripts/perf/aggregated/slurm_launch_draft.sh @@ -6,17 +6,18 @@ cleanup_on_failure() { } mkdir -p $jobWorkspace +mkdir -p "$testOutputDir" chmod +x $runScript # Run aggregated test echo "Starting aggregated test..." -world_size=$((totalNodes * gpusPerNodePerServer)) -if ! srun "${srunArgs[@]}" --kill-on-bad-exit=1 \ +world_size=${world_size:-$((totalNodes * gpusPerNodePerServer))} +if ! srun "${srunArgs[@]}" --mpi=pmi2 --kill-on-bad-exit=1 \ -N $totalNodes \ --ntasks=$world_size \ --ntasks-per-node=$gpusPerNodePerServer \ $runScript; then - cleanup_on_failure "Aggregated test failed. Check logs in ${jobWorkspace} for details" + cleanup_on_failure "Aggregated test failed. See slurm-${SLURM_JOB_ID}.out" fi echo "Aggregated test completed successfully" diff --git a/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh b/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh index 631b48eb2edf..a3712ff6d114 100644 --- a/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh +++ b/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh @@ -6,6 +6,7 @@ cleanup_on_failure() { } mkdir -p $jobWorkspace +mkdir -p "$testOutputDir" chmod +x $runScript chmod +x $installScript @@ -26,7 +27,7 @@ for i in $(seq 0 $((numGenServers - 1))); do -N $nodesPerGenServer \ --ntasks=$gen_world_size \ --ntasks-per-node=$gpusPerNodePerGenServer \ - $runScript &> $jobWorkspace/gen_server_$i.log & + $runScript &> $testOutputDir/gen_server_$i.log & echo "Started gen server $i" sleep 5 # Wait for pyxis container namespace initialization to avoid race condition done @@ -42,7 +43,7 @@ if [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" != "1" ]; then -N $nodesPerCtxServer \ --ntasks=$ctx_world_size \ --ntasks-per-node=$gpusPerNodePerCtxServer \ - $runScript &> $jobWorkspace/ctx_server_$i.log & + $runScript &> $testOutputDir/ctx_server_$i.log & echo "Started ctx server $i" sleep 5 # Wait for pyxis container namespace initialization to avoid race condition done @@ -60,7 +61,7 @@ srun "${srunArgs[@]}" --kill-on-bad-exit=1 --overlap \ -N 1 \ --ntasks=1 \ --ntasks-per-node=1 \ - $runScript &> $jobWorkspace/disagg_server.log & + $runScript &> $testOutputDir/disagg_server.log & echo "Started disagg server" sleep 5 # Wait for pyxis container namespace initialization to avoid race condition @@ -73,7 +74,7 @@ if ! srun "${srunArgs[@]}" --kill-on-bad-exit=1 --overlap \ --ntasks=1 \ --ntasks-per-node=1 \ $runScript; then - cleanup_on_failure "Benchmark failed. Check logs in ${jobWorkspace} for details" + cleanup_on_failure "Benchmark failed. See slurm-${SLURM_JOB_ID}.out" fi echo "Disagg server and benchmark completed successfully" diff --git a/jenkins/scripts/perf/disaggregated/submit.py b/jenkins/scripts/perf/disaggregated/submit.py deleted file mode 100644 index a4cbfc607ae0..000000000000 --- a/jenkins/scripts/perf/disaggregated/submit.py +++ /dev/null @@ -1,466 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os - -import yaml - -AGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/aggregated" -DISAGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/disaggregated" - - -def get_hardware_config(config, benchmark_mode): - hardware = config.get("hardware", {}) - worker_config = config.get("worker_config", {}) - - num_ctx_servers = ( - 0 if "gen_only_no_context" in benchmark_mode else hardware.get("num_ctx_servers") - ) - num_gen_servers = hardware.get("num_gen_servers") - gpus_per_node = hardware.get("gpus_per_node") - - # Get gpus_per_ctx_server and gpus_per_gen_server from worker_config's tensor_parallel_size - ctx_config = worker_config.get("ctx", {}) - gen_config = worker_config.get("gen", {}) - ctx_tp = ctx_config.get("tensor_parallel_size", 1) - ctx_pp = ctx_config.get("pipeline_parallel_size", 1) - ctx_cp = ctx_config.get("context_parallel_size", 1) - gpus_per_ctx_server = ctx_tp * ctx_pp * ctx_cp - gen_tp = gen_config.get("tensor_parallel_size", 1) - gen_pp = gen_config.get("pipeline_parallel_size", 1) - gen_cp = gen_config.get("context_parallel_size", 1) - gpus_per_gen_server = gen_tp * gen_pp * gen_cp - - if None in [ - num_ctx_servers, - num_gen_servers, - gpus_per_node, - gpus_per_ctx_server, - gpus_per_gen_server, - ]: - raise ValueError("Missing required hardware configuration") - - # Calculate nodes per server - nodes_per_ctx_server = (gpus_per_ctx_server + gpus_per_node - 1) // gpus_per_node - nodes_per_gen_server = (gpus_per_gen_server + gpus_per_node - 1) // gpus_per_node - - gpus_per_node_per_ctx_server = min(gpus_per_ctx_server, gpus_per_node) - gpus_per_node_per_gen_server = min(gpus_per_gen_server, gpus_per_node) - - total_nodes = num_ctx_servers * nodes_per_ctx_server + num_gen_servers * nodes_per_gen_server - total_gpus = total_nodes * gpus_per_node - - return { - "num_ctx_servers": num_ctx_servers, - "num_gen_servers": num_gen_servers, - "gpus_per_node": gpus_per_node, - "gpus_per_ctx_server": gpus_per_ctx_server, - "gpus_per_gen_server": gpus_per_gen_server, - "nodes_per_ctx_server": nodes_per_ctx_server, - "nodes_per_gen_server": nodes_per_gen_server, - "gpus_per_node_per_ctx_server": gpus_per_node_per_ctx_server, - "gpus_per_node_per_gen_server": gpus_per_node_per_gen_server, - "total_nodes": total_nodes, - "total_gpus": total_gpus, - } - - -def get_env_config(config): - env = config.get("environment", {}) - - container = env.get("container_image", "") - mounts = env.get("container_mount", "") - workdir = env.get("container_workdir", "") - llm_models_root = env.get("llm_models_root", "") - llmsrc = env.get("trtllm_repo", "") - build_wheel = env.get("build_wheel", False) - # Use work_dir as job_workspace - job_workspace = env.get("work_dir", "") - worker_env_var = env.get("worker_env_var", "") - server_env_var = env.get("server_env_var", "") - benchmark_env_var = env.get("benchmark_env_var", "") - open_search_db_base_url = env.get("open_search_db_base_url", "") - - return { - "container": container, - "mounts": mounts, - "workdir": workdir, - "llm_models_root": llm_models_root, - "llmsrc": llmsrc, - "build_wheel": build_wheel, - "job_workspace": job_workspace, - "worker_env_var": worker_env_var, - "server_env_var": server_env_var, - "benchmark_env_var": benchmark_env_var, - "open_search_db_base_url": open_search_db_base_url, - } - - -def get_benchmark_config(config): - benchmark = config.get("benchmark", {}) - - concurrency_str = benchmark.get("concurrency_list", "1") - concurrency = int(concurrency_str) if isinstance(concurrency_str, str) else concurrency_str - - return { - "concurrency": concurrency, - } - - -def remove_whitespace_lines(lines): - return [line.strip() for line in lines if line.strip()] - - -def get_pytest_commands(script_prefix_lines): - # Get worker, disagg_server, benchmark pytest commands from pytest command. - # Worker pytest command is pytest command with trtllm-llmapi-launch and - # without --csv, --cov, --periodic flags. - # Disagg_server pytest command is pytest command without trtllm-llmapi-launch - # and without --csv, --cov, --periodic flags. - # Benchmark pytest command is pytest command without trtllm-llmapi-launch - # and with --csv, --cov, --periodic flags. - pytest_command_line = None - for line in script_prefix_lines: - if "export pytestCommand=" in line: - pytest_command_line = line - break - - if not pytest_command_line: - return "", "", "" - - def split_pytest_command_line(command_line): - # After pytest, there are six types of substrings: - # Type 1: --xxx=yyy (long option with value, self-contained) - # Type 2: --xxx= (long option with empty value, self-contained) - # Type 3: --xxx (long option flag, no value) - # Type 4: --xxx yyy (long option with value as next arg) - # Type 5: -x yyy (short single-letter option with value as next arg) - # Type 6: -x (short option flag, e.g., -v, -vv) - parts = command_line.split() - pytest_index = None - for idx, part in enumerate(parts): - if "pytest" == part: - pytest_index = idx - break - if pytest_index is None: - return parts - - grouped_parts = parts[: pytest_index + 1] - i = pytest_index + 1 - while i < len(parts): - part = parts[i] - has_next = i + 1 < len(parts) - next_is_value = has_next and not parts[i + 1].startswith("-") - - # Type 1 & 2: --xxx=yyy or --xxx= (self-contained, has '=') - if part.startswith("--") and "=" in part: - grouped_parts.append(part) - i += 1 - continue - - # Type 4: --xxx yyy (long option with value as next arg) - if part.startswith("--") and next_is_value: - grouped_parts.append(f"{part} {parts[i + 1]}") - i += 2 - continue - - # Type 3: --xxx (long option flag) - if part.startswith("--"): - grouped_parts.append(part) - i += 1 - continue - - # Type 5: -x yyy (short single-letter option with value as next arg) - # Only single letter after dash, e.g., -o, not -vv - if part.startswith("-") and len(part) == 2 and next_is_value: - grouped_parts.append(f"{part} {parts[i + 1]}") - i += 2 - continue - - # Type 6: -x (short option flag, including combined like -vv) - if part.startswith("-"): - grouped_parts.append(part) - i += 1 - continue - - # Other parts (shouldn't happen after pytest, but handle gracefully) - grouped_parts.append(part) - i += 1 - - return grouped_parts - - def is_llmapi_launch(part): - return "trtllm-llmapi-launch" in part - - def is_output_file_part(part): - return any(flag in part for flag in ("--csv", "--cov", "--periodic")) - - worker_line = pytest_command_line.replace("pytestCommand", "partialPytestCommandWorker") - worker_parts = [ - part for part in split_pytest_command_line(worker_line) if not is_output_file_part(part) - ] - worker_pytest_command = " ".join(worker_parts) - - disagg_server_line = pytest_command_line.replace( - "pytestCommand", "partialPytestCommandDisaggServer" - ) - disagg_server_parts = [ - part - for part in split_pytest_command_line(disagg_server_line) - if not is_llmapi_launch(part) and not is_output_file_part(part) - ] - disagg_server_pytest_command = " ".join(disagg_server_parts) - - benchmark_line = pytest_command_line.replace("pytestCommand", "partialPytestCommandBenchmark") - benchmark_parts = [ - part for part in split_pytest_command_line(benchmark_line) if not is_llmapi_launch(part) - ] - benchmark_pytest_command = " ".join(benchmark_parts) - - return ( - worker_pytest_command, - disagg_server_pytest_command, - benchmark_pytest_command, - ) - - -def parse_test_case_name(test_list_path, llm_src, split_group=0): - """Parse test list to get config yaml path and benchmark mode. - - Test formats for disagg: - - Disagg e2e: disagg_upload-e2e-{config_base} - - Disagg gen_only: disagg_upload-gen_only-{config_base} - - Args: - test_list_path: Path to the test list file. - llm_src: Path to the LLM source code. - split_group: 1-indexed split group id. When > 0, selects the - split_group-th test from the list instead of the first one. - - Returns: - tuple: (config_yaml_path, benchmark_mode) - - benchmark_mode: "e2e" or "gen_only" - """ - with open(test_list_path, "r") as f: - lines = [line.strip() for line in f if line.strip()] - - if split_group > 0: - if split_group > len(lines): - raise ValueError( - f"split_group {split_group} exceeds number of tests in test list ({len(lines)})" - ) - first_line = lines[split_group - 1] - else: - first_line = lines[0] - - if "[" not in first_line or "]" not in first_line: - raise ValueError( - f"Invalid test list format. Expected test name with brackets: {first_line}" - ) - bracket_content = first_line.split("[")[-1].split("]")[0] - parts = bracket_content.split("-") - - if len(parts) < 3: - raise ValueError( - f"Invalid disagg test format. Expected: disagg-{{mode}}-{{config}}, " - f"got: {bracket_content}" - ) - - # parts[0] is the prefix, parts[1] is benchmark_mode, parts[2:] is the config name - if "disagg" not in parts[0]: - raise ValueError( - f"Invalid test name format. Expected format: disagg-mode-config_name, " - f"got: {bracket_content}" - ) - - benchmark_mode = parts[1] # e2e or gen_only - if benchmark_mode not in ("e2e", "gen_only"): - raise ValueError( - f"Invalid benchmark_mode for disagg: {benchmark_mode}. Expected 'e2e' or 'gen_only'." - ) - - config_base_name = "-".join(parts[2:]) - config_yaml_path = os.path.join(llm_src, DISAGG_CONFIG_FOLDER, f"{config_base_name}.yaml") - - if not os.path.exists(config_yaml_path): - raise FileNotFoundError(f"Config file not found: {config_yaml_path}") - - return config_yaml_path, benchmark_mode - - -def main(): - parser = argparse.ArgumentParser( - description="Generate SLURM launch script for both CI and local modes" - ) - parser.add_argument( - "--run-ci", - action="store_true", - default=False, - help="Run in CI mode (true) or local mode (false)", - ) - parser.add_argument("--draft-launch-sh", required=True, help="Path to draft-launch.sh script") - parser.add_argument("--launch-sh", required=True, help="Path to output launch.sh script") - parser.add_argument("--run-sh", required=True, help="Path to slurm_run.sh script") - parser.add_argument("--install-sh", required=True, help="Path to slurm_install.sh script") - - # Optional arguments for local mode - parser.add_argument("--config-yaml", default="", help="Path to config YAML file") - parser.add_argument("--stage-name", default="", help="Stage name (optional, local mode only)") - - # Optional arguments for CI mode - parser.add_argument("--llm-src", default="", help="Path to LLM source code") - parser.add_argument("--test-list", default="", help="Path to test list file") - parser.add_argument( - "--script-prefix", - default="", - help="Launch script prefix file path (optional, CI mode only)", - ) - parser.add_argument( - "--srun-args", - default="", - help="Path to file containing srun args (optional, CI mode only)", - ) - parser.add_argument( - "--split-group", - type=int, - default=0, - help="1-indexed split group id. Selects the N-th test from the test list.", - ) - - args = parser.parse_args() - - config_yaml, benchmark_mode = parse_test_case_name( - args.test_list, args.llm_src, args.split_group - ) - - with open(config_yaml, "r") as f: - config = yaml.safe_load(f) - - is_gb300 = "GB300" in args.stage_name.upper() - is_b200 = "B200" in args.stage_name.upper() and "GB200" not in args.stage_name.upper() - - # Determine install script path - install_script = args.install_sh - - env_config = get_env_config(config) - print(f"Environment configuration: {env_config}") - - benchmark_config = get_benchmark_config(config) - print(f"Benchmark configuration: {benchmark_config}") - - hardware_config = get_hardware_config(config, benchmark_mode) - print(f"Hardware configuration: {hardware_config}") - - script_prefix_lines = [] - srun_args_lines = [] - - with open(args.script_prefix, "r") as f: - script_prefix_content = f.read() - script_prefix_lines = script_prefix_content.split("\n") - with open(args.srun_args, "r") as f: - srun_args_content = f.read() - - srun_args_lines = srun_args_content.split() - - # Extract pytestCommand and generate partial pytest commands - ( - worker_pytest_command, - disagg_server_pytest_command, - benchmark_pytest_command, - ) = get_pytest_commands(script_prefix_lines) - - # Build worker env vars (split into ctx and gen for role-specific settings) - base_worker_env_vars = ( - f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " - f"HF_HOME=/tmp/hf_home " - f"{env_config['worker_env_var']}" - ) - ctx_worker_env_vars = base_worker_env_vars - gen_worker_env_vars = base_worker_env_vars - server_env_vars = env_config["server_env_var"] - # Handle gen only mode - if "gen_only_no_context" in benchmark_mode: - gen_worker_env_vars = f"TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 {gen_worker_env_vars}" - server_env_vars = f"TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 {server_env_vars}" - script_prefix_lines.append("export TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1") - srun_args_lines.append("--container-env=TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") - elif "gen_only" in benchmark_mode: - concurrency = benchmark_config.get("concurrency", 1) - ctx_worker_env_vars = f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 {ctx_worker_env_vars}" - gen_worker_env_vars = ( - f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 " - f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={concurrency} {gen_worker_env_vars}" - ) - - pytest_common_vars = "" - - if is_gb300: - ucx_tls_cmd = "export UCX_TLS=cuda_copy,cuda_ipc,sm,self,tcp &&" - elif is_b200: - ucx_tls_cmd = "export UCX_TLS=^ib &&" - else: - ucx_tls_cmd = "unset UCX_TLS UCX_NET_DEVICES &&" - ucx_tls_server_cmd = ucx_tls_cmd - - script_prefix_lines.extend( - [ - worker_pytest_command, - disagg_server_pytest_command, - benchmark_pytest_command, - f'export PYTEST_COMMON_VARS="{pytest_common_vars}"', - f'export CTX_WORKER_ENV_VARS="{ctx_worker_env_vars}"', - f'export GEN_WORKER_ENV_VARS="{gen_worker_env_vars}"', - f'export SERVER_ENV_VARS="{server_env_vars}"', - f'export BENCHMARK_ENV_VARS="{env_config["benchmark_env_var"]}"', - f'export pytestCommandCTXWorker="{ucx_tls_cmd} $CTX_WORKER_ENV_VARS' - ' $PYTEST_COMMON_VARS $partialPytestCommandWorker"', - f'export pytestCommandGENWorker="{ucx_tls_cmd} $GEN_WORKER_ENV_VARS' - ' $PYTEST_COMMON_VARS $partialPytestCommandWorker"', - f'export pytestCommandDisaggServer="{ucx_tls_server_cmd}' - ' $SERVER_ENV_VARS $PYTEST_COMMON_VARS $partialPytestCommandDisaggServer"', - f'export pytestCommandBenchmark="{ucx_tls_cmd} $BENCHMARK_ENV_VARS' - ' $PYTEST_COMMON_VARS $partialPytestCommandBenchmark"', - f"export runScript={args.run_sh}", - f"export installScript={install_script}", - f"export configYamlPath={config_yaml}", - f"export numCtxServers={hardware_config['num_ctx_servers']}", - f"export numGenServers={hardware_config['num_gen_servers']}", - f"export gpusPerNode={hardware_config['gpus_per_node']}", - f"export gpusPerCtxServer={hardware_config['gpus_per_ctx_server']}", - f"export gpusPerGenServer={hardware_config['gpus_per_gen_server']}", - f"export nodesPerCtxServer={hardware_config['nodes_per_ctx_server']}", - f"export nodesPerGenServer={hardware_config['nodes_per_gen_server']}", - f"export gpusPerNodePerCtxServer={hardware_config['gpus_per_node_per_ctx_server']}", - f"export gpusPerNodePerGenServer={hardware_config['gpus_per_node_per_gen_server']}", - f"export totalNodes={hardware_config['total_nodes']}", - f"export totalGpus={hardware_config['total_gpus']}", - ] - ) - - remove_whitespace_lines(script_prefix_lines) - script_prefix = "\n".join(script_prefix_lines) - - remove_whitespace_lines(srun_args_lines) - srun_args_lines.extend( - [ - "--container-env=DISAGG_SERVING_TYPE", - "--container-env=pytestCommand", - ] - ) - srun_args_lines = ["srunArgs=("] + [f' "{line}"' for line in srun_args_lines] + [")"] - srun_args = "\n".join(srun_args_lines) - - with open(args.draft_launch_sh, "r") as f: - draft_launch_content = f.read() - draft_launch_lines = draft_launch_content.split("\n") - remove_whitespace_lines(draft_launch_lines) - draft_launch_content = "\n".join(draft_launch_lines) - - with open(args.launch_sh, "w") as f: - f.write(f"{script_prefix}\n{srun_args}\n{draft_launch_content}") - - print(f"Launch script generated at: {args.launch_sh}") - print(f"Launch script:\n{script_prefix}\n{srun_args}\n{draft_launch_content}") - - -if __name__ == "__main__": - main() diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py index fffee23e7481..0c2db5ef0681 100755 --- a/jenkins/scripts/perf/local/submit.py +++ b/jenkins/scripts/perf/local/submit.py @@ -249,15 +249,41 @@ def get_hardware_config(config, runtime_mode, benchmark_mode, test_name=None): } -def get_env_config(config, runtime_mode): - """Get env config based on mode.""" +def get_env_config(config, runtime_mode, benchmark_mode=None, server_name=None): + """Get worker / server / benchmark env vars from the yaml. + + Aggregated yaml stores env vars per server config under + `server_configs[i].server_env_var`. Disaggregated yaml stores them at the + top-level `environment.{worker,server,benchmark}_env_var`. + + ctx_only is a hybrid: the launch path is aggregated, but the yaml is the + disagg one, so the agg launch's "server_env_var" comes from + `environment.worker_env_var`. + + Returns: {worker_env_var, server_env_var, benchmark_env_var}. + """ + env = config.get("environment", {}) or {} if runtime_mode == "aggregated": - return {} - env = config.get("environment", {}) + if benchmark_mode == "ctx_only": + return { + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("worker_env_var", "") or "", + "benchmark_env_var": env.get("benchmark_env_var", "") or "", + } + agg_server_env_var = "" + for sc in config.get("server_configs", []) or []: + if sc.get("name") == server_name: + agg_server_env_var = sc.get("server_env_var", "") or "" + break + return { + "worker_env_var": "", + "server_env_var": agg_server_env_var, + "benchmark_env_var": "", + } return { - "worker_env_var": env.get("worker_env_var", ""), - "server_env_var": env.get("server_env_var", ""), - "benchmark_env_var": env.get("benchmark_env_var", ""), + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("server_env_var", "") or "", + "benchmark_env_var": env.get("benchmark_env_var", "") or "", } @@ -515,6 +541,19 @@ def main(): else: raise ValueError("Either --test-list or --config-file must be provided") + # Always rebuild test_case_name from parsed components so it matches the + # pytest test id written into test_list.txt by generate_pytest_command + # (which strips the `_upload` prefix). Without this, when --test-list is + # supplied with the long form (e.g. `disagg_upload-...`), test_output_dir + # would carry `_upload` while test_perf_sanity.py creates its working dir + # under the stripped form — producing two divergent folders. + if runtime_mode == "disaggregated": + test_case_name = f"disagg-{benchmark_mode}-{config_file_base_name}" + elif benchmark_mode == "ctx_only": + test_case_name = f"aggr-ctx_only-{config_file_base_name}" + else: + test_case_name = f"aggr-{config_file_base_name}-{select_pattern}" + # Load config if not already loaded if not args.config_file: with open(config_yaml, "r") as f: @@ -533,6 +572,10 @@ def main(): work_dir = os.path.join(llm_src, "jenkins", "scripts", "perf", "local", timestamp) os.makedirs(work_dir, exist_ok=True) + # Per-test output directory consumed by slurm_launch_draft.sh. Same shape + # as PerfSanityTestConfig.get_commands: /. + test_output_dir = os.path.join(work_dir, test_case_name) + test_prefix = args.test_prefix if args.test_prefix else f"{llm_src}/tests/integration/defs" # Determine paths @@ -559,7 +602,7 @@ def main(): ) # Get configs based on mode - env_config = get_env_config(config, runtime_mode) + env_config = get_env_config(config, runtime_mode, benchmark_mode, select_pattern) bm_config = get_benchmark_config(config, benchmark_mode) hardware_config = get_hardware_config( config, @@ -729,6 +772,7 @@ def main(): f"export gpusPerNodePerGenServer={hardware_config.get('gpus_per_node_per_gen_server', '')}", f"export totalNodes={hardware_config.get('total_nodes', '')}", f"export totalGpus={hardware_config.get('total_gpus', '')}", + f"export testOutputDir={test_output_dir}", ] ) @@ -745,20 +789,33 @@ def main(): f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " f"HF_HOME=/tmp/hf_home " ) - # Aggregated mode (including ctx_only) + # Aggregated mode (including ctx_only). + # For regular agg, server_env_var (from the matched server_configs entry) + # carries spec-decode flags like TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS; + # for ctx_only, get_env_config maps environment.worker_env_var into the + # server_env_var slot. Both reach all SLURM ranks via env-prefix on + # pytestCommand before trtllm-llmapi-launch dispatches. + agg_server_env_vars = env_config.get("server_env_var", "") script_prefix_lines.extend( [ f'export WORKER_ENV_VARS="{worker_env_vars}"', + f'export SERVER_ENV_VARS="{agg_server_env_vars}"', ( - 'export pytestCommand="$WORKER_ENV_VARS $PYTEST_COMMON_VARS $NSYS_PREFIX $LLM_API_LAUNCH' + 'export pytestCommand="$SERVER_ENV_VARS $WORKER_ENV_VARS $PYTEST_COMMON_VARS' + " $NSYS_PREFIX $LLM_API_LAUNCH" f' $PYTEST_COMMAND --junitxml={work_dir}/report.xml"' ), f"export gpusPerNode={hardware_config.get('gpus_per_node', '')}", f"export gpusPerNodePerServer={hardware_config.get('gpus_per_node_per_server', '')}", f"export totalNodes={hardware_config.get('total_nodes', '')}", f"export totalGpus={hardware_config.get('total_gpus', '')}", + f"export testOutputDir={test_output_dir}", ] ) + # Forward pytestCommand into the pyxis container on every rank (matches the + # disagg branch's --container-env=pytestCommand). Without this, exports + # in the launch script don't survive the container boundary on multi-node. + srun_args_lines.append("--container-env=pytestCommand") # Export accuracy config to BENCHMARK node (disagg only) if runtime_mode == "disaggregated": diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py new file mode 100755 index 000000000000..a621d4ca1f8c --- /dev/null +++ b/jenkins/scripts/perf/submit.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +"""Generate the SLURM launch script for multi-node PerfSanity tests (CI mode). + +Unified replacement for jenkins/scripts/perf/aggregated/submit.py and +jenkins/scripts/perf/disaggregated/submit.py. + +Three test shapes are supported (all flow through the same parsing logic): + 1. Multi-node aggregated: aggr[_upload]-{config_base}-{server_name} + runtime_mode = "aggregated", benchmark_mode = None + 2. Multi-node ctx_only disagg: aggr[_upload]-ctx_only-{config_base} + runtime_mode = "aggregated", benchmark_mode = "ctx_only" + (reads disagg yaml, but launches via the aggregated single-pytest path + using the ctx worker's parallel sizes) + 3. Multi-node disagg e2e/gen: disagg[_upload]-{e2e|gen_only}-{config_base} + runtime_mode = "disaggregated", benchmark_mode in {"e2e", "gen_only"} + +Test name → yaml folder mapping mirrors test_perf_sanity.py:parse_test_string. +""" + +import argparse +import math +import os +import re + +import yaml + +AGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/aggregated" +DISAGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/disaggregated" + + +# --------------------------------------------------------------------------- # +# Test list parsing +# --------------------------------------------------------------------------- # +def parse_test_case_name(test_list_path, llm_src, split_group=0): + """Parse the selected line of the test list. + + Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode). + See the module docstring for the supported test name shapes. + """ + with open(test_list_path, "r") as f: + lines = [line.strip() for line in f if line.strip()] + + if not lines: + raise ValueError(f"Test list is empty: {test_list_path}") + + if split_group > 0: + if split_group > len(lines): + raise ValueError( + f"split_group {split_group} exceeds number of tests in test list ({len(lines)})" + ) + line = lines[split_group - 1] + else: + line = lines[0] + + if "[" not in line or "]" not in line: + raise ValueError(f"Invalid test list format. Expected name with brackets: {line}") + bracket_content = line.split("[")[-1].split("]")[0] + parts = bracket_content.split("-") + + if len(parts) < 2: + raise ValueError(f"Invalid test name (need at least prefix and config): {bracket_content}") + + prefix = parts[0] + if "disagg" in prefix: + if len(parts) < 3: + raise ValueError( + f"Invalid disagg test format. Expected disagg[_upload]-{{e2e|gen_only}}-" + f"{{config_base}}, got: {bracket_content}" + ) + benchmark_mode = parts[1] + if benchmark_mode not in ("e2e", "gen_only"): + raise ValueError( + f"Invalid disagg benchmark_mode: {benchmark_mode}. Expected 'e2e' or 'gen_only'." + ) + runtime_mode = "disaggregated" + server_name = None + config_base_name = "-".join(parts[2:]) + config_yaml_path = os.path.join(llm_src, DISAGG_CONFIG_FOLDER, f"{config_base_name}.yaml") + elif "aggr" in prefix: + if len(parts) > 2 and parts[1] == "ctx_only": + # ctx_only: aggr[_upload]-ctx_only-{config_base}; reads disagg yaml. + benchmark_mode = "ctx_only" + runtime_mode = "aggregated" + server_name = None + config_base_name = "-".join(parts[2:]) + config_yaml_path = os.path.join( + llm_src, DISAGG_CONFIG_FOLDER, f"{config_base_name}.yaml" + ) + else: + # Regular agg: aggr[_upload]-{config_base}-{server_name}. + # config_base_name is a single label and server_name is everything + # after it — mirrors test_perf_sanity.py:parse_test_string so the + # launch script and the test runner agree on the config path. + if len(parts) < 3: + raise ValueError( + f"Invalid agg test format. Expected aggr[_upload]-{{config_base}}-" + f"{{server_name}}, got: {bracket_content}" + ) + benchmark_mode = None + runtime_mode = "aggregated" + config_base_name = parts[1] + server_name = "-".join(parts[2:]) + config_yaml_path = os.path.join(llm_src, AGG_CONFIG_FOLDER, f"{config_base_name}.yaml") + else: + raise ValueError( + f"Invalid test name prefix '{prefix}'. Expected starts-with 'aggr' or 'disagg'." + ) + + if not os.path.exists(config_yaml_path): + raise FileNotFoundError(f"Config file not found: {config_yaml_path}") + + return config_yaml_path, server_name, benchmark_mode, runtime_mode + + +# --------------------------------------------------------------------------- # +# Hardware / env / benchmark config (unified across agg and disagg) +# --------------------------------------------------------------------------- # +def get_hardware_config(config, runtime_mode, benchmark_mode, server_name): + """Compute the hardware layout. Mirrors local/submit.py:get_hardware_config. + + Aggregated (incl. ctx_only) returns: + gpus_per_node, gpus_per_server, nodes_per_server, gpus_per_node_per_server, + total_nodes, total_gpus, world_size + Disaggregated returns: + num_ctx_servers, num_gen_servers, gpus_per_node, + gpus_per_ctx_server, gpus_per_gen_server, + nodes_per_ctx_server, nodes_per_gen_server, + gpus_per_node_per_ctx_server, gpus_per_node_per_gen_server, + total_nodes, total_gpus + """ + hardware = config.get("hardware", {}) or {} + gpus_per_node = hardware.get("gpus_per_node") + if gpus_per_node is None: + raise ValueError("hardware.gpus_per_node is required") + + if benchmark_mode == "ctx_only": + # ctx_only reads disagg yaml; size the launch from worker_config.ctx. + worker_config = config.get("worker_config", {}) or {} + ctx_config = worker_config.get("ctx", {}) or {} + if not ctx_config: + raise ValueError("worker_config.ctx is required for ctx_only mode") + tp = ctx_config.get("tensor_parallel_size", 1) + pp = ctx_config.get("pipeline_parallel_size", 1) + cp = ctx_config.get("context_parallel_size", 1) + gpus_per_server = ctx_config.get("world_size") or (tp * pp * cp) + elif runtime_mode == "aggregated": + # Regular agg: match server_configs by name. + server_configs = config.get("server_configs", []) or [] + server_config = next((sc for sc in server_configs if sc.get("name") == server_name), None) + if server_config is None: + raise ValueError(f"server_config not found for name: {server_name}") + tp = server_config.get("tensor_parallel_size", 1) + pp = server_config.get("pipeline_parallel_size", 1) + cp = server_config.get("context_parallel_size", 1) + gpus_per_server = server_config.get("world_size") or (tp * pp * cp) + else: + # Disaggregated: separate ctx + gen workers. + worker_config = config.get("worker_config", {}) or {} + ctx_config = worker_config.get("ctx", {}) or {} + gen_config = worker_config.get("gen", {}) or {} + + # gen_only_no_context comes from the yaml's benchmark.mode, not the + # test name (test name is always "gen_only" for both gen_only and + # gen_only_no_context tests). When set, ctx workers are not launched. + yaml_mode = (config.get("benchmark", {}) or {}).get("mode", "") + is_gen_only_no_context = benchmark_mode == "gen_only" and "gen_only_no_context" in yaml_mode + num_ctx_servers = 0 if is_gen_only_no_context else hardware.get("num_ctx_servers") + num_gen_servers = hardware.get("num_gen_servers") + + ctx_tp = ctx_config.get("tensor_parallel_size", 1) + ctx_pp = ctx_config.get("pipeline_parallel_size", 1) + ctx_cp = ctx_config.get("context_parallel_size", 1) + gpus_per_ctx_server = ctx_tp * ctx_pp * ctx_cp + gen_tp = gen_config.get("tensor_parallel_size", 1) + gen_pp = gen_config.get("pipeline_parallel_size", 1) + gen_cp = gen_config.get("context_parallel_size", 1) + gpus_per_gen_server = gen_tp * gen_pp * gen_cp + + if None in [num_ctx_servers, num_gen_servers, gpus_per_ctx_server, gpus_per_gen_server]: + raise ValueError("Missing required disagg hardware configuration") + + nodes_per_ctx_server = math.ceil(gpus_per_ctx_server / gpus_per_node) + nodes_per_gen_server = math.ceil(gpus_per_gen_server / gpus_per_node) + gpus_per_node_per_ctx_server = min(gpus_per_ctx_server, gpus_per_node) + gpus_per_node_per_gen_server = min(gpus_per_gen_server, gpus_per_node) + + total_nodes = ( + num_ctx_servers * nodes_per_ctx_server + num_gen_servers * nodes_per_gen_server + ) + total_gpus = total_nodes * gpus_per_node + + return { + "num_ctx_servers": num_ctx_servers, + "num_gen_servers": num_gen_servers, + "gpus_per_node": gpus_per_node, + "gpus_per_ctx_server": gpus_per_ctx_server, + "gpus_per_gen_server": gpus_per_gen_server, + "nodes_per_ctx_server": nodes_per_ctx_server, + "nodes_per_gen_server": nodes_per_gen_server, + "gpus_per_node_per_ctx_server": gpus_per_node_per_ctx_server, + "gpus_per_node_per_gen_server": gpus_per_node_per_gen_server, + "total_nodes": total_nodes, + "total_gpus": total_gpus, + } + + # Aggregated (regular or ctx_only) shared layout. + nodes_per_server = math.ceil(gpus_per_server / gpus_per_node) + total_nodes = nodes_per_server + gpus_per_node_per_server = min(gpus_per_server, gpus_per_node) + world_size = total_nodes * gpus_per_node_per_server + return { + "gpus_per_node": gpus_per_node, + "gpus_per_server": gpus_per_server, + "nodes_per_server": nodes_per_server, + "gpus_per_node_per_server": gpus_per_node_per_server, + "total_nodes": total_nodes, + "total_gpus": total_nodes * gpus_per_node, + "world_size": world_size, + } + + +def get_env_config(config, runtime_mode, benchmark_mode, server_name): + """Get worker / server / benchmark env vars from the yaml. + + Aggregated yaml stores env vars per server config under + `server_configs[i].server_env_var`. Disaggregated yaml stores them at the + top-level `environment.{worker,server,benchmark}_env_var`. + + ctx_only is a hybrid: the launch path is aggregated, but the yaml is the + disagg one, so the agg launch's "server_env_var" comes from + `environment.worker_env_var`. + + Returns: {worker_env_var, server_env_var, benchmark_env_var}. + """ + env = config.get("environment", {}) or {} + if runtime_mode == "aggregated": + if benchmark_mode == "ctx_only": + return { + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("worker_env_var", "") or "", + "benchmark_env_var": env.get("benchmark_env_var", "") or "", + } + agg_server_env_var = "" + for sc in config.get("server_configs", []) or []: + if sc.get("name") == server_name: + agg_server_env_var = sc.get("server_env_var", "") or "" + break + return { + "worker_env_var": "", + "server_env_var": agg_server_env_var, + "benchmark_env_var": "", + } + return { + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("server_env_var", "") or "", + "benchmark_env_var": env.get("benchmark_env_var", "") or "", + } + + +def get_benchmark_config(config): + benchmark = config.get("benchmark", {}) or {} + concurrency_str = benchmark.get("concurrency_list", "1") + concurrency = int(concurrency_str) if isinstance(concurrency_str, str) else concurrency_str + return { + "mode": benchmark.get("mode", ""), + "concurrency": concurrency, + } + + +# --------------------------------------------------------------------------- # +# pytestCommand splitting +# --------------------------------------------------------------------------- # +def _split_pytest_command_line(command_line): + """Group the pytest tail of `command_line` into self-contained tokens. + + After `pytest`, six argument shapes appear: + Type 1: --xxx=yyy (long option, value via '=') + Type 2: --xxx= (long option, empty value) + Type 3: --xxx (long option flag) + Type 4: --xxx yyy (long option, value as next token) + Type 5: -x yyy (short option, value as next token) + Type 6: -x / -vv (short flag(s)) + Tokens before `pytest` are kept as-is (the env-var prefix, `pytest` itself). + """ + parts = command_line.split() + pytest_index = None + for idx, part in enumerate(parts): + if part == "pytest": + pytest_index = idx + break + if pytest_index is None: + return parts + + grouped = parts[: pytest_index + 1] + i = pytest_index + 1 + while i < len(parts): + part = parts[i] + has_next = i + 1 < len(parts) + next_is_value = has_next and not parts[i + 1].startswith("-") + + if part.startswith("--") and "=" in part: # Type 1 & 2 + grouped.append(part) + i += 1 + elif part.startswith("--") and next_is_value: # Type 4 + grouped.append(f"{part} {parts[i + 1]}") + i += 2 + elif part.startswith("--"): # Type 3 + grouped.append(part) + i += 1 + elif part.startswith("-") and len(part) == 2 and next_is_value: # Type 5 + grouped.append(f"{part} {parts[i + 1]}") + i += 2 + elif part.startswith("-"): # Type 6 + grouped.append(part) + i += 1 + else: + grouped.append(part) + i += 1 + return grouped + + +def get_pytest_commands(script_prefix_lines, runtime_mode): + """Emit the partial pytestCommand variants needed by the launch path. + + Finds the inbound `export pytestCommand=...` line and rewrites it. + + Aggregated (incl. ctx_only) returns a 4-tuple: + (agg_partial_line, "", "", "") + where agg_partial_line is the original line with `pytestCommand` renamed + to `partialPytestCommand`. + + Disaggregated returns a 4-tuple: + ("", worker_line, disagg_server_line, benchmark_line) + where: + - worker_line: rename to partialPytestCommandWorker; drop --csv/--cov/--periodic + - disagg_server_line: rename to partialPytestCommandDisaggServer; drop trtllm-llmapi-launch + and --csv/--cov/--periodic + - benchmark_line: rename to partialPytestCommandBenchmark; drop trtllm-llmapi-launch + """ + pytest_command_line = next( + (ln for ln in script_prefix_lines if "export pytestCommand=" in ln), None + ) + if not pytest_command_line: + return ("", "", "", "") + + if runtime_mode == "aggregated": + agg_line = pytest_command_line.replace("pytestCommand", "partialPytestCommand") + return (agg_line, "", "", "") + + def _is_llmapi_launch(part): + return "trtllm-llmapi-launch" in part + + def _is_output_file_part(part): + return any(flag in part for flag in ("--csv", "--cov", "--periodic")) + + worker_line = pytest_command_line.replace("pytestCommand", "partialPytestCommandWorker") + worker_parts = [ + p for p in _split_pytest_command_line(worker_line) if not _is_output_file_part(p) + ] + worker_pytest_command = " ".join(worker_parts) + + disagg_server_line = pytest_command_line.replace( + "pytestCommand", "partialPytestCommandDisaggServer" + ) + disagg_server_parts = [ + p + for p in _split_pytest_command_line(disagg_server_line) + if not _is_llmapi_launch(p) and not _is_output_file_part(p) + ] + disagg_server_pytest_command = " ".join(disagg_server_parts) + + benchmark_line = pytest_command_line.replace("pytestCommand", "partialPytestCommandBenchmark") + benchmark_parts = [ + p for p in _split_pytest_command_line(benchmark_line) if not _is_llmapi_launch(p) + ] + benchmark_pytest_command = " ".join(benchmark_parts) + + return ("", worker_pytest_command, disagg_server_pytest_command, benchmark_pytest_command) + + +def get_test_output_dir(script_prefix_lines, test_case_name): + """Build the per-test output directory from the inbound pytestCommand. + + Picks `--output-dir` out of the inbound pytestCommand and appends the + test_case_name — same shape as + PerfSanityTestConfig.get_commands: /. + """ + pytest_command_line = next( + (ln for ln in script_prefix_lines if "export pytestCommand=" in ln), "" + ) + if not pytest_command_line: + return "" + m = re.search(r'--output-dir[=\s]+"?([^"\s]+)"?', pytest_command_line) + if not m: + return "" + output_dir = m.group(1) + return os.path.join(output_dir, test_case_name) if test_case_name else output_dir + + +def remove_whitespace_lines(lines): + return [line.strip() for line in lines if line.strip()] + + +# --------------------------------------------------------------------------- # +# Main +# --------------------------------------------------------------------------- # +def main(): + parser = argparse.ArgumentParser( + description=( + "Generate the SLURM launch script for multi-node aggregated and " + "disaggregated PerfSanity tests (CI mode)." + ) + ) + parser.add_argument("--draft-launch-sh", required=True, help="Path to draft-launch.sh script") + parser.add_argument("--launch-sh", required=True, help="Path to output launch.sh script") + parser.add_argument("--run-sh", required=True, help="Path to slurm_run.sh script") + parser.add_argument("--install-sh", required=True, help="Path to slurm_install.sh script") + parser.add_argument("--llm-src", required=True, help="Path to LLM source code") + parser.add_argument("--test-list", required=True, help="Path to test list file") + parser.add_argument( + "--script-prefix", + required=True, + help="Launch script prefix file path", + ) + parser.add_argument( + "--srun-args", + required=True, + help="Path to file containing srun args", + ) + parser.add_argument( + "--split-group", + type=int, + default=0, + help="1-indexed split group id. Selects the N-th test from the test list.", + ) + parser.add_argument("--stage-name", default="", help="Stage name (for logging / GPU detect)") + + args = parser.parse_args() + + config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name( + args.test_list, args.llm_src, args.split_group + ) + + with open(config_yaml, "r") as f: + config = yaml.safe_load(f) + + # Recover test_case_name (the bracketed pytest test id) for the per-test + # output dir — same line/split logic as parse_test_case_name. + with open(args.test_list, "r") as f: + lines = [ln.strip() for ln in f if ln.strip()] + sel = lines[args.split_group - 1] if args.split_group > 0 else lines[0] + test_case_name = sel.split("[")[-1].split("]")[0] if "[" in sel else "" + + hardware_config = get_hardware_config(config, runtime_mode, benchmark_mode, server_name) + env_config = get_env_config(config, runtime_mode, benchmark_mode, server_name) + benchmark_config = get_benchmark_config(config) + + print(f"runtime_mode: {runtime_mode!r}") + print(f"benchmark_mode: {benchmark_mode!r}") + print(f"server_name: {server_name!r}") + print(f"Hardware configuration: {hardware_config}") + print(f"Environment configuration: {env_config}") + print(f"Benchmark configuration: {benchmark_config}") + + with open(args.script_prefix, "r") as f: + script_prefix_content = f.read() + script_prefix_lines = script_prefix_content.split("\n") + + with open(args.srun_args, "r") as f: + srun_args_content = f.read() + srun_args_lines = srun_args_content.split() + + ( + agg_pytest_command, + worker_pytest_command, + disagg_server_pytest_command, + benchmark_pytest_command, + ) = get_pytest_commands(script_prefix_lines, runtime_mode) + test_output_dir = get_test_output_dir(script_prefix_lines, test_case_name) + + is_gb300 = "GB300" in args.stage_name.upper() + is_b200 = "B200" in args.stage_name.upper() and "GB200" not in args.stage_name.upper() + + if runtime_mode == "aggregated": + # Aggregated (incl. ctx_only): single pytestCommand built from the + # matched server_config's server_env_var (regular agg) or the disagg + # yaml's environment.worker_env_var (ctx_only). The prefix runs on + # every rank before trtllm-llmapi-launch dispatches to pytest (rank 0) + # or mgmn_worker_node (others). + server_env_var = env_config["server_env_var"] + if server_env_var.strip(): + pytest_command_with_env = ( + f'export pytestCommand="{server_env_var} $partialPytestCommand"' + ) + else: + pytest_command_with_env = 'export pytestCommand="$partialPytestCommand"' + + script_prefix_lines.extend( + [ + agg_pytest_command, + pytest_command_with_env, + f"export runScript={args.run_sh}", + f"export installScript={args.install_sh}", + f"export configYamlPath={config_yaml}", + f"export gpusPerNode={hardware_config['gpus_per_node']}", + f"export gpusPerNodePerServer={hardware_config['gpus_per_node_per_server']}", + f"export nodesPerServer={hardware_config['nodes_per_server']}", + f"export totalNodes={hardware_config['total_nodes']}", + f"export world_size={hardware_config['world_size']}", + f"export testOutputDir={test_output_dir}", + ] + ) + srun_args_lines.append("--container-env=pytestCommand") + else: + # Disaggregated (e2e or gen_only). + base_worker_env_vars = ( + f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " + f"HF_HOME=/tmp/hf_home " + f"{env_config['worker_env_var']}" + ) + ctx_worker_env_vars = base_worker_env_vars + gen_worker_env_vars = base_worker_env_vars + server_env_vars = env_config["server_env_var"] + + # gen_only_no_context comes from yaml's benchmark.mode, not the test + # name — see get_hardware_config. + yaml_mode = benchmark_config.get("mode", "") + if benchmark_mode == "gen_only" and "gen_only_no_context" in yaml_mode: + gen_worker_env_vars = f"TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 {gen_worker_env_vars}" + server_env_vars = f"TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 {server_env_vars}" + script_prefix_lines.append("export TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1") + srun_args_lines.append("--container-env=TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") + elif benchmark_mode == "gen_only": + concurrency = benchmark_config.get("concurrency", 1) + ctx_worker_env_vars = ( + f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 {ctx_worker_env_vars}" + ) + gen_worker_env_vars = ( + f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 " + f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={concurrency} {gen_worker_env_vars}" + ) + + if is_gb300: + ucx_tls_cmd = "export UCX_TLS=cuda_copy,cuda_ipc,sm,self,tcp &&" + elif is_b200: + ucx_tls_cmd = "export UCX_TLS=^ib &&" + else: + ucx_tls_cmd = "unset UCX_TLS UCX_NET_DEVICES &&" + ucx_tls_server_cmd = ucx_tls_cmd + + pytest_common_vars = "" + script_prefix_lines.extend( + [ + worker_pytest_command, + disagg_server_pytest_command, + benchmark_pytest_command, + f'export PYTEST_COMMON_VARS="{pytest_common_vars}"', + f'export CTX_WORKER_ENV_VARS="{ctx_worker_env_vars}"', + f'export GEN_WORKER_ENV_VARS="{gen_worker_env_vars}"', + f'export SERVER_ENV_VARS="{server_env_vars}"', + f'export BENCHMARK_ENV_VARS="{env_config["benchmark_env_var"]}"', + f'export pytestCommandCTXWorker="{ucx_tls_cmd} $CTX_WORKER_ENV_VARS' + ' $PYTEST_COMMON_VARS $partialPytestCommandWorker"', + f'export pytestCommandGENWorker="{ucx_tls_cmd} $GEN_WORKER_ENV_VARS' + ' $PYTEST_COMMON_VARS $partialPytestCommandWorker"', + f'export pytestCommandDisaggServer="{ucx_tls_server_cmd}' + ' $SERVER_ENV_VARS $PYTEST_COMMON_VARS $partialPytestCommandDisaggServer"', + f'export pytestCommandBenchmark="{ucx_tls_cmd} $BENCHMARK_ENV_VARS' + ' $PYTEST_COMMON_VARS $partialPytestCommandBenchmark"', + f"export runScript={args.run_sh}", + f"export installScript={args.install_sh}", + f"export configYamlPath={config_yaml}", + f"export numCtxServers={hardware_config['num_ctx_servers']}", + f"export numGenServers={hardware_config['num_gen_servers']}", + f"export gpusPerNode={hardware_config['gpus_per_node']}", + f"export gpusPerCtxServer={hardware_config['gpus_per_ctx_server']}", + f"export gpusPerGenServer={hardware_config['gpus_per_gen_server']}", + f"export nodesPerCtxServer={hardware_config['nodes_per_ctx_server']}", + f"export nodesPerGenServer={hardware_config['nodes_per_gen_server']}", + f"export gpusPerNodePerCtxServer={hardware_config['gpus_per_node_per_ctx_server']}", + f"export gpusPerNodePerGenServer={hardware_config['gpus_per_node_per_gen_server']}", + f"export totalNodes={hardware_config['total_nodes']}", + f"export totalGpus={hardware_config['total_gpus']}", + f"export testOutputDir={test_output_dir}", + ] + ) + srun_args_lines.extend( + [ + "--container-env=DISAGG_SERVING_TYPE", + "--container-env=pytestCommand", + ] + ) + + script_prefix_lines = remove_whitespace_lines(script_prefix_lines) + script_prefix = "\n".join(script_prefix_lines) + + srun_args_lines = remove_whitespace_lines(srun_args_lines) + srun_args_lines = ["srunArgs=("] + [f' "{line}"' for line in srun_args_lines] + [")"] + srun_args = "\n".join(srun_args_lines) + + with open(args.draft_launch_sh, "r") as f: + draft_launch_content = f.read() + draft_launch_lines = remove_whitespace_lines(draft_launch_content.split("\n")) + draft_launch_content = "\n".join(draft_launch_lines) + + with open(args.launch_sh, "w") as f: + f.write(f"{script_prefix}\n{srun_args}\n{draft_launch_content}") + + print(f"Launch script generated at: {args.launch_sh}") + print(f"Launch script:\n{script_prefix}\n{srun_args}\n{draft_launch_content}") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index fdaeeaa7c55b..a1c534507634 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -23,9 +23,33 @@ For the underlying regression pipeline architecture (three-layer design, baselin | List | Count | Contents | |------|-------|----------| -| `MAXIMIZE_METRICS` | 7 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + TPOT (`d_mean_tpot`, `d_median_tpot`, `d_p99_tpot`) | -| `MINIMIZE_METRICS` | 9 | TTFT, ITL, E2EL latencies (mean/median/P99 for each) | -| `REGRESSION_METRICS` | 2 | `d_token_throughput`, `d_total_token_throughput` — only these gate pass/fail | +| `MAXIMIZE_METRICS` | 8 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + TPOT (`d_mean_tpot`, `d_median_tpot`, `d_p99_tpot`) + spec-decoding `d_al` | +| `MINIMIZE_METRICS` | 10 | TTFT, ITL, E2EL latencies (mean/median/P99 for each) + `d_mean_gen_worker_per_iter_device_step_time` (gen_only-only) | +| `REGRESSION_METRICS` | 2 default | `d_token_throughput`, `d_total_token_throughput` — gate pass/fail for all modes **except disagg gen_only**. `d_al` is appended at runtime when any client runs spec decoding. | + +**Disagg gen_only override**: For `disagg_upload-gen_only-*` tests, regression is gated **only** on `d_mean_gen_worker_per_iter_device_step_time`. Token-based throughput numbers are dominated by KV-cache transfer time in gen_only mode and are not a useful regression signal there. + +#### `d_mean_gen_worker_per_iter_device_step_time` (gen_only only) + +This metric is parsed from each `gen_server_{i}.log` produced by the disagg run (one per gen worker, in the run's `output_dir`). Lines look like: + +``` +[TRT-LLM] [I] [_torch][RANK 0] iter = 5, ..., host_step_time = 6.79ms, prev_device_step_time = 6.94ms, ... +``` + +The device value reported at iter `N` is the device step time of iter `N-1` (device runs async). + +**Per-client computation** (DisaggTestCmds.run_cmd, BENCHMARK branch): +1. Immediately before launching each client, snapshot `os.path.getsize()` of every `gen_server_{i}.log`. After the client's benchmark subprocess returns, only the bytes between that snapshot and current EOF are parsed — so each client gets its own segment of gen-worker iterations rather than sharing a single global average. +2. Per file (per segment): streaming Welford mean of `prev_device_step_time` over all iters with `iter >= 5` (iters 0-4 are excluded: iter 0/1 include KV-cache transfer wait time, and iters 2-4 are warmup that has not yet reached steady state). Lines where `prev_device_step_time = N/A` (e.g. iter 1) do not match the parser and are skipped. Welford keeps memory at O(1) and is numerically stable for arbitrarily large iteration counts. +3. Across files: average the per-file means → per-client metric value. +4. The per-client value is appended as the trailing line of `trtllm-benchmark.{server_idx}.{client_idx}.log`: + ``` + Average Per Iter Device Step Time (ms): + ``` + Downstream `parse_metrics_from_output` picks it up via `GEN_ONLY_PERF_METRIC_LOG_QUERIES`. + +If the metric cannot be parsed for a `gen_only` run, `check_test_failure` raises `RuntimeError` and no data is uploaded. ### Match Keys diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 40deafd9b15f..818ccd955733 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -122,6 +122,114 @@ def ensure_bench_serving_repo() -> str: "p99_e2el": re.compile(r"P99 E2EL \(ms\):\s+(-?[\d\.]+)"), } +# Spec-decoding-only metrics: parsed from benchmark output but only stored +# (and regression-checked) when the test runs with speculative decoding. +SPEC_DECODING_PERF_METRIC_LOG_QUERIES = { + "al": re.compile(r"Mean Avg Decoded Tokens per Iter:\s+(-?[\d\.]+)"), +} + +# gen_only-only metric: appended to each trtllm-benchmark log by +# DisaggTestCmds.run_cmd after parsing gen_server_*.log; only forwarded to +# the database for gen_only mode. +GEN_ONLY_PERF_METRIC_LOG_QUERIES = { + "mean_gen_worker_per_iter_device_step_time": re.compile( + r"Average Per Iter Device Step Time \(ms\):\s+(-?[\d\.]+)" + ), +} + +# Per-iter prev_device_step_time logged by each gen worker. Example line: +# [TRT-LLM] [I] [_torch][RANK 0] iter = 5, global_rank = 0, ..., +# host_step_time = 6.79ms, prev_device_step_time = 6.94ms, ... +# Only the gen worker (decode) emits this. The device value reported at iter N +# is the device step time of iter N-1 (device runs async). Iters 0-4 are +# skipped: iter 0/1 include KV-cache transfer wait time, and iters 2-4 are +# warmup that has not yet reached steady state. prev_device_step_time may be +# 'N/A' (e.g. iter 1); the regex requires a numeric value so those lines do +# not match. +_DEVICE_STEP_TIME_RE = re.compile(r"iter\s*=\s*(\d+),.*?prev_device_step_time\s*=\s*([\d.]+)\s*ms") + + +def gen_worker_log_sizes(output_dir: str, num_gen_servers: int) -> List[int]: + """Current byte size of each gen_server_{i}.log (0 if missing). + + Used to delimit per-client segments in DisaggTestCmds.run_cmd: snapshot + sizes before launching a client, then pass the snapshot as start_offsets + to parse_gen_worker_device_step_time after the client exits. + """ + sizes: List[int] = [] + for i in range(num_gen_servers): + log_path = os.path.join(output_dir, f"gen_server_{i}.log") + sizes.append(os.path.getsize(log_path) if os.path.isfile(log_path) else 0) + return sizes + + +def parse_gen_worker_device_step_time( + output_dir: str, + num_gen_servers: int, + start_offsets: Optional[List[int]] = None, +) -> Optional[float]: + """Mean per-iter prev_device_step_time (ms) across all gen workers. + + For each gen_server_{i}.log, average prev_device_step_time over iters >= 5, + then average those per-file means across the num_gen_servers workers. + Returns None if no usable line is found in any file. + + When start_offsets is provided, only the bytes from start_offsets[i] to + end-of-file are considered for gen_server_{i}.log — used to slice out a + single client's iteration segment. + """ + per_file_means: List[float] = [] + for i in range(num_gen_servers): + log_path = os.path.join(output_dir, f"gen_server_{i}.log") + if not os.path.isfile(log_path): + continue + # Welford streaming mean: O(1) memory and numerically stable for + # large iteration counts. + count = 0 + mean = 0.0 + with open(log_path) as f: + if start_offsets is not None and i < len(start_offsets) and start_offsets[i]: + f.seek(start_offsets[i]) + for line in f: + m = _DEVICE_STEP_TIME_RE.search(line) + if m is None: + continue + iter_idx = int(m.group(1)) + if iter_idx < 5: + continue + count += 1 + mean += (float(m.group(2)) - mean) / count + if count: + per_file_means.append(mean) + if not per_file_means: + return None + return sum(per_file_means) / len(per_file_means) + + +def add_perf_metric_value( + new_data: dict, + metrics: dict, + spec_decoding: bool, + benchmark_mode: Optional[str] = None, +) -> None: + """Populate `new_data` with per-test perf metrics from `metrics`. + + - Always copies every key in PERF_METRIC_LOG_QUERIES as `d_`. + - Adds `d_al` only when spec_decoding=True; non-spec rows omit it so + OpenSearch baselines don't blend the two populations. + - Adds `d_mean_gen_worker_per_iter_device_step_time` only for the + disagg gen_only mode (the only mode whose regression is gated on it). + """ + for metric_name in PERF_METRIC_LOG_QUERIES: + new_data[f"d_{metric_name}"] = metrics[metric_name] + if spec_decoding: + new_data["d_al"] = metrics["al"] + if benchmark_mode == "gen_only": + new_data["d_mean_gen_worker_per_iter_device_step_time"] = metrics[ + "mean_gen_worker_per_iter_device_step_time" + ] + + # Metrics where larger is better MAXIMIZE_METRICS = [ "d_seq_throughput", @@ -131,6 +239,7 @@ def ensure_bench_serving_repo() -> str: "d_mean_tpot", "d_median_tpot", "d_p99_tpot", + "d_al", ] # Metrics where smaller is better @@ -144,9 +253,12 @@ def ensure_bench_serving_repo() -> str: "d_mean_e2el", "d_median_e2el", "d_p99_e2el", + # gen_only-only: per-iter device step time averaged across gen workers + "d_mean_gen_worker_per_iter_device_step_time", ] -# Key metrics that determine regression (throughput metrics only) +# Default key metrics that determine regression (throughput metrics only). +# d_al is appended at runtime when any client runs spec decoding. REGRESSION_METRICS = [ "d_token_throughput", "d_total_token_throughput", @@ -186,6 +298,15 @@ def to_env_dict(env_vars: str) -> Dict[str, str]: return env +def force_num_accepted_tokens_from_env_str(env_vars: str) -> int: + """Extract TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS from a space-separated KEY=val env-var string. + + Returns 0 when not set. + """ + val = to_env_dict(env_vars).get("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") + return int(val) if val is not None else 0 + + def add_host_port_to_cmd(cmd: List[str], host: str, port: int) -> List[str]: """Add host and port to command.""" return cmd + ["--host", host, "--port", str(port)] @@ -200,6 +321,7 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): self.model_name = server_config_data["model_name"] self.model_path = "" self.env_vars = env_vars + self.force_num_accepted_tokens = force_num_accepted_tokens_from_env_str(env_vars) self.disagg_run_type = server_config_data.get("disagg_run_type", "aggr") # Extract optional fields with defaults @@ -313,6 +435,7 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): "match_mode", "backend", "extra_llm_api_config_path", + "server_env_var", ] self.extra_llm_api_config_data = { k: v for k, v in server_config_data.items() if k not in exclude_keys @@ -365,6 +488,7 @@ def to_match_keys(self) -> List[str]: # backfill completes. "s_spec_decoding_type", "l_num_nextn_predict_layers", + "l_force_num_accepted_tokens", # moe_config "l_load_balancer_num_slots", ] @@ -419,6 +543,7 @@ def to_db_data(self) -> dict: "l_num_nextn_predict_layers": self.max_draft_len, "s_eagle3_layers_to_capture": ",".join(map(str, self.eagle3_layers_to_capture)), "l_max_draft_len": self.max_draft_len, + "l_force_num_accepted_tokens": self.force_num_accepted_tokens, "s_speculative_model_dir": self.speculative_model, "b_eagle3_one_model": self.eagle3_one_model, "s_server_log_link": "", @@ -455,6 +580,134 @@ def generate_extra_llm_api_config(self) -> str: return yaml.dump(config_data, default_flow_style=False, sort_keys=False) +class AccuracyConfig: + """Accuracy test configuration (lm_eval against the running server). + + Shape mirrors the existing top-level `accuracy:` block in disagg yamls: + enable_accuracy_test: bool + env_var: dict[str, str] + tasks: + : + model: local-completions | local-chat-completions + model_args_extra: str + extra_kwargs: dict # forwarded to lm_eval as -- + """ + + _ENDPOINT_MAP = { + "local-completions": "v1/completions", + "local-chat-completions": "v1/chat/completions", + } + + def __init__(self, accuracy_data: dict): + self.enable_accuracy_test = bool(accuracy_data.get("enable_accuracy_test", False)) + self.env_var = dict(accuracy_data.get("env_var") or {}) + self.tasks = dict(accuracy_data.get("tasks") or {}) + + @classmethod + def from_dict(cls, data: Optional[dict]) -> Optional["AccuracyConfig"]: + if not data: + return None + # Short-circuit when accuracy is disabled so we don't try to parse + # legacy-shape `tasks:` (a string instead of the expected dict). + if not bool(data.get("enable_accuracy_test", False)): + return None + return cls(data) + + def build_lm_eval_invocations( + self, + model_name: str, + server_hostname: str, + server_port: int, + output_dir: str, + server_idx: int, + ) -> List[Tuple[List[str], str, Dict[str, str], str]]: + """Build (cmd, log_file, env, task_name) tuples for each configured task.""" + model_path = get_model_dir(model_name) + invocations = [] + for task_name, task_cfg in self.tasks.items(): + model_type = task_cfg.get("model", "local-completions") + model_args_extra = task_cfg.get("model_args_extra", "") + extra_kwargs = dict(task_cfg.get("extra_kwargs") or {}) + base_url = ( + f"http://{server_hostname}:{server_port}/" + f"{self._ENDPOINT_MAP.get(model_type, 'v1/completions')}" + ) + model_args = f"model={model_path},base_url={base_url},{model_args_extra}" + + acc_output_dir = os.path.join(output_dir, f"accuracy_eval_{task_name}.{server_idx}") + log_file = os.path.join(output_dir, f"accuracy_eval_{task_name}.{server_idx}.log") + os.makedirs(acc_output_dir, exist_ok=True) + + cmd = [ + "lm_eval", + "--model", + model_type, + "--tasks", + task_name, + "--model_args", + model_args, + "--log_samples", + "--output_path", + acc_output_dir, + ] + + include_path = extra_kwargs.pop("include_path", None) + custom_config = extra_kwargs.pop("custom_config", None) + if custom_config and not include_path: + # Substitute LLM_MODELS_ROOT (and other env vars) in the lm_eval + # task yaml, write to /lm_eval_configs/, and pass the + # directory to --include_path. lm_eval requires a directory. + cfg_path = ( + custom_config + if os.path.isabs(custom_config) + else os.path.join(get_llm_root(), custom_config) + ) + lm_eval_dir = os.path.join(output_dir, "lm_eval_configs") + os.makedirs(lm_eval_dir, exist_ok=True) + with open(cfg_path, "r", encoding="utf-8") as f: + content = f.read() + content = content.replace("LLM_MODELS_ROOT", llm_models_root()) + out_path = os.path.join(lm_eval_dir, os.path.basename(cfg_path)) + with open(out_path, "w", encoding="utf-8") as f: + f.write(content) + # Copy sibling utils.py if present (some tasks like GPQA need it) + sibling_utils = os.path.join(os.path.dirname(cfg_path), "utils.py") + if os.path.exists(sibling_utils): + shutil.copy(sibling_utils, lm_eval_dir) + include_path = lm_eval_dir + if include_path: + cmd += ["--include_path", include_path] + + for k, v in extra_kwargs.items(): + if isinstance(v, bool): + if v: + cmd += [f"--{k}"] + else: + cmd += [f"--{k}", str(v)] + + run_env = copy.deepcopy(os.environ) + run_env.update({k: str(v) for k, v in self.env_var.items()}) + invocations.append((cmd, log_file, run_env, task_name)) + return invocations + + def run( + self, + model_name: str, + server_hostname: str, + server_port: int, + output_dir: str, + server_idx: int, + ) -> None: + """Run all configured accuracy tasks against the live server.""" + for cmd, log_file, run_env, task_name in self.build_lm_eval_invocations( + model_name, server_hostname, server_port, output_dir, server_idx + ): + print_info(f"[Accuracy] Running {task_name}, output: {log_file}") + with open(log_file, "w") as lf: + ret = subprocess.run(cmd, env=run_env, stdout=lf, stderr=subprocess.STDOUT) + print_info(f"[Accuracy] {task_name} done, exit_code={ret.returncode}") + + class ClientConfig: """Configurations of benchmark client.""" @@ -479,10 +732,19 @@ def __init__( self.dataset_file = client_config_data.get("dataset_file", "") self.use_nv_sa_benchmark = client_config_data.get("use_nv_sa_benchmark", False) self.env_vars = env_vars - # --ignore-eos must be off when spec decoding is enabled: forcing generation - # past EOS produces unstable acceptance rates. + # spec_decoding flag is retained for DB matching (b_eos column). --ignore-eos + # is now always passed; output-length stability with spec decoding comes from + # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS (set per-yaml). self.spec_decoding = spec_decoding + # Accuracy testing (lm_eval after benchmark). only_run_accuracy is silently + # ignored when no accuracy_config is present or enable_accuracy_test=False. + self.accuracy_config = AccuracyConfig.from_dict(client_config_data.get("accuracy_config")) + only_run_accuracy = bool(client_config_data.get("only_run_accuracy", False)) + self.only_run_accuracy = only_run_accuracy and bool( + self.accuracy_config and self.accuracy_config.enable_accuracy_test + ) + # Generate default name if not provided self.name = client_config_data.get("name", "") if not self.name: @@ -521,9 +783,8 @@ def _to_sa_benchmark_cmd(self) -> List[str]: "--save-result", "--percentile-metrics", "ttft,tpot,itl,e2el", + "--ignore-eos", ] - if not self.spec_decoding: - benchmark_cmd.append("--ignore-eos") if self.backend: benchmark_cmd.extend(["--backend", self.backend]) if self.trust_remote_code: @@ -551,9 +812,8 @@ def _to_default_benchmark_cmd(self) -> List[str]: "--no-test-input", "--percentile-metrics", "ttft,tpot,itl,e2el", + "--ignore-eos", ] - if not self.spec_decoding: - benchmark_cmd.append("--ignore-eos") if dataset_path: benchmark_cmd.append("--dataset-name") benchmark_cmd.append("trtllm_custom") @@ -605,10 +865,9 @@ def to_match_keys(self) -> List[str]: def to_db_data(self) -> dict: """Convert ClientConfig to database data.""" - # b_eos = True when --ignore-eos is NOT used (EOS honored). Historical - # rows lack this field; _match() treats missing b_* as False, so legacy - # baselines (which always had --ignore-eos) only match new b_eos=False - # rows, while spec-decoding (b_eos=True) becomes a new test case. + # b_eos retained for baseline-matching continuity. spec-decoding runs are now + # further differentiated from non-spec-decoding runs via the + # l_force_num_accepted_tokens match key on ServerConfig. db_data = { "s_client_name": self.name, "l_concurrency": self.concurrency, @@ -669,16 +928,26 @@ class AggrTestCmds(NamedTuple): timeout: int output_dir: str test_output_dir: str + client_configs: Dict[int, List["ClientConfig"]] = {} + model_name: str = "" + server_configs: List["ServerConfig"] = [] def get_server_logs(self, server_idx) -> List[str]: server_file_path = os.path.join(self.test_output_dir, f"trtllm-serve.{server_idx}.log") return [server_file_path] def run_cmd(self, server_idx: int) -> List[str]: - """Run all clients for a server and return outputs.""" + """Run all clients for a server and return outputs. + + For each client: starts benchmark unless only_run_accuracy=True; runs + accuracy_config (lm_eval) afterward when configured. Empty string is + appended to outputs for only_run_accuracy clients to keep client_idx + aligned with self.client_cmds[server_idx]. + """ outputs = [] server_proc = None server_cmd = self.server_cmds[server_idx] + client_configs = self.client_configs.get(server_idx, []) try: server_hostname = "localhost" @@ -687,10 +956,13 @@ def run_cmd(self, server_idx: int) -> List[str]: print_info(f"Starting server. cmd is {server_cmd_with_port}") server_file_path = os.path.join(self.test_output_dir, f"trtllm-serve.{server_idx}.log") + server_env = copy.deepcopy(os.environ) + if server_idx < len(self.server_configs): + server_env.update(self.server_configs[server_idx].to_env()) with open(server_file_path, "w") as server_ctx: server_proc = subprocess.Popen( server_cmd_with_port, - env=copy.deepcopy(os.environ), + env=server_env, stdout=server_ctx, stderr=subprocess.STDOUT, ) @@ -704,23 +976,50 @@ def run_cmd(self, server_idx: int) -> List[str]: # Run all clients for this server for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]): - client_file_path = os.path.join( - self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" - ) - client_cmd_with_port = add_host_port_to_cmd( - client_cmd, server_hostname, server_port + client_config = ( + client_configs[client_idx] if client_idx < len(client_configs) else None ) - print_info(f"Starting client. cmd is {client_cmd_with_port}") + only_run_accuracy = bool(client_config and client_config.only_run_accuracy) - output = subprocess.check_output( - client_cmd_with_port, - stderr=subprocess.STDOUT, - env=copy.deepcopy(os.environ), - ).decode() + if not only_run_accuracy: + client_file_path = os.path.join( + self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" + ) + client_cmd_with_port = add_host_port_to_cmd( + client_cmd, server_hostname, server_port + ) + print_info(f"Starting client. cmd is {client_cmd_with_port}") + + client_env = copy.deepcopy(os.environ) + if client_config: + client_env.update(client_config.to_env()) + output = subprocess.check_output( + client_cmd_with_port, + stderr=subprocess.STDOUT, + env=client_env, + ).decode() + + with open(client_file_path, "w") as client_ctx: + client_ctx.write(output) + outputs.append(output) + else: + print_info( + f"Skipping perf benchmark for client {client_idx}: only_run_accuracy=True" + ) + outputs.append("") - with open(client_file_path, "w") as client_ctx: - client_ctx.write(output) - outputs.append(output) + if ( + client_config + and client_config.accuracy_config + and client_config.accuracy_config.enable_accuracy_test + ): + client_config.accuracy_config.run( + model_name=self.model_name or client_config.model_name, + server_hostname=server_hostname, + server_port=server_port, + output_dir=self.test_output_dir, + server_idx=server_idx, + ) finally: if server_proc: @@ -746,6 +1045,13 @@ class DisaggTestCmds(NamedTuple): output_dir: str test_output_dir: str model_name: str = "" + client_configs: Dict[int, List["ClientConfig"]] = {} + # Per-server-index ServerConfig triples (ctx_config, gen_config, disagg_config). + # Used by run_cmd() to merge per-config env vars into the appropriate + # subprocess env based on this rank's disagg_serving_type. For multi-node + # disagg, only rank-0 pytest goes through this path; multi-rank workers + # receive env via SLURM env propagation set up by submit.py. + server_configs: List[Tuple["ServerConfig", "ServerConfig", "DisaggConfig"]] = [] def _generate_hostname_file(self, server_idx: int, port: int): """Create hostname file for coordination.""" @@ -861,16 +1167,16 @@ def get_server_logs(self, server_idx: int) -> List[str]: server_logs.append( os.path.join(self.test_output_dir, f"trtllm-serve.CTX_{i}.{server_idx}.log") ) - server_logs.append(os.path.join(self.output_dir, f"ctx_server_{i}.log")) + server_logs.append(os.path.join(self.test_output_dir, f"ctx_server_{i}.log")) for i in range(self.num_gen_servers): server_logs.append( os.path.join(self.test_output_dir, f"trtllm-serve.GEN_{i}.{server_idx}.log") ) - server_logs.append(os.path.join(self.output_dir, f"gen_server_{i}.log")) + server_logs.append(os.path.join(self.test_output_dir, f"gen_server_{i}.log")) server_logs.append( os.path.join(self.test_output_dir, f"trtllm-serve.DISAGG_SERVER.{server_idx}.log") ) - server_logs.append(os.path.join(self.output_dir, "disagg_server.log")) + server_logs.append(os.path.join(self.test_output_dir, "disagg_server.log")) return server_logs @staticmethod @@ -893,6 +1199,9 @@ def run_cmd(self, server_idx: int) -> List[str]: self.test_output_dir, f"benchmark_status.{server_idx}.txt" ) ctx_cmd, gen_cmd, disagg_cmd = self.server_cmds[server_idx] + configs_for_idx = ( + self.server_configs[server_idx] if server_idx < len(self.server_configs) else None + ) if "CTX" in self.disagg_serving_type or "GEN" in self.disagg_serving_type: port = get_free_port() self._generate_hostname_file(server_idx, port) @@ -913,10 +1222,14 @@ def run_cmd(self, server_idx: int) -> List[str]: self.test_output_dir, f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log", ) + worker_env = copy.deepcopy(os.environ) + if configs_for_idx is not None: + ctx_cfg, gen_cfg, _ = configs_for_idx + worker_env.update((ctx_cfg if is_ctx else gen_cfg).to_env()) with open(server_file_path, "w") as server_ctx: server_proc = subprocess.Popen( server_cmd, - env=copy.deepcopy(os.environ), + env=worker_env, stdout=server_ctx, stderr=subprocess.STDOUT, ) @@ -934,10 +1247,14 @@ def run_cmd(self, server_idx: int) -> List[str]: self.test_output_dir, f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log", ) + disagg_env = copy.deepcopy(os.environ) + if configs_for_idx is not None: + _, _, disagg_cfg = configs_for_idx + disagg_env.update(to_env_dict(disagg_cfg.server_env_var)) with open(disagg_server_file_path, "w") as disagg_server_ctx: disagg_server_proc = subprocess.Popen( disagg_cmd, - env=copy.deepcopy(os.environ), + env=disagg_env, stdout=disagg_server_ctx, stderr=subprocess.STDOUT, ) @@ -959,41 +1276,86 @@ def run_cmd(self, server_idx: int) -> List[str]: check_files=self.get_server_logs(server_idx), ) + client_configs = self.client_configs.get(server_idx, []) + # Run all clients for this server for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]): - benchmark_file_path = os.path.join( - self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" - ) - client_cmd_with_port = add_host_port_to_cmd( - client_cmd, disagg_server_hostname, disagg_server_port + client_config = ( + client_configs[client_idx] if client_idx < len(client_configs) else None ) - print_info(f"Starting benchmark. cmd is {client_cmd_with_port}") + only_run_accuracy = bool(client_config and client_config.only_run_accuracy) - output = subprocess.check_output( - client_cmd_with_port, - env=copy.deepcopy(os.environ), - stderr=subprocess.STDOUT, - ).decode() + if not only_run_accuracy: + benchmark_file_path = os.path.join( + self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" + ) + client_cmd_with_port = add_host_port_to_cmd( + client_cmd, disagg_server_hostname, disagg_server_port + ) + print_info(f"Starting benchmark. cmd is {client_cmd_with_port}") - with open(benchmark_file_path, "w") as benchmark_ctx: - benchmark_ctx.write(output) - outputs.append(output) + # Snapshot gen_server log sizes so the per-client + # average covers only iterations driven by this client. + gen_log_start_offsets = gen_worker_log_sizes( + self.test_output_dir, self.num_gen_servers + ) - # Run accuracy tests after benchmark (if configured) - acc_cfg_json = os.environ.get("ACCURACY_CONFIG_JSON") - if acc_cfg_json: - import json as _json - - acc_cfg = _json.loads(acc_cfg_json) - if acc_cfg.get("enable_accuracy_test"): - _run_accuracy_tests( - acc_cfg, - self.model_name, - disagg_server_hostname, - disagg_server_port, + bench_env = copy.deepcopy(os.environ) + if client_config: + bench_env.update(client_config.to_env()) + output = subprocess.check_output( + client_cmd_with_port, + env=bench_env, + stderr=subprocess.STDOUT, + ).decode() + + with open(benchmark_file_path, "w") as benchmark_ctx: + benchmark_ctx.write(output) + + # Only gen_only emits prev_device_step_time; other + # modes yield None and we skip writing the line. + device_step_time_mean = parse_gen_worker_device_step_time( self.test_output_dir, - server_idx, + self.num_gen_servers, + start_offsets=gen_log_start_offsets, + ) + if device_step_time_mean is not None: + summary_line = ( + f"Average Per Iter Device Step Time (ms): {device_step_time_mean}" + ) + with open(benchmark_file_path, "a") as benchmark_ctx: + benchmark_ctx.write(f"\n{summary_line}\n") + output = f"{output}\n{summary_line}\n" + + outputs.append(output) + else: + print_info( + f"Skipping perf benchmark for client {client_idx}: " + "only_run_accuracy=True" ) + outputs.append("") + + # Prefer per-client AccuracyConfig (sourced from yaml). Fall back + # to ACCURACY_CONFIG_JSON env-var injected by submit.py for older + # workflows. + accuracy_cfg = None + if client_configs and client_configs[0].accuracy_config: + accuracy_cfg = client_configs[0].accuracy_config + else: + acc_cfg_json = os.environ.get("ACCURACY_CONFIG_JSON") + if acc_cfg_json: + import json as _json + + accuracy_cfg = AccuracyConfig.from_dict(_json.loads(acc_cfg_json)) + + if accuracy_cfg and accuracy_cfg.enable_accuracy_test: + accuracy_cfg.run( + model_name=self.model_name, + server_hostname=disagg_server_hostname, + server_port=disagg_server_port, + output_dir=self.test_output_dir, + server_idx=server_idx, + ) finally: with open(benchmark_status_file, "w") as status_file: @@ -1005,60 +1367,6 @@ def get_cmd_str(self, server_idx: int) -> List[str]: return ["multi-node disaggregated server tests, please check config files"] -def _run_accuracy_tests( - accuracy_cfg: dict, - model_name: str, - server_hostname: str, - server_port: int, - output_dir: str, - server_idx: int, -) -> None: - """Run lm_eval against the running disagg server. Saves results only — no validation.""" - endpoint_map = { - "local-completions": "v1/completions", - "local-chat-completions": "v1/chat/completions", - } - env_var = accuracy_cfg.get("env_var") or {} - model_path = get_model_dir(model_name) - - for task_name, task_cfg in accuracy_cfg.get("tasks", {}).items(): - model_type = task_cfg.get("model", "local-completions") - model_args_extra = task_cfg.get("model_args_extra", "") - extra_kwargs = task_cfg.get("extra_kwargs", {}) - base_url = f"http://{server_hostname}:{server_port}/{endpoint_map.get(model_type, 'v1/completions')}" - model_args = f"model={model_path},base_url={base_url},{model_args_extra}" - - acc_output_dir = os.path.join(output_dir, f"accuracy_eval_{task_name}.{server_idx}") - log_file = os.path.join(output_dir, f"accuracy_eval_{task_name}.{server_idx}.log") - os.makedirs(acc_output_dir, exist_ok=True) - - cmd = [ - "lm_eval", - "--model", - model_type, - "--tasks", - task_name, - "--model_args", - model_args, - "--log_samples", - "--output_path", - acc_output_dir, - ] - if "include_path" in extra_kwargs: - cmd += ["--include_path", extra_kwargs["include_path"]] - for k, v in extra_kwargs.items(): - if k == "include_path": - continue - cmd += [f"--{k}"] if isinstance(v, bool) and v else [f"--{k}", str(v)] - - run_env = copy.deepcopy(os.environ) - run_env.update({k: str(v) for k, v in env_var.items()}) - print_info(f"[Accuracy] Running {task_name}, output: {log_file}") - with open(log_file, "w") as lf: - ret = subprocess.run(cmd, env=run_env, stdout=lf, stderr=subprocess.STDOUT) - print_info(f"[Accuracy] {task_name} done, exit_code={ret.returncode}") - - def parse_select_pattern(select_pattern: str) -> list: """Parse select pattern (server config names). @@ -1227,13 +1535,10 @@ def _parse_aggr_config_file(self, config_file_path: str): config = yaml.safe_load(f) metadata = config.get("metadata", {}) - environment = config.get("environment", {}) hardware = config.get("hardware", {}) gpus_per_node = hardware.get("gpus_per_node", 0) model_name = metadata.get("model_name", "") - server_env_var = environment.get("server_env_var", "") - client_env_var = environment.get("client_env_var", "") server_configs = [] server_client_configs = {} @@ -1254,12 +1559,16 @@ def _parse_aggr_config_file(self, config_file_path: str): server_config_data["concurrency"] = -1 server_config_data["gpus_per_node"] = gpus_per_node + # Per-config env vars: server_env_var lives on each server_config entry, + # client_env_var lives on each client_config entry. + server_env_var = server_config_data.get("server_env_var", "") server_config = ServerConfig(server_config_data, server_env_var) server_id = len(server_configs) server_configs.append(server_config) client_configs = [] for client_config_data in server_config_data["client_configs"]: + client_env_var = client_config_data.get("client_env_var", "") client_config = ClientConfig( client_config_data, server_config_data["model_name"], @@ -1398,6 +1707,11 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): gen_server_config.spec_decoding_type ) + # Accuracy lives at the top of disagg yamls; only_run_accuracy lives inside + # benchmark: (since `benchmark` is what becomes the disagg ClientConfig). + accuracy_data = config.get("accuracy") or None + only_run_accuracy = bool(benchmark.get("only_run_accuracy", False)) + client_configs = [] for concurrency in concurrency_values: client_config_data = { @@ -1413,6 +1727,8 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): "streaming": benchmark.get("streaming", True), "dataset_file": dataset_file, "use_nv_sa_benchmark": use_nv_sa_benchmark, + "accuracy_config": accuracy_data, + "only_run_accuracy": only_run_accuracy, } client_config = ClientConfig( client_config_data, @@ -1458,12 +1774,20 @@ def _get_aggr_commands(self, output_dir: str, test_output_dir: str): client_cmd = client_config.to_cmd() client_cmds[server_idx].append(client_cmd) + # AggrTestCmds needs the model name (for lm_eval --model_args). All + # server_configs in an agg yaml share the same model_name. + first_server = self.server_configs[0] if self.server_configs else None + agg_model_name = first_server.model_name if first_server else "" + return AggrTestCmds( server_cmds=server_cmds, client_cmds=client_cmds, timeout=DEFAULT_TIMEOUT, output_dir=output_dir, test_output_dir=test_output_dir, + client_configs=self.server_client_configs, + model_name=agg_model_name, + server_configs=list(self.server_configs), ) def _get_disagg_commands(self, output_dir: str, test_output_dir: str): @@ -1528,6 +1852,8 @@ def _get_disagg_commands(self, output_dir: str, test_output_dir: str): output_dir=output_dir, test_output_dir=test_output_dir, model_name=disagg_config.model_name, + client_configs=self.server_client_configs, + server_configs=list(self.server_configs), ) def _check_benchmark_errors(self, output: str) -> None: @@ -1577,10 +1903,16 @@ def run_ex(self, commands) -> Dict[int, List[str]]: except Exception as e: outputs[server_idx] = [] - report_error( - error_msg=e, - log_files=commands.get_server_logs(server_idx), - ) + # Aggregated mode does not set DISAGG_SERVING_TYPE, so the + # default "BENCHMARK" applies and report_error is always called. + # Disagg mode sets DISAGG_SERVING_TYPE per srun; only the + # BENCHMARK srun reports errors gathered from sibling logs. + if os.environ.get("DISAGG_SERVING_TYPE", "BENCHMARK") == "BENCHMARK": + report_error( + error_msg=e, + log_files=commands.get_server_logs(server_idx), + ) + raise return outputs @@ -1590,8 +1922,13 @@ def get_perf_result(self, outputs: Dict[int, List[str]]): def parse_metrics_from_output(output: str) -> Optional[Dict[str, float]]: """Parse all metrics from a single output string.""" metrics = {} + all_queries = { + **PERF_METRIC_LOG_QUERIES, + **SPEC_DECODING_PERF_METRIC_LOG_QUERIES, + **GEN_ONLY_PERF_METRIC_LOG_QUERIES, + } for line in output.split("\n"): - for metric_type, regex in PERF_METRIC_LOG_QUERIES.items(): + for metric_type, regex in all_queries.items(): if metric_type in metrics: continue match = regex.search(line) @@ -1605,6 +1942,14 @@ def parse_metrics_from_output(output: str) -> Optional[Dict[str, float]]: self._perf_results[server_idx] = [] server_outputs = outputs.get(server_idx, []) for client_idx, output in enumerate(server_outputs): + # only_run_accuracy clients have no benchmark output to parse; + # use None sentinel so check/upload paths can skip them. + if ( + client_idx < len(client_configs) + and client_configs[client_idx].only_run_accuracy + ): + self._perf_results[server_idx].append(None) + continue metrics = parse_metrics_from_output(output) # SA benchmark (bench_serving) doesn't report user_throughput. # Use None as sentinel to distinguish "not available" from actual zero. @@ -1628,13 +1973,48 @@ def check_test_failure(self): f"is not equal to client number: {len(client_configs)}. " ) for client_idx, metrics in enumerate(server_perf_results): - if len(metrics) != len(PERF_METRIC_LOG_QUERIES): + # only_run_accuracy clients produce no perf metrics by design. + if ( + client_idx < len(client_configs) + and client_configs[client_idx].only_run_accuracy + ): + continue + missing = [k for k in PERF_METRIC_LOG_QUERIES if k not in (metrics or {})] + if missing: + error_msg += ( + f"Some metrics in Server {server_idx} Client {client_idx} are missing: " + f"{missing}. The parsed metrics is {metrics}. " + ) + # Spec-decoding tests must report 'Mean Avg Decoded Tokens per Iter' + # (parsed as 'al'). If the field is missing the test fails here so the + # data is never uploaded to OpenSearch. + if ( + client_idx < len(client_configs) + and client_configs[client_idx].spec_decoding + and "al" not in metrics + ): + error_msg += ( + f"Speculative decoding test Server {server_idx} Client {client_idx} " + f"is missing 'Mean Avg Decoded Tokens per Iter' in benchmark output. " + ) + # gen_only tests must report mean_gen_worker_per_iter_device_step_time + # (parsed from gen_server_*.log). It is the sole regression metric for + # gen_only, so a missing value must hard-fail rather than silently upload. + if ( + self.runtime == "multi_node_disagg_server" + and self.server_configs[server_idx][2].benchmark_mode == "gen_only" + and ( + not metrics + or metrics.get("mean_gen_worker_per_iter_device_step_time") is None + ) + ): error_msg += ( - f"Some metrics in Server {server_idx} Client {client_idx} are missing. " - f"The broken metrics is {metrics}. " + f"gen_only test Server {server_idx} Client {client_idx} is " + f"missing 'prev_device_step_time' in gen_server_*.log under " + f"{self._output_dir}. " ) if error_msg: - raise Exception(error_msg) + raise RuntimeError(error_msg) def upload_test_results_to_database(self): """Upload test results and baseline to database.""" @@ -1686,8 +2066,11 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{server_config.name}-{client_config.name}" - for metric_name in PERF_METRIC_LOG_QUERIES: - new_data[f"d_{metric_name}"] = server_perf_results[client_idx][metric_name] + add_perf_metric_value( + new_data, + server_perf_results[client_idx], + spec_decoding=client_config.spec_decoding, + ) new_data_dict[cmd_idx] = new_data cmd_idx += 1 @@ -1748,8 +2131,12 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{disagg_config.name}-{client_config.name}" - for metric_name in PERF_METRIC_LOG_QUERIES: - new_data[f"d_{metric_name}"] = server_perf_results[client_idx][metric_name] + add_perf_metric_value( + new_data, + server_perf_results[client_idx], + spec_decoding=client_config.spec_decoding, + benchmark_mode=disagg_config.benchmark_mode, + ) new_data_dict[cmd_idx] = new_data cmd_idx += 1 @@ -1784,12 +2171,30 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: "s_test_list": self._test_param_labels, } + # gen_only tests are gated solely on per-iter prev_device_step_time, not + # token throughput (token-based numbers are dominated by KV cache transfer + # time in gen_only mode and are not a useful regression signal there). + # For all other modes, d_al is added when any client runs spec decoding. + if self.runtime == "multi_node_disagg_server" and any( + sc[2].benchmark_mode == "gen_only" for sc in self.server_configs + ): + regression_metrics = ["d_mean_gen_worker_per_iter_device_step_time"] + else: + regression_metrics = list(REGRESSION_METRICS) + has_spec_decoding = any( + cc.spec_decoding + for clients in self.server_client_configs.values() + for cc in clients + ) + if has_spec_decoding: + regression_metrics.append("d_al") + process_and_upload_test_results( new_data_dict=new_data_dict, match_keys=match_keys, maximize_metrics=MAXIMIZE_METRICS, minimize_metrics=MINIMIZE_METRICS, - regression_metrics=REGRESSION_METRICS, + regression_metrics=regression_metrics, extra_fields=extra_fields, upload_to_db=self.upload_to_db, ) diff --git a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml index f751033a0b6e..3747dd2b0cc6 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml @@ -1,5 +1,6 @@ server_configs: - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -33,6 +34,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -66,6 +68,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -104,6 +107,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -137,6 +141,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -169,6 +174,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc2048_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -207,6 +213,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -240,6 +247,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -278,6 +286,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc2048_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -316,6 +325,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -348,6 +358,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -415,6 +426,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -448,6 +460,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -481,6 +494,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc1024_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -518,6 +532,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc2048_gpu4 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario @@ -555,6 +570,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -588,6 +604,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc16_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -621,6 +638,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc512_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -657,6 +675,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc1024_gpu4 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario @@ -695,6 +714,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc2048_gpu4 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario diff --git a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml index d28014bc0027..165680ce14ac 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml @@ -1,5 +1,6 @@ server_configs: - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -33,6 +34,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc32_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -66,6 +68,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -104,6 +107,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -137,6 +141,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc16_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -170,6 +175,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc512_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -207,6 +213,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc1_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -240,6 +247,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc16_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -273,6 +281,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc256_gpu8 + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml index 19811cd50cf4..217fb82d6e33 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP16 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep16_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 16 moe_expert_parallel_size: 16 @@ -43,6 +44,7 @@ server_configs: # 8k1k configs - DEP16 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep16_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 16 moe_expert_parallel_size: 16 @@ -79,6 +81,7 @@ server_configs: # 1k1k configs - TEP16 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep16_mtp3" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 16 moe_expert_parallel_size: 16 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index b3bee56a5a8a..19bafb9ffe88 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -44,6 +45,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -81,6 +83,7 @@ server_configs: # 1k1k configs - TEP8 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep8_mtp3" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml index f92f3d80f876..268389e6c1bf 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - TP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tp4_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 1 @@ -38,6 +39,7 @@ server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -74,6 +76,7 @@ server_configs: # 8k1k configs - TP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tp4_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 1 @@ -105,6 +108,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml index fad67e46ba77..8d3623c18c34 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml @@ -8,6 +8,7 @@ hardware: server_configs: # 1k1k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -44,6 +45,7 @@ server_configs: # 1k1k configs - TEP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep4_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -75,6 +77,7 @@ server_configs: # 1k1k configs - TP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tp4_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 1 @@ -106,6 +109,7 @@ server_configs: # 8k1k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -142,6 +146,7 @@ server_configs: # 8k1k configs - TEP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep4_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -173,6 +178,7 @@ server_configs: # 8k1k configs - TP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tp4_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 1 @@ -204,6 +210,7 @@ server_configs: # 1k8k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_1k8k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -240,6 +247,7 @@ server_configs: # 1k8k configs - TEP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep4_mtp3_1k8k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -271,6 +279,7 @@ server_configs: # 1k8k configs - TP4 with TRTLLM, MTP3 - name: "r1_fp4_v2_tp4_mtp3_1k8k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml index 22dc57e9c93d..f6509f0ffeae 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - TP8 with TRTLLM, MTP3 - name: "r1_fp8_tp8_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 1 @@ -38,6 +39,7 @@ server_configs: # 1k1k configs - DEP8 with DEEPGEMM, MTP1 - name: "r1_fp8_dep8_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -74,6 +76,7 @@ server_configs: # 8k1k configs - TP8 with TRTLLM, MTP3 - name: "r1_fp8_tp8_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 1 @@ -105,6 +108,7 @@ server_configs: # 8k1k configs - DEP8 with DEEPGEMM, MTP1 - name: "r1_fp8_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml index a8407ed03c8c..108f5f7cac52 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 8k1k configs - TEP8 with TRTLLM, MTP3 - name: "v32_fp4_tep8_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_v32_fp4" tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -38,6 +39,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "v32_fp4_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_v32_fp4" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml index a6f7206d5054..77554cd50771 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - TEP4 with TRTLLM, MTP3 - name: "v32_fp4_tep4_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -38,6 +39,7 @@ server_configs: # 1k1k configs - DEP4 with CUTEDSL, MTP1 - name: "v32_fp4_dep4_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -74,6 +76,7 @@ server_configs: # 8k1k configs - TEP4 with TRTLLM, MTP3 - name: "v32_fp4_tep4_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -105,6 +108,7 @@ server_configs: # 8k1k configs - DEP4 with CUTEDSL, MTP1 - name: "v32_fp4_dep4_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 diff --git a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index 2d4822a3e121..b67170b6f52d 100644 --- a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -7,6 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -44,6 +45,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -81,6 +83,7 @@ server_configs: # 1k1k configs - TEP8 with TRTLLM, MTP3 - name: "r1_fp4_v2_tep8_mtp3" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml index aa4853911bc2..e176fe7dd517 100644 --- a/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml @@ -127,6 +127,7 @@ server_configs: dataset_file: datasets/perf-ci/gpt_oss_120b-1k8k-20480-ratio-1_for_serve.json - name: "gpt_oss_fp4_tp4_eagle3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "gpt_oss_120b_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/aggregated/host_perf_llama8b_spec_decode.yaml b/tests/scripts/perf-sanity/aggregated/host_perf_llama8b_spec_decode.yaml index fdbc1a6a140a..513e5d01da40 100644 --- a/tests/scripts/perf-sanity/aggregated/host_perf_llama8b_spec_decode.yaml +++ b/tests/scripts/perf-sanity/aggregated/host_perf_llama8b_spec_decode.yaml @@ -30,6 +30,7 @@ server_configs: # The host overhead per iteration is ~2x non-spec-decode due to # draft token management and verification. - name: "llama8b_spec_bs1_128_128" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "llama_v3.1_8b_instruct" tensor_parallel_size: 1 max_batch_size: 4 @@ -55,6 +56,7 @@ server_configs: # BS=8 with spec decode means 8 requests each generating 3 draft tokens, # stressing the scheduler, KV cache, and token verification paths. - name: "llama8b_spec_bs8_128_128" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" model_name: "llama_v3.1_8b_instruct" tensor_parallel_size: 1 max_batch_size: 16 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 51ecc9a9593f..2fbda813db13 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index e069a98b1a99..a1cae7e00b90 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml index 7e32761afbc0..7424aec950a5 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 9026428e15b3..0c479ed61ce8 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2adda4bac501..a3421223a0eb 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 5a15c598d61d..9014fd9b86b5 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 12916108ff24..c185f1c1a28a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index e37b60c77438..23d0854324d2 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index e115acaa05af..74801acda6f8 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 5f9722f34ff2..e57ea6d424e8 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 6b0245deb307..50c56d99eb38 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 20706a01c585..c3ff2770bcb5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -41,8 +41,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 5ea03a618905..e13f5348ad24 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml index ab1dcc0a27fe..5cc8664e6dcb 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml index 4c3e453b405e..3e06cdd34340 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 43f0535d8383..2ed04b60d6f4 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index d161caea68eb..7ab651ae8e5c 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2d449f57a7a6..08d59450f353 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index c31910ec28f1..6f9aaf5a11a5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 216d38bcc41b..2ffed4acf11e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index cfd1a7826312..13763203780c 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index bf9ba21f0835..91c7d66ecc69 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index a0b88f589cf8..eb93763ec738 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 930876aa4403..d790464b256a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 2651f02c2b57..07fbb43780e9 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 9158df38b979..63f2ef1ae8f9 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 194e61864a20..ef23747a7ac6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 562c388d8450..883b280a28bd 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 877516abbe54..4691eb174302 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 23f9239f77d7..82b59b7097b5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index f378906e9b0e..e0b6133ae940 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 696147466387..1afe92603861 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 1e55e8fe0bc7..d267f10570a6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 021265456d6d..3761761e4e9d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index b6fba51707bf..7d7891f82571 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 6d2ef88af19e..2190a3cce964 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 3a3747731f78..0d19f8e4e27e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index ca773d4b268d..88c2fb4472ee 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 6417a0c38c8d..2f3d0e8a64ef 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -46,8 +46,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 0765412f2e62..4cfefd539365 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index bb027c657d2c..51365542c142 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 3a1d7fdbeffd..f1a77c15f944 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 8eb7293b9a87..df79c140338f 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index b03708e45ad6..187f5d8cb68e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 3ee92df5b4a2..1bd705d266db 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 00028abf8eee..38cd6e603fbd 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml index 4b7f537cb9a3..72e30ff24707 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index cef09d7dd066..536e81290c98 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 83aefbff6137..9f88d5fcca26 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index dde9fb64b2d3..934316dacfdc 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -40,11 +40,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 6a74a4fcfc36..28e604679b99 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 99440337a369..cecdab72aafd 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,16 +35,11 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index f01db9b09770..abb0195ae2fe 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,16 +35,11 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 1d6d0bdfcae8..719401153151 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,16 +35,11 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index c2bc0fee249b..5d77929a60a7 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 9a3f636173c3..89458072b018 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 50f19df4ec25..dd30d2b390d5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 780b4d71e1a0..a54762c3050c 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 6f3c36cda444..446b7663f2f1 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 802e1777d732..7bbe7d158147 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 669718d13894..11610c5a26ee 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index bd6d75d283a1..86b5e51260f7 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 73c0910392bb..c22d57a60b3a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 0eab593e0908..699424bd083e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index fea3b1a1cfc5..aa0f97a859a3 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 609c0c43d6d2..2919dec781b5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index fef69dbf9866..09ed51589940 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 5d4325064c64..609ef00b6968 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 16d743d373d8..2a119e6dcba7 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index ab0c5dd4949a..74efcde4fc8c 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 2b50cd55d8a7..c4955d589d93 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 640d363ba39c..0238f7e3b765 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 40c2b2f3b6d7..0af3d5389f3d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index a8ef83c55353..725cda90bb36 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml index 31a59ef73017..f45d79568878 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml @@ -39,11 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 90f3bcbf737b..537900ff777a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/h200_nemotron-super-fp8_8k1k_con64_ctx1_tp2_gen1_tp2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/h200_nemotron-super-fp8_8k1k_con64_ctx1_tp2_gen1_tp2_eplb0_mtp0_ccb-UCX.yaml index 2499fb4cf7f7..d0909bb78d58 100644 --- a/tests/scripts/perf-sanity/disaggregated/h200_nemotron-super-fp8_8k1k_con64_ctx1_tp2_gen1_tp2_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf-sanity/disaggregated/h200_nemotron-super-fp8_8k1k_con64_ctx1_tp2_gen1_tp2_eplb0_mtp0_ccb-UCX.yaml @@ -42,8 +42,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/h200_qwen3-235b-a22b-fp8_8k1k_con512_ctx1_tp2_gen1_tep4_eplb0_mtp0_ccb-DEFAULT.yaml b/tests/scripts/perf-sanity/disaggregated/h200_qwen3-235b-a22b-fp8_8k1k_con512_ctx1_tp2_gen1_tep4_eplb0_mtp0_ccb-DEFAULT.yaml index 0bf12a3fe300..743d8378bb07 100644 --- a/tests/scripts/perf-sanity/disaggregated/h200_qwen3-235b-a22b-fp8_8k1k_con512_ctx1_tp2_gen1_tep4_eplb0_mtp0_ccb-DEFAULT.yaml +++ b/tests/scripts/perf-sanity/disaggregated/h200_qwen3-235b-a22b-fp8_8k1k_con512_ctx1_tp2_gen1_tep4_eplb0_mtp0_ccb-DEFAULT.yaml @@ -42,8 +42,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT.yaml b/tests/scripts/perf-sanity/disaggregated/h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT.yaml index e7eb2ce64804..b12d5f5a93f9 100644 --- a/tests/scripts/perf-sanity/disaggregated/h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT.yaml +++ b/tests/scripts/perf-sanity/disaggregated/h200_qwen3-32b-fp8_4k1k_con128_ctx1_tp1_gen1_tp2_eplb0_mtp0_ccb-DEFAULT.yaml @@ -42,8 +42,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 12916108ff24..8b835dafdd08 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 355a7625bd3d..87b7e8bab148 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index eafb048e68b9..4ecc2c109ed4 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 0117cf4d0b5a..5fd232627cf8 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml index eb3f24492539..fd2677ca1579 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml @@ -39,8 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 1 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 5f9722f34ff2..e57ea6d424e8 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index aa31b144efbc..5d2b230f8195 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 20706a01c585..c3ff2770bcb5 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -41,8 +41,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 8 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index ce7d3b5be45d..e3b3ef2506d5 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml index ab1dcc0a27fe..84334736ffeb 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml index 3370d1faa8cb..9165da2fec27 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 43f0535d8383..4462db6324aa 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 1c56c251756d..b517a04bc69e 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml index 58ff5712bf75..dfe64bdcfc64 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 8 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml index ab47b8c1ef94..481d5978511f 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml @@ -35,13 +35,11 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 MIMALLOC_PURGE_DELAY=0 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 16 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 8c08fcde53d2..f2f793df5219 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 60bec288c4a2..7c2e9f2d770c 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index c31910ec28f1..33b4cb397858 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 7e77615fae38..4b24f9aa580e 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml index baf99fb99dcf..ee18edcd2fce 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml @@ -39,8 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 8 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index fcfbe0616235..b089131feecf 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2c3b2b19759d..9da91a2d426d 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index bf9ba21f0835..f0a97545cf0d 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 80602cb3df0a..89ec0ebd6acb 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2a91f3633c0d..f72d1b107939 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 930876aa4403..f08e6ef8c0eb 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 71a4e13ca5be..041ff3fb130e 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 99002c9f2558..40f1d141ab89 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 9158df38b979..63f2ef1ae8f9 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index e40afbe44267..78973f5ecc75 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 0a7b3a052e37..4fda7f7549f2 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 877516abbe54..0257c507e4d1 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml index 2c4a93be46a0..3256aebcc704 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 6d2ef88af19e..8eef9c444195 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml index c739b0f68751..c46d573cf25d 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 3a3747731f78..ff52e6819ca1 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index ca773d4b268d..8618b346f7ea 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 6417a0c38c8d..2f3d0e8a64ef 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -46,8 +46,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml index ec990076ef08..a3684f5fca53 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml @@ -46,8 +46,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 0765412f2e62..38b46c465050 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index bb027c657d2c..f663223f9b46 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml index 3a1d7fdbeffd..56521e5293c9 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 8eb7293b9a87..5853a2570802 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index b03708e45ad6..0c94eb253749 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml index 1ea511a1e7a3..a4c68609cb29 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 3ee92df5b4a2..268db9e6f545 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 00028abf8eee..b3d5ea9ab114 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml index 4b7f537cb9a3..8b4dff7bdf9c 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml index d84d38d742b7..04a85f5b3f40 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index cef09d7dd066..7633ba6bc22f 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml index 0848573d59fc..2e3783298ba9 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml @@ -39,8 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 8 diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml index 0210b1476498..9b97a86243ca 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 16 diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 83aefbff6137..d5909d08b308 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml index a97acb893057..e850787be69c 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index dde9fb64b2d3..b61478add80d 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -42,9 +42,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 6a74a4fcfc36..01315474b462 100644 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 00266300a083..12479d46933c 100644 --- a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -41,7 +41,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index ad6b5d9a8c47..67719a2796ce 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -41,7 +41,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index c455cc0644d4..9b541c5ed874 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -41,7 +41,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml index 2095c7e71106..fa8fbbf1027e 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml @@ -39,8 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml index ddd305d6c631..81d204655861 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml @@ -39,8 +39,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml index 96d798d29fb3..e0546168bf38 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml index 9c5e8d11f3e9..0e84cd8498ec 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml index a69cf3ef6920..c75b02889bd3 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml @@ -40,8 +40,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 51bf9b6c2dd0..68f84616e696 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml index c7d6e1c18dfe..f70a01669734 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml @@ -40,8 +40,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 32 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml index 2fe0b91a9c7d..dce80cbd3c11 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml @@ -36,12 +36,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 16 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml index 715893038b1f..892ab01d13bf 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml @@ -40,8 +40,6 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 48 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 9e738304bb68..1463bc140873 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -36,12 +36,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 32 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index 06f5edadd981..0cbb533b8876 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -41,7 +41,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml index 2fbdb9ac8eed..d1e7de8d11f6 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -41,7 +41,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 99440337a369..a4f0a8a690d4 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,16 +35,22 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 686fb268738a..b626ef0af46c 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,16 +35,22 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml index de316f58a87e..8b8814b0c968 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml @@ -35,16 +35,22 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 1d6d0bdfcae8..ef5b0242b386 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,16 +35,22 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 2cdb9a868988..f0c728ac2eec 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 9a3f636173c3..75b20315942c 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 50f19df4ec25..ad0501ef145c 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 4dec347902ec..39d7e2edb5d4 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 780b4d71e1a0..c26f1c2c5f20 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 6f3c36cda444..3bb488a84f18 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 802e1777d732..42234d195b27 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 4add7253487d..66b3869debda 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 669718d13894..c437058dc4b7 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index bd6d75d283a1..5369f95e2555 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 73c0910392bb..b701c83eac69 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 33150a7ba51b..2e9e45b7b915 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 0eab593e0908..785583ae10c6 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index fea3b1a1cfc5..31c2d8923c4b 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 3b5498cb847d..3e9994e21151 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 609c0c43d6d2..80bd1609ab66 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index fef69dbf9866..09ed51589940 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -44,8 +44,6 @@ environment: server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 5d4325064c64..b152141dd661 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 16d743d373d8..01726b0b4c3e 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index ab0c5dd4949a..872efcf4f66f 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml index 5c3217039fcb..c188a43ca143 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index 2b50cd55d8a7..5dfd6bcc60b9 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 640d363ba39c..474f526110bd 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml index 06abcbdbce8d..35413b5ad7b5 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml index 40c2b2f3b6d7..20303670ffba 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index a8ef83c55353..b9e3f0e89ba5 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml index 31a59ef73017..e170b4e32595 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml index effc2e77af0e..4fde4a0dcd0a 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX.yaml @@ -41,9 +41,15 @@ profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 90f3bcbf737b..fb53d40386c2 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -35,15 +35,21 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false accuracy: enable_accuracy_test: false - model: local-completions - tasks: gsm8k - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml index 88c5885ca740..e077a304f857 100644 --- a/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 16 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml index 59c7438ce4a2..885e3918f290 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml index 1bcc8a7e6d17..320fab5661eb 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 918fd1b8f227..5d972ec794e0 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -35,12 +35,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: enable_layerwise_nvtx_marker: true diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml index b97d19707710..28b3d8f13c11 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml @@ -36,12 +36,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 16 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index a825adb88bae..fca868eceb88 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -36,12 +36,10 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false worker_config: gen: tensor_parallel_size: 32 From 46bbdf540e42904f8912d14b3d47c638238da41c Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Sun, 31 May 2026 03:11:12 +0000 Subject: [PATCH 146/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/models/contrib/grok/poetry.lock | 8 ++++---- security_scanning/examples/serve/poetry.lock | 6 +++--- security_scanning/metadata.json | 4 ++-- security_scanning/poetry.lock | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/security_scanning/examples/models/contrib/grok/poetry.lock b/security_scanning/examples/models/contrib/grok/poetry.lock index bd28544ad248..41a75eaabd28 100644 --- a/security_scanning/examples/models/contrib/grok/poetry.lock +++ b/security_scanning/examples/models/contrib/grok/poetry.lock @@ -1791,15 +1791,15 @@ files = [ [[package]] name = "nvidia-cudnn-cu12" -version = "9.22.0.52" +version = "9.23.0.39" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" groups = ["main"] files = [ - {file = "nvidia_cudnn_cu12-9.22.0.52-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:cd9011812498376f866b9826331cc965ac922eb2df8549c9f2989f10255e5001"}, - {file = "nvidia_cudnn_cu12-9.22.0.52-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:391b9a7ee6386daaca7f8dca41e83c2c99f760c9581a0400755e87b4287b8847"}, - {file = "nvidia_cudnn_cu12-9.22.0.52-py3-none-win_amd64.whl", hash = "sha256:5d10117314c861245992dbcf8a6f8ae1f54852137a7c9f80cc9de9fa596f7d62"}, + {file = "nvidia_cudnn_cu12-9.23.0.39-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:3da81d70ef4db38952a7f37403434cf52dddbc1b8bee428016fa0ca9f2ff2697"}, + {file = "nvidia_cudnn_cu12-9.23.0.39-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:89d53e2a2b0614278afbeda67ac89594bdd74f9f283f22f2d34409d55859846f"}, + {file = "nvidia_cudnn_cu12-9.23.0.39-py3-none-win_amd64.whl", hash = "sha256:357e5d59a1b79d27eef754aa79b3d9e7adf11baf86dc928dc114df0033c2c912"}, ] [package.dependencies] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index a545112f8821..9f94c9bdfe07 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -4904,14 +4904,14 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "starlette" -version = "1.2.0" +version = "1.2.1" description = "The little ASGI library that shines." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7"}, - {file = "starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553"}, + {file = "starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89"}, + {file = "starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6"}, ] [package.dependencies] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 0d258b14a124..acbce6f5b0ee 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "74d7c3acbb0cf1670dc3284970a5bf3a5b468187", - "timestamp": "2026-05-30T02:47:10Z" + "commit_hash": "26c099f52ddac75a00dcaa6f172eff334f14be5f", + "timestamp": "2026-05-31T02:46:53Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index c3081a6ca191..3d8379d82af8 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -6347,14 +6347,14 @@ uvicorn = ["uvicorn (>=0.34.0)"] [[package]] name = "starlette" -version = "1.2.0" +version = "1.2.1" description = "The little ASGI library that shines." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7"}, - {file = "starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553"}, + {file = "starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89"}, + {file = "starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6"}, ] [package.dependencies] From 0fb9a36d3dc89903149ee3bc6f56484a4c1cf397 Mon Sep 17 00:00:00 2001 From: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Date: Sun, 31 May 2026 03:00:39 -0700 Subject: [PATCH 147/308] [https://nvbugs/6165866][infra] Waive 1 failed cases for main in pre-merge 40081 - Fix prefix (#14756) Signed-off-by: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 1ffe62ada48a..da27c539bd09 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -213,8 +213,8 @@ full:B200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) -full:B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:B300/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) +full:DGX_B200/unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k "TRTLLM" SKIP (https://nvbugs/6165866) full:GB200-OCI/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm] SKIP (https://nvbugs/6185150) full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1] SKIP (https://nvbugs/4731514) full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) From 54259ed5d2322cd6086fbd878bc0ab0f3edbf5f0 Mon Sep 17 00:00:00 2001 From: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> Date: Sun, 31 May 2026 16:07:00 -0700 Subject: [PATCH 148/308] [https://nvbugs/6196391][fix] Carryover disagg TTFT improvements (#14719) Signed-off-by: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> --- .../batch_manager/kvCacheManager.cpp | 5 +- tensorrt_llm/serve/openai_server.py | 62 +++++-- tests/integration/defs/test_e2e.py | 9 + .../test_lists/qa/llm_function_core.txt | 1 + .../test_lists/test-db/l0_h100.yml | 1 + .../llmapi/apps/_test_openai_chat_harmony.py | 49 ++++++ .../_test_openai_chat_harmony_perf_metrics.py | 162 ++++++++++++++++++ 7 files changed, 270 insertions(+), 19 deletions(-) create mode 100644 tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp index 03a4908e7c27..c6a6f392129d 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp @@ -2072,10 +2072,11 @@ std::shared_ptr WindowBlockManager::findBlocksInReuseTreeByBlockKe = chopVectorIntoBlocks(blockKey.uniqueTokens, blockKey.uniqueTokens.size(), mTokensPerBlock, true); std::vector blockKeys; + blockKeys.reserve(blockedUniqueTokens.size()); for (auto const& blockedUniqueTokensList : blockedUniqueTokens) { - blockKeys.push_back(blockKey); - blockKeys.back().uniqueTokens = blockedUniqueTokensList; + blockKeys.emplace_back(blockKey.usesExtraIds, blockKey.loraTaskId, blockedUniqueTokensList, blockKey.extraKeys, + blockKey.cacheSaltID); } return searchReuseTree(blockKeys); } diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index c53abe885713..55e624d7bf8a 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -1591,16 +1591,34 @@ async def chat_harmony(self, request: ChatCompletionRequest, async def create_streaming_generator(promise: RequestOutput, postproc_params: PostprocParams): - async for res in promise: + try: if not self.postproc_worker_enabled: post_processor, args = postproc_params.post_processor, postproc_params.postproc_args - pp_results = post_processor(res, args) - else: - pp_results = res.outputs[0]._postprocess_result + # Stamp first-token time on the first response, then append a + # /perf_metrics entry after [DONE]. The deque is only + # populated inside _extract_metrics. + first_response = await anext(promise) + raw_request.state.server_first_token_time = ( + get_steady_clock_now_in_seconds()) + pp_results = (first_response.outputs[0]._postprocess_result if + self.postproc_worker_enabled else post_processor( + first_response, args)) for pp_res in pp_results: yield pp_res - - yield "data: [DONE]\n\n" + res = first_response + async for res in promise: + pp_results = (res.outputs[0]._postprocess_result + if self.postproc_worker_enabled else + post_processor(res, args)) + for pp_res in pp_results: + yield pp_res + yield "data: [DONE]\n\n" + await self._extract_metrics(res, raw_request) + except asyncio.CancelledError: + raise + except Exception: + logger.error(traceback.format_exc()) + raise try: # Initialize HarmonyAdapter @@ -1618,17 +1636,23 @@ async def create_streaming_generator(promise: RequestOutput, # Get tool_choice from request tool_choice = getattr(request, 'tool_choice', None) - try: - harmony_tokens = self.harmony_adapter.openai_to_harmony_tokens( - request.messages, - tools_dict, - reasoning_effort=reasoning_effort, - tool_choice=tool_choice) - except Exception as e: - logger.error(f"messages_dict: {request.messages}") - logger.error(f"tools_dict: {tools_dict}") - logger.error(f"request: {request}") - raise e + # Reuse pre-tokenized harmony tokens when forwarded by an upstream + # context worker (disaggregated serving). Otherwise, run the + # Harmony adapter on the request messages. + if request.prompt_token_ids is not None: + harmony_tokens = request.prompt_token_ids + else: + try: + harmony_tokens = self.harmony_adapter.openai_to_harmony_tokens( + request.messages, + tools_dict, + reasoning_effort=reasoning_effort, + tool_choice=tool_choice) + except Exception: + logger.error(f"messages_dict: {request.messages}") + logger.error(f"tools_dict: {tools_dict}") + logger.error(f"request: {request}") + raise # Get harmony stop tokens harmony_stop_tokens = self.harmony_adapter.get_stop_tokens() @@ -1645,6 +1669,10 @@ async def create_streaming_generator(promise: RequestOutput, "thinking_token_budget is not supported by the Harmony " "GPT-OSS serving path; use reasoning_effort instead") sampling_params.detokenize = False # Harmony adapter handles detokenization + # Enable per-request perf metrics when the env var is set. + # Otherwise the /perf_metrics deque stays empty on this path. + if len(os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH", "")) > 0: + sampling_params.return_perf_metrics = True disaggregated_params = to_llm_disaggregated_params( request.disaggregated_params) trace_headers = (None if raw_request is None else diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 947c191394ba..fcf861543aa4 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1141,6 +1141,15 @@ def test_openai_chat_harmony(llm_root, llm_venv): str(test_root / "_test_openai_chat_harmony.py")]) +@skip_pre_hopper +def test_openai_chat_harmony_perf_metrics(llm_root, llm_venv): + test_root = unittest_path() / "llmapi" / "apps" + llm_venv.run_cmd([ + "-m", "pytest", + str(test_root / "_test_openai_chat_harmony_perf_metrics.py") + ]) + + def test_openai_responses(llm_root, llm_venv): test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd( diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 88ce413981c6..63d144aeac69 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -914,6 +914,7 @@ test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/Qwen3-30B-A3B-Qwen3/Qwen3 test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf-Qwen3/qwen3-235B-eagle3] test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-Qwen3/qwen3-235B-eagle3] test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] +test_e2e.py::test_openai_chat_harmony_perf_metrics test_e2e.py::test_openai_kv_cache_contamination test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 06b537f64f00..acf3f8e19f1c 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -166,6 +166,7 @@ l0_h100: - test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate] # negative test - test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B] - test_e2e.py::test_openai_chat_harmony + - test_e2e.py::test_openai_chat_harmony_perf_metrics - test_e2e.py::test_openai_responses - test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct] - test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B] diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_harmony.py b/tests/unittest/llmapi/apps/_test_openai_chat_harmony.py index e247cb21d9aa..317f23f39f84 100644 --- a/tests/unittest/llmapi/apps/_test_openai_chat_harmony.py +++ b/tests/unittest/llmapi/apps/_test_openai_chat_harmony.py @@ -256,3 +256,52 @@ async def test_streaming_tool_call(client: openai.AsyncOpenAI, model: str): assert reasoning args = json.loads(tool_args) get_current_weather(**args) + + +@pytest.mark.asyncio(loop_scope="module") +async def test_prompt_token_ids_skips_retokenization(client: openai.AsyncOpenAI, + model: str): + """Guard the disagg-style optimization that reuses pre-tokenized Harmony + tokens forwarded from the context worker via `request.prompt_token_ids`. + + Strategy: locally tokenize a "France" prompt with the same HarmonyAdapter + the server would use, then submit a chat request whose `messages` say + something entirely different ("a joke about cats") but whose `prompt_token_ids` + encode the France prompt. If the server skips re-tokenization, the response + should reflect the token-id prompt (mention Paris) rather than the messages. + If a future refactor breaks the `if request.prompt_token_ids is not None` + branch, the response will follow `messages` instead and this assertion fails. + """ + from tensorrt_llm.serve.harmony_adapter import get_harmony_adapter + + france_messages = [{ + "role": + "user", + "content": + "What is the capital of France? Reply with just the city name." + }] + decoy_messages = [{ + "role": "user", + "content": "Tell me a long joke about cats." + }] + + adapter = get_harmony_adapter() + france_tokens = adapter.openai_to_harmony_tokens(france_messages, None) + assert isinstance(france_tokens, list) and len(france_tokens) > 0 + + response = await client.chat.completions.create( + model=model, + messages=decoy_messages, + extra_body={ + "prompt_token_ids": france_tokens, + "top_k": 1, + }, + ) + content = response.choices[0].message.content or "" + reasoning = response.choices[0].message.reasoning or "" + combined = f"{content}\n{reasoning}".lower() + assert "paris" in combined, ( + "Expected the response to follow the pre-tokenized France prompt " + "(mention 'Paris'), suggesting the server bypassed the Harmony " + "adapter via `request.prompt_token_ids`. Got:\n" + f"content={content!r}\nreasoning={reasoning!r}") diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py b/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py new file mode 100644 index 000000000000..85cd2a073b1a --- /dev/null +++ b/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for /perf_metrics population on the Harmony (GPT-OSS) chat path. + +Asserts two behaviors of the Harmony chat endpoint: + * The streaming generator stamps `server_first_token_time` and calls + `_extract_metrics(...)` after `data: [DONE]`, so the `/perf_metrics` + deque is populated for streaming chat requests. + * When `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` is set, the per-request + `sampling_params.return_perf_metrics` flag is enabled so the engine + emits metrics for each request. Without it, the deque stays empty + even with `return_perf_metrics: True` at the LLM-args level. +""" + +import json +import os +import tempfile +from urllib.request import urlopen + +import openai +import pytest +import yaml +from utils.llm_data import llm_datasets_root + +from ..test_llm import get_model_path +from .openai_server import RemoteOpenAIServer + +pytestmark = pytest.mark.threadleak(enabled=False) +os.environ["TIKTOKEN_RS_CACHE_DIR"] = os.path.join(llm_datasets_root(), "tiktoken_vocab") +os.environ["TIKTOKEN_ENCODINGS_BASE"] = os.path.join(llm_datasets_root(), "tiktoken_vocab") + + +@pytest.fixture(scope="module", ids=["GPT-OSS-20B"]) +def model(): + return "gpt_oss/gpt-oss-20b/" + + +@pytest.fixture(scope="module", params=[0, 2], ids=["disable_processpool", "enable_processpool"]) +def num_postprocess_workers(request): + return request.param + + +@pytest.fixture(scope="module") +def kv_cache_time_output_dir(tmp_path_factory): + return str(tmp_path_factory.mktemp("kv_cache_time_output")) + + +@pytest.fixture(scope="module") +def extra_llm_api_options_file(): + fd, path = tempfile.mkstemp(suffix=".yaml", prefix="extra_llm_api_options_") + os.close(fd) + try: + with open(path, "w") as f: + yaml.dump( + { + "return_perf_metrics": True, + "perf_metrics_max_requests": 16, + }, + f, + ) + yield path + finally: + if os.path.exists(path): + os.remove(path) + + +@pytest.fixture(scope="module") +def server( + model: str, + num_postprocess_workers: int, + kv_cache_time_output_dir: str, + extra_llm_api_options_file: str, +): + model_path = get_model_path(model) + args = [ + "--num_postprocess_workers", + f"{num_postprocess_workers}", + "--extra_llm_api_options", + extra_llm_api_options_file, + ] + env = os.environ.copy() + # Non-empty value flips `sampling_params.return_perf_metrics` per request + # inside the Harmony chat path; the value itself is also consumed by the + # C++ KV cache layer for CSV dumps. + env["TRTLLM_KVCACHE_TIME_OUTPUT_PATH"] = kv_cache_time_output_dir + with RemoteOpenAIServer(model_path, args, env=env) as remote_server: + yield remote_server + + +@pytest.fixture(scope="module") +def async_client(server: RemoteOpenAIServer): + return server.get_async_client() + + +def _drain_perf_metrics(server: RemoteOpenAIServer): + response = urlopen(f"{server.url_root}/perf_metrics") + assert response.status == 200 + return json.loads(response.read()) + + +def _assert_perf_metrics_entry_well_formed(entry: dict): + assert "request_id" in entry + assert "perf_metrics" in entry + pm = entry["perf_metrics"] + assert "first_iter" in pm and "last_iter" in pm + assert pm["first_iter"] <= pm["last_iter"] + + tm = pm["timing_metrics"] + for key in ("arrival_time", "first_scheduled_time", "first_token_time", "last_token_time"): + assert key in tm, f"missing timing_metrics.{key}" + assert tm["arrival_time"] <= tm["first_scheduled_time"] + assert tm["first_scheduled_time"] <= tm["first_token_time"] + assert tm["first_token_time"] <= tm["last_token_time"] + + +@pytest.mark.asyncio(loop_scope="module") +async def test_non_streaming_perf_metrics( + async_client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model: str +): + # Drain anything from prior tests in this module session. + _drain_perf_metrics(server) + response = await async_client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "Reply with exactly the single word: PONG."}], + extra_body={"top_k": 1}, + ) + assert response.choices[0].message.content is not None + + entries = _drain_perf_metrics(server) + assert len(entries) == 1, ( + "Expected exactly one /perf_metrics entry after a single non-streaming " + f"harmony chat completion, got {len(entries)}: {entries}" + ) + _assert_perf_metrics_entry_well_formed(entries[0]) + + +@pytest.mark.asyncio(loop_scope="module") +async def test_streaming_perf_metrics( + async_client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model: str +): + # Drain anything from prior tests in this module session. + _drain_perf_metrics(server) + stream = await async_client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "Explain transformers in one sentence."}], + stream=True, + extra_body={"top_k": 1}, + ) + saw_done = False + async for _chunk in stream: + # We don't need to inspect content; just consume the stream so the + # server-side generator runs through `[DONE]` and `_extract_metrics`. + saw_done = True + assert saw_done, "Streaming chat returned no chunks" + + entries = _drain_perf_metrics(server) + assert len(entries) == 1, ( + "Expected exactly one /perf_metrics entry after a single streaming " + f"harmony chat completion, got {len(entries)}: {entries}. " + "This usually means _extract_metrics did not run after [DONE]." + ) + _assert_perf_metrics_entry_well_formed(entries[0]) From a422420db98c2d71a7011c9f12d3a79c194161a4 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 Date: Mon, 1 Jun 2026 09:11:18 +0800 Subject: [PATCH 149/308] [https://nvbugs/6244695][fix] Revert Pass IPC HMAC key through file descriptor (#14782) Signed-off-by: Chenfei Zhang --- tensorrt_llm/commands/serve.py | 26 ++------- tensorrt_llm/executor/utils.py | 56 ++----------------- tensorrt_llm/llmapi/trtllm-llmapi-launch | 19 ++----- tests/unittest/executor/test_launcher_envs.py | 54 ------------------ 4 files changed, 13 insertions(+), 142 deletions(-) delete mode 100644 tests/unittest/executor/test_launcher_envs.py diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index ef99deacb0c0..5608872da2dd 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -24,8 +24,7 @@ from tensorrt_llm._tensorrt_engine import LLM from tensorrt_llm._utils import mpi_rank from tensorrt_llm.commands.utils import get_is_diffusion_model -from tensorrt_llm.executor.utils import (LlmLauncherEnvs, - set_spawn_proxy_process_ipc_hmac_key) +from tensorrt_llm.executor.utils import LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import (BuildConfig, CapacitySchedulerPolicy, DynamicBatchConfig, KvCacheConfig, @@ -1404,13 +1403,11 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, # This mimics the behavior of trtllm-llmapi-launch # TODO: Make the port allocation atomic free_ipc_addr = find_free_ipc_addr() - ipc_hmac_key = secrets.token_hex(32) - set_spawn_proxy_process_ipc_hmac_key(ipc_hmac_key) - os.environ.pop( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, None) os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS] = "1" os.environ[ LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR.value] = free_ipc_addr + os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY. + value] = secrets.token_hex(32) os.environ[DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT. value] = "1" os.environ[DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX] = str(instance_idx) @@ -1426,6 +1423,7 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS in non_mpi_env assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR in non_mpi_env + assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT in non_mpi_env @@ -1448,24 +1446,13 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, signal.signal(signal.SIGTERM, _signal_handler_cleanup_child) signal.signal(signal.SIGINT, _signal_handler_cleanup_child) - read_fd = -1 - write_fd = -1 try: - read_fd, write_fd = os.pipe() - os.write(write_fd, ipc_hmac_key.encode("ascii")) - os.close(write_fd) - write_fd = -1 - non_mpi_env[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD. - value] = str(read_fd) _child_p_global = subprocess.Popen( command, env=non_mpi_env, stdout=sys.stdout, # Redirect to parent's stdout stderr=sys.stderr, # Redirect to parent's stderr - pass_fds=(read_fd, ), start_new_session=True) - os.close(read_fd) - read_fd = -1 logger.info( f"Parent process (PID {os.getpid()}) launched child process (PID {_child_p_global.pid})." @@ -1479,11 +1466,6 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, launch_remote_mpi_session_server(sub_comm) finally: - if write_fd != -1: - os.close(write_fd) - if read_fd != -1: - os.close(read_fd) - # Restore original signal handlers signal.signal(signal.SIGTERM, original_sigterm_handler) signal.signal(signal.SIGINT, original_sigint_handler) diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 421d943916fd..12eaa9afefa9 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -23,49 +23,12 @@ class LlmLauncherEnvs(StrEnum): # Spawn a process for the LLM-API Proxy TLLM_SPAWN_PROXY_PROCESS = "TLLM_SPAWN_PROXY_PROCESS" TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR = "TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR" - TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD = ( - "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD") + TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY" # Whether to use periodical responses handler in await_responses TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT = "TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT" -_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY: bytes | None = None - - -def _normalize_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> bytes: - if isinstance(key, bytes): - if len(key) == 32: - return key - key = key.decode("ascii") - - key_bytes = bytes.fromhex(key) - if len(key_bytes) != 32: - raise ValueError("IPC HMAC key must be 32 bytes.") - return key_bytes - - -def set_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> None: - global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( - _normalize_spawn_proxy_process_ipc_hmac_key(key)) - - -def _read_spawn_proxy_process_ipc_hmac_key_fd(fd_value: str) -> bytes: - fd = int(fd_value) - chunks: list[bytes] = [] - try: - while True: - chunk = os.read(fd, 4096) - if not chunk: - break - chunks.append(chunk) - finally: - os.close(fd) - - return _normalize_spawn_proxy_process_ipc_hmac_key(b"".join(chunks)) - - def get_spawn_proxy_process_ipc_addr_env() -> str | None: ''' Get the IPC address for the spawn proxy process dynamically. ''' return os.getenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR) @@ -73,20 +36,11 @@ def get_spawn_proxy_process_ipc_addr_env() -> str | None: def get_spawn_proxy_process_ipc_hmac_key_env() -> bytes: ''' Get the HMAC key for the spawn proxy process dynamically. ''' - global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - if _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY is not None: - return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - - key_fd = os.environ.pop( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD, None) - if key_fd is not None: - _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( - _read_spawn_proxy_process_ipc_hmac_key_fd(key_fd)) - return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - - raise AssertionError( - f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD} is not set. " + key = os.getenv("TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY") + assert key is not None, ( + f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY} is not set. " "HMAC encryption is required for IPC communication.") + return bytes.fromhex(key) def get_spawn_proxy_process_env() -> bool: diff --git a/tensorrt_llm/llmapi/trtllm-llmapi-launch b/tensorrt_llm/llmapi/trtllm-llmapi-launch index d35b60046a26..c11a3bbeb171 100755 --- a/tensorrt_llm/llmapi/trtllm-llmapi-launch +++ b/tensorrt_llm/llmapi/trtllm-llmapi-launch @@ -40,17 +40,7 @@ function maybe_export_free_tcp_addr_for_spawn_proxy_process { export tllm_mpi_size=$(mpi_world_size) log_stderr "tllm_mpi_size: $tllm_mpi_size" -unset TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD -ipc_hmac_key=$(openssl rand -hex 32) - -function run_with_ipc_hmac_key { - local fd - exec {fd}<<<"$ipc_hmac_key" - TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD="$fd" "$@" - local status=$? - exec {fd}<&- - return $status -} +export TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY=$(openssl rand -hex 32) if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then @@ -84,13 +74,12 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then set +e # Execute the task with cleaned environment - run_with_ipc_hmac_key "${task_with_command[@]}" + "${task_with_command[@]}" task_exit_code=$? log_stderr "Rank${mpi_rank} Task exit code: $task_exit_code" # Stop the MPI Comm server - run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node \ - --action stop + python3 -m tensorrt_llm.llmapi.mgmn_leader_node --action stop mpi_exit_code=$? log_stderr "Rank${mpi_rank} MPI Comm server exit code: $mpi_exit_code" @@ -111,7 +100,7 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then log_stderr "Rank${mpi_rank} run mgmn leader node with mpi_world_size: $(mpi_world_size) ..." log_stderr "Rank0 host: $HOSTNAME" - run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node + python3 -m tensorrt_llm.llmapi.mgmn_leader_node mgmn_leader_node_exit_code=$? log_stderr "Rank${mpi_rank} MGMN leader node exit code: $mgmn_leader_node_exit_code" diff --git a/tests/unittest/executor/test_launcher_envs.py b/tests/unittest/executor/test_launcher_envs.py deleted file mode 100644 index 0d1f96ef1620..000000000000 --- a/tests/unittest/executor/test_launcher_envs.py +++ /dev/null @@ -1,54 +0,0 @@ -import os - -import pytest - -from tensorrt_llm.executor import utils as executor_utils -from tensorrt_llm.executor.utils import LlmLauncherEnvs - - -def _reset_ipc_hmac_key_env(monkeypatch): - monkeypatch.setattr(executor_utils, "_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY", None) - monkeypatch.delenv( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, raising=False - ) - - -def _write_key_fd(key_hex: str) -> int: - read_fd, write_fd = os.pipe() - os.write(write_fd, key_hex.encode("ascii")) - os.close(write_fd) - return read_fd - - -def test_get_spawn_proxy_process_ipc_hmac_key_from_fd(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - key_hex = "01" * 32 - read_fd = _write_key_fd(key_hex) - monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) - - key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() - - assert key == bytes.fromhex(key_hex) - assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ - with pytest.raises(OSError): - os.fstat(read_fd) - - -def test_get_spawn_proxy_process_ipc_hmac_key_caches_fd_key(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - key_hex = "02" * 32 - read_fd = _write_key_fd(key_hex) - monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) - - key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() - - assert key == bytes.fromhex(key_hex) - assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ - assert executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() == key - - -def test_get_spawn_proxy_process_ipc_hmac_key_missing(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - - with pytest.raises(AssertionError, match="HMAC encryption is required"): - executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() From 5f5b77239ad5ccad19a944e40aee9ef7d6016101 Mon Sep 17 00:00:00 2001 From: JennyLiu <141791095+JennyLiu-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:42:53 +0800 Subject: [PATCH 150/308] [None][test] Update datasets path (#14671) Signed-off-by: Jenny Liu Co-authored-by: Jenny Liu --- tests/integration/defs/perf/test_perf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 9550f1f502a0..71c5b732e0fe 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -1318,7 +1318,8 @@ def generate_trtllm_custom_dataset(self, dst_dataset_path: str, model_dir, trust_remote_code=self._config.model_name in TRUST_REMOTE_CODE_MODELS) - dataset = load_dataset("cnn_dailymail", + dataset = load_dataset(os.path.join(llm_models_root(), "datasets", + "cnn_dailymail"), "3.0.0", split="validation", streaming=True, From 9ed1669593fda9c00d90a054bbd586f3f0f42844 Mon Sep 17 00:00:00 2001 From: Emma Qiao Date: Mon, 1 Jun 2026 10:48:51 +0800 Subject: [PATCH 151/308] [None][infra] Update new .test_durations (#14661) Signed-off-by: EmmaQiaoCh --- jenkins/L0_Test.groovy | 44 +- tests/integration/defs/.test_durations | 1592 +++++++++++++----------- 2 files changed, 914 insertions(+), 722 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 26ca41050ff8..a8572fe55a77 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3900,15 +3900,18 @@ def launchTestJobs(pipeline, testFilter) fullSet = parallelJobs.keySet() x86SlurmTestConfigs = [ - "DGX_H100-PyTorch-1": ["auto:dgx-h100-x1", "l0_h100", 1, 4], - "DGX_H100-PyTorch-2": ["auto:dgx-h100-x1", "l0_h100", 2, 4], - "DGX_H100-PyTorch-3": ["auto:dgx-h100-x1", "l0_h100", 3, 4], - "DGX_H100-PyTorch-4": ["auto:dgx-h100-x1", "l0_h100", 4, 4], + "DGX_H100-PyTorch-1": ["auto:dgx-h100-x1", "l0_h100", 1, 6], + "DGX_H100-PyTorch-2": ["auto:dgx-h100-x1", "l0_h100", 2, 6], + "DGX_H100-PyTorch-3": ["auto:dgx-h100-x1", "l0_h100", 3, 6], + "DGX_H100-PyTorch-4": ["auto:dgx-h100-x1", "l0_h100", 4, 6], + "DGX_H100-PyTorch-5": ["auto:dgx-h100-x1", "l0_h100", 5, 6], + "DGX_H100-PyTorch-6": ["auto:dgx-h100-x1", "l0_h100", 6, 6], "DGX_H100-PyTorch-Post-Merge-1": ["auto:dgx-h100-x1", "l0_h100", 1, 2], "DGX_H100-PyTorch-Post-Merge-2": ["auto:dgx-h100-x1", "l0_h100", 2, 2], "DGX_H100-2_GPUs-PyTorch-Others-1": ["auto:dgx-h100-x2", "l0_dgx_h100", 1, 2, 2], "DGX_H100-2_GPUs-PyTorch-Others-2": ["auto:dgx-h100-x2", "l0_dgx_h100", 2, 2, 2], - "DGX_H100-2_GPUs-PyTorch-GptOss-1": ["auto:dgx-h100-x2", "l0_dgx_h100", 1, 1, 2], + "DGX_H100-2_GPUs-PyTorch-GptOss-1": ["auto:dgx-h100-x2", "l0_dgx_h100", 1, 2, 2], + "DGX_H100-2_GPUs-PyTorch-GptOss-2": ["auto:dgx-h100-x2", "l0_dgx_h100", 2, 2, 2], "DGX_H100-2_GPUs-PyTorch-Ray-1": ["auto:dgx-h100-x2", "l0_dgx_h100", 1, 1, 2], "DGX_H100-4_GPUs-PyTorch-DeepSeek-1": ["auto:dgx-h100-x4", "l0_dgx_h100", 1, 2, 4], "DGX_H100-4_GPUs-PyTorch-DeepSeek-2": ["auto:dgx-h100-x4", "l0_dgx_h100", 2, 2, 4], @@ -3918,11 +3921,13 @@ def launchTestJobs(pipeline, testFilter) "DGX_H100-4_GPUs-PyTorch-Ray-1": ["auto:dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], "DGX_H100-4_GPUs-AutoDeploy-1": ["auto:dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], "DGX_H100-4_GPUs-AutoDeploy-Post-Merge-1": ["auto:dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], - "DGX_B200-PyTorch-1": ["auto:dgx-b200-flex", "l0_b200", 1, 5, 1, 1, true], - "DGX_B200-PyTorch-2": ["auto:dgx-b200-flex", "l0_b200", 2, 5, 1, 1, true], - "DGX_B200-PyTorch-3": ["auto:dgx-b200-flex", "l0_b200", 3, 5, 1, 1, true], - "DGX_B200-PyTorch-4": ["auto:dgx-b200-flex", "l0_b200", 4, 5, 1, 1, true], - "DGX_B200-PyTorch-5": ["auto:dgx-b200-flex", "l0_b200", 5, 5, 1, 1, true], + "DGX_B200-PyTorch-1": ["auto:dgx-b200-flex", "l0_b200", 1, 7, 1, 1, true], + "DGX_B200-PyTorch-2": ["auto:dgx-b200-flex", "l0_b200", 2, 7, 1, 1, true], + "DGX_B200-PyTorch-3": ["auto:dgx-b200-flex", "l0_b200", 3, 7, 1, 1, true], + "DGX_B200-PyTorch-4": ["auto:dgx-b200-flex", "l0_b200", 4, 7, 1, 1, true], + "DGX_B200-PyTorch-5": ["auto:dgx-b200-flex", "l0_b200", 5, 7, 1, 1, true], + "DGX_B200-PyTorch-6": ["auto:dgx-b200-flex", "l0_b200", 6, 7, 1, 1, true], + "DGX_B200-PyTorch-7": ["auto:dgx-b200-flex", "l0_b200", 7, 7, 1, 1, true], "DGX_B200-AutoDeploy-1": ["auto:dgx-b200-flex", "l0_b200", 1, 1, 1, 1, true], "DGX_B200-Triton-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200", 1, 1, 1, 1, true], "DGX_B200-PyTorch-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200", 1, 2, 1, 1, true], @@ -3932,13 +3937,17 @@ def launchTestJobs(pipeline, testFilter) "DGX_B200-4_GPUs-PyTorch-3": ["auto:dgx-b200-flex", "l0_dgx_b200", 3, 3, 4, 1, true], "DGX_B200-4_GPUs-PyTorch-Ray-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 1, 4, 1, true], "DGX_B200-4_GPUs-AutoDeploy-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 1, 4, 1, true], - "DGX_B200-4_GPUs-PyTorch-Post-Merge-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 2, 4, 1, true], - "DGX_B200-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 2, 4, 1, true], - "DGX_B200-8_GPUs-PyTorch-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 2, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 2, 8, 1, true], + "DGX_B200-4_GPUs-PyTorch-Post-Merge-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 4, 4, 1, true], + "DGX_B200-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 4, 4, 1, true], + "DGX_B200-4_GPUs-PyTorch-Post-Merge-3": ["auto:dgx-b200-flex", "l0_dgx_b200", 3, 4, 4, 1, true], + "DGX_B200-4_GPUs-PyTorch-Post-Merge-4": ["auto:dgx-b200-flex", "l0_dgx_b200", 4, 4, 4, 1, true], + "DGX_B200-8_GPUs-PyTorch-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 3, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-2": ["auto:dgx-b200-flex", "l0_dgx_b200", 2, 3, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-3": ["auto:dgx-b200-flex", "l0_dgx_b200", 3, 3, 8, 1, true], "DGX_B200-8_GPUs-AutoDeploy-Post-Merge-1": ["auto:dgx-b200-flex", "l0_dgx_b200", 1, 1, 8, 1, true], "DGX_B200-4_GPUs-Verl-Post-Merge-1": ["auto:dgx-b200-flex", "l0_verl", 1, 1, 4, 1, true], - "B300-PyTorch-1": ["auto:dgx-b300-flex", "l0_b300", 1, 1, 1, 1, true], + "B300-PyTorch-1": ["auto:dgx-b300-flex", "l0_b300", 1, 2, 1, 1, true], + "B300-PyTorch-2": ["auto:dgx-b300-flex", "l0_b300", 2, 2, 1, 1, true], "DGX_B300-4_GPUs-PyTorch-1": ["auto:dgx-b300-flex", "l0_dgx_b300", 1, 1, 4, 1, true], "DGX_B300-4_GPUs-PyTorch-Post-Merge-1": ["auto:dgx-b300-flex", "l0_dgx_b300", 1, 2, 4, 1, true], "DGX_B300-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b300-flex", "l0_dgx_b300", 2, 2, 4, 1, true], @@ -4002,8 +4011,9 @@ def launchTestJobs(pipeline, testFilter) "GB200-4_GPUs-PyTorch-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus", 1, 1, 4], "GB10-PyTorch-Post-Merge-1": ["gb10x-single", "l0_gb10", 1, 1], "GB300-PyTorch-1": ["auto:gb300-x4", "l0_gb300", 1, 1], - "GB300-4_GPUs-PyTorch-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus", 1, 2, 4], - "GB300-4_GPUs-PyTorch-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus", 2, 2, 4], + "GB300-4_GPUs-PyTorch-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus", 1, 3, 4], + "GB300-4_GPUs-PyTorch-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus", 2, 3, 4], + "GB300-4_GPUs-PyTorch-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus", 3, 3, 4], // PerfSanity pre-merge tests "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 2, 4], "GB200-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 2, 4], diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 236449183fb7..9b2e9851b370 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -1,707 +1,889 @@ { - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 229.88366167014465, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 1291.2246230191085, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 226.58150382409804, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 230.04569808510132, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 153.0148511910811, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 151.06352188810706, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 157.06875282898545, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 153.20611042692326, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 71.2399792142678, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False]": 286.7775873204227537, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True]": 286.6778334858827293, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 781.7928658421151, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 270.3750694899354, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 195.4896494857967, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 205.93911361903884, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 188.56422709790058, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 199.29050170327537, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 127.93316596397199, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 129.7617962991353, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 127.73340241820551, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 129.19083517300896, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 329.7597716320306, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 232.4293970640283, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 198.69031671597622, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 199.68801344232634, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 161.8193401999306, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 107.20401050220244, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 163.33450849773362, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 137.72523731505498, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 209.07444706000388, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 149.8242256443482, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 286.3013918437063694, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.44922504294664, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 173.89489603298716, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 172.84839022206143, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 128.10282056825235, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 121.90447079204023, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 117.0786016730126, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 64428.639228201006, - "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 572.5455802679062, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False]": 472.62511800276116, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True]": 273.7770717362873, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 56.07656032079831, - "accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise": 361.5573864541948, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph": 95.2069768682122, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 92.18154831900029, - "accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 92.545280149206519, - "accuracy/test_llm_api.py::TestLlama3_2_1B::test_auto_dtype": 48.46169294399442, - "accuracy/test_llm_api.py::TestLlama3_2_1B::test_fp8_rowwise": 46.47006619500462, - "accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2": 226.48782553203637, - "accuracy/test_llm_api.py::TestQwen2_5_0_5BInstruct::test_fp8": 93.85909735393943, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype": 173.21916160301771, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_fp8": 176.52263558004051, - "accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only": 137.54228180646896, - "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8": 223.2249169460265, - "accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8_kvcache": 293.131719612516463, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_auto_dtype": 146.72803921002196, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_fp8": 149.11297382606426, - "accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_weight_only": 594.9357111975551, - "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_auto_dtype": 3600.001755183912, - "accuracy/test_llm_api_pytorch.py::TestBielik11BInstruct::test_fp8": 3600.0018868579646, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput]": 7291.4317992888391, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[latency]": 7291.9177003782242537, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale_chunked_prefill[throughput]": 7291.818120779618621, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency]": 7292.3961801091209054, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp]": 7292.2807201944456277, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen]": 7292.6643901705286214, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp]": 7292.5567501965034455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput]": 7292.690330147743225, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4]": 7292.468819195404649, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp8]": 7292.699600299820304, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[latency]": 7292.5356449983082712, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_chunked_prefill[throughput_tp4]": 7292.413349622860551, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_corner_case": 7292.248919965699315, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 591.2785023800097, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 107.58471493399702, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 143.84012729604729, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 295.3527018489549, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 306.84709841990843, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 220.57452515885234, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 165.08514453098178, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 202.22269394202158, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False]": 113.82226522010751, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False]": 205.7252635700861, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True]": 213.78996226208983, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=0-moe_backend=WIDEEP]": 292.7267158059985377, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 292.5756296711042523, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 326.1317654890008, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 647.6109309499152, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 184.20976317999884, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 226.01353620411828, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 506.1045090719126, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-attention_dp-cuda_graph-overlap_scheduler-torch_compile=False]": 336.02580665098503, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 413.903915906325, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 143.841789112892, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 246.64391099987552, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 202.37037238897756, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 3600.0020909640007, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 3600.441698686045129, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 3600.591623628977686, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 3600.642337311059237, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 3600.572194146050606, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=0]": 3600.003432472993, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2]": 3600.21731633093441, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0]": 3600.002661294944, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2]": 3600.0022540579666, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 89.92349556891713, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 85.24235329206567, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 252.70569713797886, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 90.21807348495349, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 175.661773331929, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003233290044590831, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=False-moe_backend=WIDEEP]": 360.9275938109494746, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=WIDEEP]": 360.0002855450729839504, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=0.75-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 360.0003064870252273977, - "accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype": 3600.0004039629711769521, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto]": 360.6586053780047223, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 360.00032637204276397824, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-fp8]": 360.0003586999955587089, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-triton-auto]": 360.6586053780047223, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-auto]": 360.0003633099840953946, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-fp8]": 360.00036422599805518985, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-cutlass-auto]": 360.0003378289984539151, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-triton-auto]": 360.9436147869564593, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-auto]": 360.0003398499684408307, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[dp2-trtllm-fp8]": 360.0002922280109487474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-cutlass-auto]": 360.0003666180418804288, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-triton-auto]": 360.9300670439261012, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-auto]": 360.0002812399761751294, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[ep2-trtllm-fp8]": 360.0008064290159381926, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-cutlass-auto]": 360.0003697940264828503, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-triton-auto]": 360.8670774899655953, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-auto]": 360.00040231598541140556, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_2gpus[tp2-trtllm-fp8]": 360.0003254589391872287, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-cutlass-auto]": 745.8583740849863, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 745.9345730679342523, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-cutlass-auto]": 658.1757711600512, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 745.9436021829606034, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto]": 676.3980704760179, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 745.0292645250447094, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto]": 745.8583740849863, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 745.9345730679342523, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto]": 745.0004936959594488144, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8]": 745.00031642295653000474, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutlass-auto]": 658.1757711600512, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 745.9436021829606034, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 745.0004371170070953667, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-fp8]": 745.0004142870311625302, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto]": 676.3980704760179, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-triton-auto]": 745.0292645250447094, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 745.0003769229515455663, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-fp8]": 677.000331886054482311, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto]": 643.3513998010312, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto]": 764.9216735750087537, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-auto]": 764.0002969659981317818, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8]": 764.0008383550448343158, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 764.8800516680348665, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8]": 764.00035094103077426553, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 14400, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 14400, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 88.30407958402066, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 892.3242024959764, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 451.8630696780165, - "accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype": 86.96273394301534, - "accuracy/test_llm_api_pytorch.py::TestKimiK2::test_fp8_blockscale[latency]": 0.19557904999237508, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 56.31924073398113, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 115.23303954396397, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 115.0261728389305, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 98.29266697796993, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 96.56704166403506, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 98.47511212801328, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 96.76115251897136, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 307.12596721109, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 443.91388061689213, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 191.10617867391557, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 166.85348949534819, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 167.15153613401344, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 90.12104846700095, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 1112.0988524899585, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 979.2759481471148, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 237.24446990108117, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 226.39608797896653, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 174.38962662010454, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 313.69273760309443, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 409.8932851999998, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 344.8807112099603, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 103.82129427790642, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 164.91815144987777, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False]": 124.62386814301135, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True]": 121.9821372089209, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False]": 77.16783500701422, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=True]": 75.89485901995795, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False]": 77.46995012206025, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True]": 75.93042165302904, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_sampler": 61.986203632026445, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 44.978786278981715, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 46.60325947904494, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[llguidance]": 52.44408063602168, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 52.84089746000245, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False]": 36.739327706047334, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 67.41741452802671, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 36.758924948982894, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 32.73584783205297, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance]": 31.598825128981844, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar]": 31.997449714050163, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_ngram": 404.38593446696177, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_auto_dtype": 49.4644276680192, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_1B::test_fp8_prequantized": 40.83171262202086, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype": 60.00025596999330446124, - "accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_fp8_prequantized": 67.63213626696961, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False]": 3770.7388199700508, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False]": 4010.9139676889754, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False]": 2616.262340236979, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 3600.00026297004660591483, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4-cuda_graph=False]": 7200.00030252401484176517, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 7200.0002791329752653837, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 7200.00028608099091798067, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False]": 7200.0002498350222595036, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 7200.00023661594605073333, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 7200.00023339298786595464, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=FLASHINFER]": 7200.0002561529981903732, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_chunked_prefill[attn_backend=TRTLLM]": 7200.0052206520340405405, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4-cuda_graph=True]": 7200.2161163400160149, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep2-cuda_graph=True]": 7200.2132116109714843, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp4ep4-cuda_graph=True]": 7200.3660773960873485, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8-cuda_graph=True]": 7200.00025232898769900203, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep4-cuda_graph=True]": 7200.00025386200286448, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8[tp8ep8-cuda_graph=True]": 7200.00023917207727208734, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=False]": 7200.5301868109382, - "accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_chunked_prefill[tp8ep8-cuda_graph=True]": 72600.1488124400494, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4-cuda_graph=False]": 3600.0010551271309959702, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep2-cuda_graph=True]": 3600.0009890546519891359, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp4ep4-cuda_graph=True]": 3600.000870058874017559, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False]": 3600.0022709049517, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True]": 3600.0008703919593, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True]": 3600.001674739062, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp4-cuda_graph=True]": 3600.0003222679952159524, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4[tp8ep8-cuda_graph=True]": 3600.0004189839819446206, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp4_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0009446179610677063, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp4-cuda_graph=True]": 3600.0600651470012963, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8[tp8ep8-cuda_graph=True]": 3600.0020443379763, - "accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_fp8_chunked_prefill[tp4ep4-cuda_graph=True]": 3600.0016280449927, - "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype": 300.0017418859643, - "accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8": 300.001715709921, - "accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized": 48.064747432945296, - "accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype": 96.73990149900783, - "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype": 3600.0012007239857, - "accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype_tp2": 3600.001961252012, - "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_auto_dtype": 3600.0003251209855079651, - "accuracy/test_llm_api_pytorch.py::TestMistralSmall24B::test_fp8": 196.57955891895108, - "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2": 3600.0022730380297, - "accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2": 3600.7994798690197058, - "accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8": 3600.5389363930444, - "accuracy/test_llm_api_pytorch.py::TestPhi4::test_auto_dtype": 3600.0017098310054, - "accuracy/test_llm_api_pytorch.py::TestPhi4::test_fp8": 3600.0018840720295, - "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype": 3600.000989172084, - "accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype_long_rope": 3600.001809718029, - "accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype": 3600.0018334789784, - "accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype": 49.38216367200948, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency]": 7576.4847942629713, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency]": 75934.1885519769276, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass]": 7570.00041150598553940654, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm]": 7570.0005082660354673862, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp]": 7570.0003929820377379656, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_trtllm_eagle3]": 7570.000994006055407226, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 3240.3035402488895, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 3240.3568000392988324, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False]": 3240.326488903677091, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True]": 3240.3251879825256765, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]": 3240.3220480284653604, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]": 3240.12236930197104812, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]": 3240.3637070371955633, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]": 3240.5122077039559372, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]": 3240.46138638898264617, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]": 3240.4374056719825603, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]": 3240.4651110970880836, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]": 3240.4263977239606902, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.5-fp8kv=False]": 1202.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.9-fp8kv=False]": 1127.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS]": 1091.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM]": 1091.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False]": 313.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True]": 313.0, - "accuracy/test_llm_api_pytorch.py::TestQwen3_6_27B::test_fp8": 569.04, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency-torch_compile=False]": 149.19146074401215, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 104.32479889906244, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 240.30756398336961865, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency]": 240.27633493300527334, - "cpp/test_e2e.py::test_benchmarks[bart-90]": 271.95234084688127, - "cpp/test_e2e.py::test_benchmarks[gpt-80]": 1376.0404928650241, - "cpp/test_e2e.py::test_benchmarks[t5-90]": 523.07, - "cpp/test_e2e.py::test_model[-bart-90]": 391.84748707409017, - "cpp/test_e2e.py::test_model[-eagle-86]": 850.5158995762467, - "cpp/test_e2e.py::test_model[-enc_dec_language_adapter-90]": 416.06, - "cpp/test_e2e.py::test_model[-gpt-80]": 1568.98, - "cpp/test_e2e.py::test_model[-gpt_executor-80]": 1495.14, - "cpp/test_e2e.py::test_model[-gpt_tests-80]": 1206.79, - "cpp/test_e2e.py::test_model[-mamba-86]": 893.8684413917363, - "cpp/test_e2e.py::test_model[-medusa-86]": 577.0913726426661, - "cpp/test_e2e.py::test_model[-redrafter-86]": 356.56682327389717, - "cpp/test_e2e.py::test_model[-t5-90]": 170.26, - "cpp/test_e2e.py::test_model[fp8-llama-90]": 385.98, - "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 1005.24, - "cpp/test_unit_tests.py::test_unit_tests[common-80]": 38.98, - "cpp/test_unit_tests.py::test_unit_tests[common-90]": 25.06, - "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 425.16, - "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 2009.96, - "cpp/test_unit_tests.py::test_unit_tests[kernels-90]": 1333.18, - "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 2209.11, - "cpp/test_unit_tests.py::test_unit_tests[layers-90]": 1627.07, - "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 1671.42, - "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 6.76, - "cpp/test_unit_tests.py::test_unit_tests[thop-90]": 4.16, - "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 8.53, - "cpp/test_unit_tests.py::test_unit_tests[utils-90]": 5.28, - "disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0]": 54.26733888499439, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 64.01942083099857, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.32925470802002, - "disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0]": 50.57874152995646, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 105.12023228011094, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8]": 98.13158084987663, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one_mtp[DeepSeek-V3-Lite-fp8]": 104.78005758393556, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_mpi[DeepSeek-V3-Lite-fp8]": 680.20395052596, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 102.03138376423158, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 90.40784636512399, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8]": 124.17078560194932, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 102.67233599093743, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 224.28071974776685, - "disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0]": 52.78952780482359, - "disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0]": 73.48997121001594, - "disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0]": 67.3897166326642, - "disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0]": 54.22262764698826, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 98.97588296607137, - "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0]": 67.9668476767838, - "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_trt_backend[TinyLlama-1.1B-Chat-v1.0]": 82.28277984517626, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 238.76137515995651, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 78.98068026197143, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 77.51831256924197, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 99.81417108187452, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 67.32832619687542, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 48.16434509307146, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.88020430901088, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 46.302398771978915, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 38.81214914191514, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 31.12580855889246, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 29.8858498937916, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 55.79476427496411, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 55.29072010796517, - "disaggregated/test_workers.py::test_workers_conditional_disaggregation[TinyLlama-1.1B-Chat-v1.0]": 99.67464494984597, - "disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0]": 113.69421282503754, - "disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0]": 195.86076739802957, - "disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0]": 76.3156570668798, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-BertModel-bert/bert-base-uncased]": 111.17977902293205, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:1-pp:1-float32-RobertaModel-bert/roberta-base]": 115.47540166974068, - "examples/test_bert.py::test_llm_bert_general[compare_hf-disable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 64.32757595699513, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.807815481035504, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-BertModel-bert/bert-base-uncased]": 60.69481396203628, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForQuestionAnswering-bert/bert-base-cased-squad2]": 73.41519981494639, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 73.00019321503350511193, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 121.36917966976762, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForQuestionAnswering-bert/roberta-base-squad2]": 74.99331583303865, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaForSequenceClassification-bert/twitter-roberta-base-emotion]": 73.000189481012057513, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-RobertaModel-bert/roberta-base]": 122.99858937039971, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha_fp32_acc-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 55.26069303997792, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-disable_weight_only]": 3640.0045928549953, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-chat-enable_weight_only]": 3235.5094731519348, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-disable_weight_only]": 3273.7859199331142, - "examples/test_chatglm.py::test_llm_glm_4_9b_single_gpu_summary[glm-4-9b-enable_weight_only]": 3602.5807401749771, - "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[disable_weight_only]": 1856.5704392530024, - "examples/test_commandr.py::test_llm_commandr_v01_single_gpu_summary[enable_weight_only]": 654.6126579149859, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 244.9744301661849, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_logits-draft_len_8-float16-bs1]": 398.78198734589387, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 241.73137632384896, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[no_streaming-llama_v2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 1099.3959164519329, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 249.52418848499656, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_4-float16-bs2]": 257.3995385244489, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-gpt2-use_cpp_session-use_tokens-draft_len_8-float16-bs1]": 397.20915103994776, - "examples/test_draft_target_model.py::test_llm_draft_target_model_1gpu[streaming-llama_v2-use_cpp_session-use_logits-draft_len_4-float16-bs2]": 582.6556157508167, - "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle1]": 254.24225717037916, - "examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle2]": 246.98607586231083, - "examples/test_eagle.py::test_llm_eagle_1gpu_modelopt_ckpt[llama3.1-eagle-8b-hf_v0.5-float16-bs8]": 489.4540088879876, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 232.2934926636517, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-bfloat16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 281.1201816312969, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 2954.5586752621457, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-bart-large-cnn-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 276.10329104214907, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 233.94542215764523, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 283.8865255280398, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 220.32321695238352, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-disable_gemm_plugin-enable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.02495155483484, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 96.25796012202045, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 568.2032693652436, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-flan-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 354.8090338760521, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 160.50506488600513, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-mbart-large-50-many-to-one-mmt-float16-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:2-nb:1-enable_fp8]": 300.20793505245819688, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-disable_gemm_plugin-disable_attention_plugin-disable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 218.7171499580145, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-disable_fp8]": 217.96836187317967, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 498.1236152825877, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:2-disable_fp8]": 205.83720442652702, - "examples/test_enc_dec.py::test_llm_enc_dec_general[compare_hf-t5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-enable_fp8]": 295.6078515049885, - "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:1-pp:1-nb:1-enable_fp8]": 252.0442810030654, - "examples/test_enc_dec.py::test_llm_enc_dec_general[no_compare_hf-byt5-small-float32-enable_gemm_plugin-enable_attention_plugin-enable_paged_kv_cache-tp:2-pp:1-nb:1-disable_fp8]": 95.48429084802046, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 422.4394793640822, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-enable_fp8]": 1074.875556848012, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:1]": 3002.19654186069965, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_3.0_7.8b_instruct-float16-nb:4]": 3074.5337073504925, - "examples/test_exaone.py::test_llm_exaone_1gpu[disable_weight_only-exaone_deep_2.4b-float16-nb:4]": 243.4259528592229, - "examples/test_exaone.py::test_llm_exaone_1gpu[enable_weight_only-exaone_deep_2.4b-float16-nb:1]": 212.50885355699575, - "examples/test_exaone.py::test_llm_exaone_2gpu[exaone_3.0_7.8b_instruct-float16-nb:1]": 7155.35844087804435, - "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-27b-it]": 317.7816583644599, - "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 317.7816583644599, - "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 195.3050664511975, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_cpp_session-enable_gemm_plugin]": 114.20040711760521, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[non_streaming-use_py_session-disable_gemm_plugin]": 105.38906556204893, - "examples/test_gpt.py::test_llm_gpt2_medium_1gpu[streaming-use_cpp_session-enable_gemm_plugin]": 113.51056583970785, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_cpp_session]": 194.89961875230074, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[non_streaming-use_py_session]": 200.52475621178746, - "examples/test_gpt.py::test_llm_gpt2_medium_bad_words_1gpu[streaming-use_cpp_session]": 195.00627667084336, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_cpp_session]": 194.90547297894955, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[non_streaming-use_py_session]": 194.89357279613614, - "examples/test_gpt.py::test_llm_gpt2_medium_stop_words_1gpu[streaming-use_cpp_session]": 194.72326660901308, - "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_cpp_session-tp1]": 460.1370678450912, - "examples/test_gpt.py::test_llm_gpt2_next_prompt_tuning[use_py_session-tp1]": 403.39630596572533, - "examples/test_gpt.py::test_llm_minitron_fp8_with_pseudo_loras[4b]": 259.4826051471755, - "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct]": 140.08338637300767, - "examples/test_granite.py::test_granite_bf16_lora[granite-3.0-2b-instruct]": 146.46410073013976, - "examples/test_granite.py::test_llm_granite[granite-3.0-1b-a400m-instruct-bfloat16]": 141.05149138718843, - "examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16]": 1055.801738537848, - "examples/test_internlm.py::test_llm_internlm2_7b_1node_1gpu[bfloat16-enable_context_fmha-enable_gemm_plugin-enable_attention_plugin-nb:2]": 2690.7412294782698, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 160.33107751235366, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-1b]": 117.10959041584283, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.2-3b]": 215.69512976403348, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v2-7b-hf]": 376.96383721905295, - "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-v3-8b-instruct-hf]": 317.46144750900567, - "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 853.2910006027669, - "examples/test_llama.py::test_llm_llama_1gpu_batched_beam_search[llama-7b]": 182.20104870200157, - "examples/test_llama.py::test_llm_llama_1gpu_fp8_kv_cache[llama-v2-7b-hf-bfloat16]": 313.6555140609853, - "examples/test_llama.py::test_llm_llama_long_alpaca_8gpu_summary[pg64317-tp8pp1-nb:1]": 441.2820012559532, - "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 167.92376559507102, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_awq]": 420.1779588930076, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16]": 895.7611340929288, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp8]": 314.3205590210273, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_int8_wo]": 329.1954380639363, - "examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_sq_ootb]": 216.2645359209855, - "examples/test_llama.py::test_llm_llama_v3_1_1node_single_gpu[llama-3.2-1b-disable_fp8]": 382.12588274572045, - "examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16]": 517.2770831151865, - "examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0]": 64.8964971601963, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-1.4b-float16-enable_gemm_plugin]": 181.03255189501215, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-disable_gemm_plugin]": 136.8141469657421, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-130m-float16-enable_gemm_plugin]": 112.04011878371239, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-2.8b-float16-disable_gemm_plugin]": 336.9864642599714, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-370m-float16-enable_gemm_plugin]": 87.74965502799023, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-790m-float16-disable_gemm_plugin]": 148.56857219303492, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-disable_gemm_plugin]": 405.1586506664753, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba-codestral-7B-v0.1-float16-enable_gemm_plugin]": 384.9690850973129, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-1.3b-float16-enable_gemm_plugin]": 126.15442710905336, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-disable_gemm_plugin]": 129.8332964628935, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-130m-float16-enable_gemm_plugin]": 118.37521690130234, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-2.7b-float16-disable_gemm_plugin]": 212.40601160994265, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-370m-float16-enable_gemm_plugin]": 75.9140890170238, - "examples/test_mamba.py::test_llm_mamba_1gpu[mamba2-780m-float16-disable_gemm_plugin]": 114.81139776791679, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 217.8724013082683, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 460.24718615040183, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs1]": 129.02341048512608, - "examples/test_medusa.py::test_llm_medusa_1gpu[use_py_session-medusa-vicuna-7b-v1.3-4-heads-bfloat16-bs8]": 524.8282293006778, - "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 383.3182801879011, - "examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1]": 293.03625723719597, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-chunked_summarization_long]": 429.02448211982846, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization]": 603.6547773182392, - "examples/test_mistral.py::test_llm_mistral_v1_1gpu[mistral-7b-v0.1-float16-max_attention_window_size_4096-summarization_long]": 391.7267559207976, - "examples/test_mixtral.py::test_llm_mixtral_moe_plugin_lora_4gpus[Mixtral-8x7B-v0.1-chinese-mixtral-lora]": 3600.00015190104022622108, - "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-cnn_dailymail-Qwen2-VL-7B-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 332.0248579243198, - "examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-scienceqa-Llama-3.2-11B-Vision-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 411.7690062429756, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 385.0684349639341, - "examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:2-bfloat16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 282.8564471802674, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3-vision-128k-instruct-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 90.19939221709501, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-3.5-vision-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 275.5947739640251, - "examples/test_multimodal.py::test_llm_multimodal_general[Phi-4-multimodal-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 644.3520091949031, - "examples/test_multimodal.py::test_llm_multimodal_general[Qwen2-VL-7B-Instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:4]": 656.4784073680639, - "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 249.87254932150245, - "examples/test_multimodal.py::test_llm_multimodal_general[deplot-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 100.08572303387336, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 492.22362083010375, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1]": 333.81485258904286, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[kosmos-2-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 459.2980541479774, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1]": 176.8547668098472, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1]": 317.7816583644599, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 533.2010767317843, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-onevision-qwen2-7b-ov-hf-video-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 200.52463799191173, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 274.8723033480346, - "examples/test_multimodal.py::test_llm_multimodal_general[llava-v1.6-mistral-7b-hf-vision-trtllm-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1]": 306.38610201328993, - "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 226.57772368215956, - "examples/test_multimodal.py::test_llm_multimodal_general[nougat-base-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 198.46779074892402, - "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1]": 260.0002240890171378851, - "examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1]": 260.00014175008982419968, - "examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B]": 335.41048416192643, - "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.00026965001598000526, - "examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 196.1214354224503, - "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs1]": 200.0001331439707428217, - "examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2]": 195.90045699477196, - "examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16]": 217.61977925198153, - "examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp8]": 220.0002483620774000883, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3-mini-128k-instruct-fp8-float16]": 360.0002299160696566105, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-MoE-instruct-fp8-bfloat16]": 360.0001645770389586687, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-3.5-mini-instruct-fp8-float16]": 360.0001653869403526187, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[Phi-4-mini-instruct-fp8-bfloat16]": 360.00032463017851114273, - "examples/test_phi.py::test_llm_phi_quantization_1gpu[phi-2-fp8-bfloat16]": 154.69453522900585, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_0.5b_instruct]": 99.85199454193935, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 193.38715927954763, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2_0.5b_instruct]": 128.65416727401316, - "examples/test_qwen.py::test_llm_qwen1_5_7b_single_gpu_lora[qwen1.5_7b_chat-Qwen1.5-7B-Chat-750Mb-lora]": 338.59182655182667, - "examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct]": 423.2417808100581, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001445889938622713, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-fp8-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00012858898844569921, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int4_awq-float16-enable_attn_plugin-enable_gemm_plugin]": 648.7579195387661, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int8_sq-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00033479504054412246, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-flax-no_paged_cache-disable_quant-float16-enable_attn_plugin-disable_gemm_plugin]": 286.15992603078485, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-disable_attn_plugin-enable_gemm_plugin]": 680.0001984360278584063, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-no_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.00013055797899141908, - "examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_py_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin]": 680.0001333290128968656, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 386.68252966180444, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_cpp_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 204.7229775050655, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb5-bs8]": 411.88197461143136, - "examples/test_redrafter.py::test_llm_redrafter_1gpu[use_py_session-redrafter-vicuna-7b-v1.3-bfloat16-dl5-nb8-bs8]": 429.239758990705, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 327.95307156071067, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 249.98457504063845, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 225.60136043280363, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int4-float16-nb:1-use_cpp_runtime]": 114.39965145103633, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-int8-float16-nb:1-use_cpp_runtime]": 109.14321990706958, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-enable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime]": 110.00021485507022589445, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging": 458.4059731345624, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default": 40.70597969903611, - "llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_tensorrt": 99.26517963293009, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-codellama/CodeLlama-7b-Instruct-hf]": 383.1450063039083, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 101.96415167767555, - "llmapi/test_llm_examples.py::test_llmapi_server_example": 107.44753806525841, - "perf/test_perf.py::test_perf[bert_base-cpp-ootb-float16-bs:32-input_len:32]": 111.37450777366757, - "perf/test_perf.py::test_perf[bert_base-cpp-plugin-float16-bs:32-input_len:32]": 95.00738414749503, - "perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512]": 140.2516261599958, - "test_e2e.py::test_build_time_benchmark_sanity": 165.71592589840293, - "test_e2e.py::test_gpt3_175b_1layers_build_only": 131.34366285055876, - "test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding-]": 90.12235352798598, - "test_e2e.py::test_llama_e2e[use_py_session--]": 92.68217968731187, - "test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding-]": 93.41917416104116, - "llmapi/test_llm_examples.py::test_llmapi_chat_example": 105.19824166595936, - "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 73.23708964884281, - "llmapi/test_llm_examples.py::test_llmapi_example_inference": 66.82718145102262, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 65.93082024902105, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 67.49109892174602, - "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 72.87169548124075, - "llmapi/test_llm_e2e.py::test_llmapi_exit": 32.64902823418379, - "llmapi/test_llm_e2e.py::test_llmapi_load_ckpt_from_convert_command": 180.59318951144814, - "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 225.2778383679688, - "llmapi/test_llm_examples.py::test_llmapi_quickstart": 66.53727849572897, - "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 110.45052940770984, - "test_e2e.py::test_mistral_e2e[use_cpp_session-remove_input_padding--]": 192.74050169810653, - "test_e2e.py::test_mistral_e2e[use_py_session---]": 160.08483010903, - "test_e2e.py::test_mistral_e2e[use_py_session-remove_input_padding--]": 157.39577213302255, - "test_e2e.py::test_mistral_large_hidden_vocab_size": 81.36711680702865, - "test_e2e.py::test_openai_chat_example": 876.1966922096908, - "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 55.12449237401597, - "test_e2e.py::test_openai_chat_harmony": 1162.7252594940364, - "test_e2e.py::test_openai_chat_multimodal_example": 215.8254322744906, - "test_e2e.py::test_openai_misc_example": 256.0453990884125, - "test_e2e.py::test_openai_multi_chat_example": 684.1194127409253, - "test_e2e.py::test_ptp_quickstart": 45.67732956283726, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 37.10870039300062, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 81.43792725296225, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 56.05445321695879, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.2-11B-BF16-llama-3.2-models/Llama-3.2-11B-Vision]": 458.2611172522884, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 1198.8076391071081, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-False]": 7750.113898515003, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Maverick-17B-128E-Instruct-FP8-llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8-True]": 7404.8092977448832, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP4-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4-True]": 3600.00027390988543629646, - "test_e2e.py::test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k[Llama-4-Scout-17B-16E-Instruct-FP8-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8-True]": 2183.7999707763083, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 8346.51794013707, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8]": 11215.495469792979, - "test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B]": 109.26379436196294, - "test_e2e.py::test_ptp_quickstart_advanced_mixed_precision": 80.88908524392173, - "test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 99.42739840806462, - "test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct]": 71.96910276100971, - "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 21.019993914989755, - "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 18.753523574909195, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[llava-v1.6-mistral-7b-llava-v1.6-mistral-7b-hf-0.8-image]": 354.18758003995754, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[phi4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct-0.8-image]": 313.79139575595036, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-image]": 47.84053734713234, - "test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[qwen2.5-vl-7b-instruct-Qwen2.5-VL-7B-Instruct-0.8-video]": 58.646747548133135, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[Phi-4-multimodal-instruct-multimodals/Phi-4-multimodal-instruct]": 0.00018897396512329578, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[gemma-3-27b-it-gemma/gemma-3-27b-it]": 81.0151858178433, - "test_e2e.py::test_ptp_quickstart_multimodal_multiturn[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503]": 60.082822071621194, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[audio]": 89.50578387826681, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image]": 66.199569062097, - "test_e2e.py::test_ptp_quickstart_multimodal_phi4mm[image_audio]": 62.389084175927565, - "test_e2e.py::test_ptp_scaffolding[DeepSeek-R1-Distill-Qwen-7B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B]": 7200.0001350759994238615, - "test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B]": 137.7278483910486, - "test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]": 12134.278186964104, - "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 109.25386995915323, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 163.86223009089008, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 114.50899445591494, - "test_e2e.py::test_trtllm_bench_iteration_log[TRT-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 285.3362849447876, - "test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 310.9046222809702, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 114.17938271397725, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 115.74023819994181, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 276.64185731019825, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 252.97791706770658, - "test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b]": 482.31134233786725, - "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 80.71726501686499, - "test_e2e.py::test_trtllm_multimodal_benchmark_serving": 360.00022564898245036602, - "test_e2e.py::test_trtllm_serve_example": 200.09309104084969, - "test_e2e.py::test_trtllm_serve_multimodal_example": 130.2214687075466, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 26.32902159006335, - "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 588.56, - "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu]": 539.3006387590431, - "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 31.94, - "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 36.69, - "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 170.86, - "test_unittests.py::test_unittests_v2[unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py]": 1004.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 600.5, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 718.749935634085, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 208.1838396479725, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mllama\"]": 749.5508671940188, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron\"]": 1952.3731448464096, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 498.8839871880482, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 55.078535287990235, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 551.1881373599754, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 401.2630233000382, - "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 79.90315388399176, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules]": 158.5, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 68.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 1598.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 22.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DENSEGEMM\"]": 149.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"TRTLLM\"]": 998.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 393.0210295501165, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_external_embedding.py]": 30.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_find_num_image_tokens.py]": 240.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_fuse_input_embeds.py]": 60.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_mm_encoder_standalone.py]": 1800.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_multimodal_runtime.py]": 180.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal/test_share_multiparams.py]": 30.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1020.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/speculative]": 1850.16, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 1463.0, - "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 18.96, - "test_unittests.py::test_unittests_v2[unittest/api_stability]": 33.137137457728386, - "test_unittests.py::test_unittests_v2[unittest/bindings]": 1119.2564616799355, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 34.61376368254423, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_executor.py]": 378.7100401185453, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"not part0\"]": 1883.5484512336552, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm.py -m \"part0\"]": 1601.0243577323854, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"not (part0 or part1)\"]": 825.9972547292709, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part0\"]": 163.72848848626018, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_models.py -m \"part1\"]": 538.573951125145, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_perf_evaluator.py]": 118.36046380549669, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_gemma3_1b_instruct_multi_lora]": 101.58543362899218, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py]": 539.5857984796166, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 477.989566125907, - "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 125.15857975929976, - "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 973.1355891097337, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_bert_attention.py]": 99.96196278184652, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 77.31474154582247, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 84.67568279313855, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 75.39135546097532, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 78.77339706313796, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"trtllm_gen\"]": 376.012343961047, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 267.40264504775405, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 85.18935105204582, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 49.3486054521054, - "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_sage_attention.py unittest/llmapi/test_llm_download.py unittest/llmapi/test_llm_kv_cache_events.py unittest/llmapi/test_mpi_session.py unittest/trt/model/redrafter unittest/trt/model/test_phi.py unittest/trt/model/test_unet.py unittest/trt/python_plugin unittest/tools unittest/utils unittest/others]": 940.7867036014795, - "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_fp4_gemm.py]": 302.49857676401734, - "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 220.60184395778924, - "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 778.6451135131065, - "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 212.3223411180079, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"other\"]": 125.12117889150977, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition0\"]": 300.0489609502256, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition1\"]": 265.11671224981546, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition2\"]": 357.6496359631419, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt.py -k \"partition3\"]": 371.381394200027, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 537.5006402550498, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1494.1103300452232, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mamba.py]": 76.84791256207973, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 366.95385985821486, - "test_unittests.py::test_unittests_v2[unittest/trt/model/test_nemotron_nas.py -k \"not fp8\"]": 1041.1297603696585, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 27.33125525712967, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 33.818626184016466, - "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 493.8186915554106, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 214.35422350093722, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 100.81762219779193, - "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 673.2582192085683 + "cpp/test_e2e.py::test_model[-eagle-86]": 467.31710943579674, + "cpp/test_e2e.py::test_model[-medusa-86]": 559.4042182117701, + "test_fmha.py::test_fmha": 1843.8030557027087, + "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 371.68835608288646, + "llmapi/test_llm_examples.py::test_llmapi_example_inference": 390.4545612037182, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 28.57412961218506, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 71.92784738913178, + "llmapi/test_llm_examples.py::test_llmapi_example_logits_processor": 28.32273621391505, + "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 46.27545684855431, + "llmapi/test_llm_examples.py::test_llmapi_quickstart": 29.23038684949279, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mha]]": 16.939650654792786, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mla]]": 16.761926997452974, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]]": 12.963495340198278, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]]": 12.825301811099052, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 75.91953653842211, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 36.62238800525665, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 47.69775189831853, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[False]": 20.00915778055787, + "llmapi/test_llm_api_connector.py::test_connector_async_save[False]": 13.049194630235434, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[False]": 27.849122423678637, + "llmapi/test_llm_api_connector.py::test_connector_e2e_persistent_cache": 24.956512186676264, + "llmapi/test_llm_api_connector.py::test_connector_priorities": 15.430590681731701, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[attention_dp]": 5.342560388147831, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[host_offloading]": 12.192664381116629, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[True]": 17.182394333183765, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[True]": 11.133101604878902, + "llmapi/test_llm_api_connector.py::test_connector_simple[True]": 12.597462255507708, + "test_e2e.py::test_openai_completions_example[pytorch]": 256.56871559098363, + "test_e2e.py::test_openai_mmencoder_example": 128.8613565787673, + "test_e2e.py::test_openai_prometheus": 165.67273128777742, + "test_e2e.py::test_openai_responses_entrypoint": 100.25372431054711, + "test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0]": 70.06262697279453, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 255.5959302908741, + "test_e2e.py::test_trtllm_serve_multimodal_example": 185.39177925512195, + "thirdparty/test_git_modules.py::test_gitmodules": 0.0012150108814239502, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_async_transfer_manager.py]": 24.65272542834282, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_budget_split.py]": 24.599674943834543, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_scheduler_serializable_output.py]": 23.298357471823692, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_mistral.py]": 31.256194911897182, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py]": 30.9562314376235, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_nemotron_nano_preprocessing.py]": 56.75650619342923, + "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_weight_loader.py]": 23.097423560917377, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_sampler.py]": 261.76642061397433, + "test_unittests.py::test_unittests_v2[unittest/_torch/test_model_config.py]": 22.64773726090789, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_block.py]": 19.638685578946024, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cluster_storage.py]": 41.06863783299923, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_openai_client.py]": 26.17624157294631, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_extractor.py]": 19.388083309866488, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_messenger.py]": 21.500893251039088, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_service.py]": 27.999925918877125, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_remoteDictionary.py]": 49.35567116411403, + "test_unittests.py::test_unittests_v2[unittest/executor/test_fatal_error_health_check.py]": 25.49836080148816, + "test_unittests.py::test_unittests_v2[unittest/executor/test_rpc.py]": 497.4198516495526, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_content_format.py]": 23.498350095003843, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_url_validation.py]": 24.147701989859343, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_harmony_channel_validation.py]": 36.30120064690709, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu1\"]": 48.403994899243116, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_kv_cache_dtype_override.py]": 24.848896216601133, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryArchitectureExtraction]": 57.554038252681494, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_utils.py]": 87.86072044074535, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_reasoning_parser.py]": 23.850911132991314, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_serialization.py]": 23.949409436434507, + "test_unittests.py::test_unittests_v2[unittest/media/test_encoding.py]": 23.400118369609118, + "test_unittests.py::test_unittests_v2[unittest/others/test_lora_manager.py]": 33.45008610934019, + "test_unittests.py::test_unittests_v2[unittest/others/test_time_breakdown.py]": 25.352738097310066, + "test_unittests.py::test_unittests_v2[unittest/scripts]": 29.149033349007368, + "test_unittests.py::test_unittests_v2[unittest/usage/test_config.py]": 19.241729954257607, + "test_unittests.py::test_unittests_v2[unittest/usage/test_opt_out.py]": 19.389309899881482, + "test_unittests.py::test_unittests_v2[unittest/usage/test_schema.py]": 18.941728390287608, + "test_unittests.py::test_unittests_v2[unittest/utils/test_logger.py]": 23.649205207824707, + "test_unittests.py::test_unittests_v2[unittest/visual_gen/test_output.py]": 24.097700644284487, + "disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0]": 83.51802018284798, + "disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0]": 99.42717270739377, + "disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0]": 91.52266521565616, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 20.080937860999256, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 60.11040577664971, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[True]": 17.928842270746827, + "llmapi/test_llm_api_connector.py::test_connector_async_save[True]": 12.977049142122269, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[True]": 25.513779915869236, + "llmapi/test_llm_api_connector.py::test_connector_multi_request": 9.052204478532076, + "llmapi/test_llm_api_connector.py::test_connector_priorities_default": 12.085481505841017, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[beam_search]": 7.714497704058886, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[False]": 16.96005078405142, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[False]": 9.707948675379157, + "llmapi/test_llm_api_connector.py::test_connector_simple[False]": 13.015593154355884, + "test_e2e.py::test_get_ci_container_port": 0.0005284938961267471, + "test_e2e.py::test_openai_chat_example[pytorch]": 561.0242523606867, + "test_e2e.py::test_openai_misc_example[pytorch]": 274.1224825847894, + "test_e2e.py::test_openai_perf_metrics": 79.67633541114628, + "test_e2e.py::test_openai_reasoning[pytorch]": 240.12692171894014, + "test_e2e.py::test_openai_tool_call": 88.49165302142501, + "test_e2e.py::test_trtllm_serve_top_logprobs[pytorch]": 87.6148243919015, + "thirdparty/test_cmake_third_party.py::test_cmake_listfiles": 0.10550227761268616, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_disagg_index_mapper_early_release.py]": 25.352552112191916, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_kv_cache_estimation.py]": 25.74726054444909, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_cohere2.py]": 48.46714654006064, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_parakeet.py]": 42.047723326832056, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_radio.py]": 25.22459696419537, + "test_unittests.py::test_unittests_v2[unittest/_torch/models/checkpoints/hf/test_checkpoint_loader.py]": 56.12043555267155, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_torch_multi_arange.py]": 41.8165452927351, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_trtllm_sampler.py]": 173.59781158529222, + "test_unittests.py::test_unittests_v2[unittest/_torch/visual_gen/test_visual_gen_params.py]": 24.5291297249496, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_agent_multi_backends.py]": 55.291444917209446, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_cluster_manager_worker.py]": 50.3197379950434, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disagg_utils.py]": 18.99160304525867, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_mamba_transfer.py]": 32.886980537325144, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_openai_disagg_server.py]": 24.754129583016038, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_peer.py]": 18.989873898215592, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_router.py]": 22.49254883499816, + "test_unittests.py::test_unittests_v2[unittest/executor/test_ipc.py]": 31.610522534698248, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_chat_template_dispatch.py]": 25.169238595291972, + "test_unittests.py::test_unittests_v2[unittest/inputs/test_multimodal.py]": 25.204775847494602, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_chat_utils.py]": 24.531344229355454, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_tool_parsers.py]": 28.049183627590537, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_gc_utils.py]": 31.62360288016498, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_args.py]": 166.04291549883783, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryPyTorchBackend]": 59.91823088936508, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_request_priority.py]": 25.325514804571867, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_utils.py]": 26.306996766477823, + "test_unittests.py::test_unittests_v2[unittest/others/test_convert_utils.py]": 23.858180426061153, + "test_unittests.py::test_unittests_v2[unittest/others/test_lora_module_count.py]": 25.352086970582604, + "test_unittests.py::test_unittests_v2[unittest/others/test_tracing.py]": 134.26669367216527, + "test_unittests.py::test_unittests_v2[unittest/usage/test_collectors.py]": 21.589319362770766, + "test_unittests.py::test_unittests_v2[unittest/usage/test_e2e_capture.py]": 21.25535298185423, + "test_unittests.py::test_unittests_v2[unittest/usage/test_reporter.py]": 21.342137079220265, + "test_unittests.py::test_unittests_v2[unittest/usage/test_transport.py]": 23.60382394399494, + "test_unittests.py::test_unittests_v2[unittest/utils/test_util.py]": 26.621632516384125, + "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 85.27059441700112, + "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 193.6268605251098, + "test_unittests.py::test_unittests_v2[unittest/api_stability]": 25.643251301022246, + "test_unittests.py::test_unittests_v2[unittest/bindings]": 714.101646464318, + "test_unittests.py::test_unittests_v2[unittest/dynamo]": 26.950018653995357, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py::TestTelemetryTRTBackend]": 166.76355233497452, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command_with_lora[llama-llama-models-v2/llama-v2-7b-hf]": 185.91652243910357, + "llmapi/test_llm_examples.py::test_llmapi_chat_example": 78.83368532638997, + "llmapi/test_llm_examples.py::test_llmapi_server_example": 83.81749124731869, + "test_e2e.py::test_openai_misc_example[trt]": 193.8965124920942, + "test_e2e.py::test_trtllm_bench_sanity[--non-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 352.73563918611035, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition0\"]": 73.73264672234654, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_no_cache.py]": 54.35324762901291, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_llama.py]": 1274.6575344190933, + "test_e2e.py::test_openai_reasoning[trt]": 112.70626465464011, + "test_e2e.py::test_trtllm_serve_example": 209.98838954232633, + "test_e2e.py::test_trtllm_serve_top_logprobs[trt]": 140.92444621305913, + "test_unittests.py::test_unittests_v2[unittest/test_model_runner_cpp.py]": 476.4086992479861, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition1\"]": 73.379359818995, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition2\"]": 72.02512616757303, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention_IFB.py]": 126.87880356237292, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_mistral.py]": 328.5478213331662, + "llmapi/test_llm_e2e.py::test_llmapi_exit": 18.432179793715477, + "llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-models/llama-7b-hf]": 232.97426022961736, + "llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B]": 82.59823241084814, + "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 114.59312928467989, + "test_e2e.py::test_openai_health": 60.943585466593504, + "test_e2e.py::test_trtllm_bench_latency_sanity[FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 246.85433532483876, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_build_cache.py]": 23.103414099663496, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"partition3\"]": 68.93004452623427, + "test_unittests.py::test_unittests_v2[unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"]": 801.098383044824, + "test_unittests.py::test_unittests_v2[unittest/trt/functional]": 196.5955226458609, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization]": 487.10545333847404, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[gemma-3-1b]": 17.166659578680992, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[phi-4-mini]": 14.305965591222048, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen2-7b]": 13.09041291847825, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[qwen3-0.6b]": 5.315278757363558, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[starcoder2-3b]": 17.601711992174387, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_encode_matches_huggingface[tinyllama-1.1b]": 6.3743207305669785, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_classification[bert-yelp]": 3.595465302467346, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b]": 11.411922980099916, + "test_e2e.py::test_openai_lora": 173.81692308560014, + "test_e2e.py::test_trtllm_serve_lora_example": 98.36129532754421, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_encode.py]": 58.21390492469072, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part0\"]": 598.7036472780164, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part1\"]": 126.27206432109233, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part2\"]": 484.83895123400725, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m \"part3\"]": 208.49013988906518, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_memory_profiling.py::test_pyexecutor_and_kvcache_share_execution_stream]": 40.03293885290623, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_quantization.py]": 23.93977590277791, + "triton_server/test_triton.py::test_medusa[medusa]": 569.5001850500703, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/compile]": 37.790911596268415, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/custom_ops]": 623.6737900990993, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/models]": 115.3445010241121, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/shim]": 22.633222276344895, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/smoke]": 670.1154091879725, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/transformations]": 300.3593855574727, + "test_unittests.py::test_unittests_v2[unittest/auto_deploy/singlegpu/utils]": 24.93424261920154, + "cpp/test_e2e.py::test_model[-gpt_tests-80]": 615.122148225084, + "cpp/test_unit_tests.py::test_unit_tests[common-80]": 12.664204696193337, + "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 687.9856592249125, + "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 1.2464399840682745, + "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 1.8540223143063486, + "cpp/test_e2e.py::test_model[-gpt_executor-80]": 814.0748630631715, + "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 292.39521062839776, + "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 278.46642002556473, + "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 856.879348281771, + "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 548.5782862240449, + "test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]": 73.78876239433885, + "test_e2e.py::test_openai_chat_with_logit_bias[trtllm_sampler]": 73.2108123600483, + "test_e2e.py::test_openai_completions_with_logit_bias[trtllm_sampler]": 72.54250191617757, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen\"]": 754.3435163404793, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search.py]": 789.6666672006249, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_logits_logprobs.py]": 392.0245815804228, + "test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]": 75.10106470668688, + "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 28.074005853384733, + "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 24.315578892827034, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nas\"]": 32.59954630583525, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_out_of_tree\"]": 138.8535058517009, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_phi3\"]": 26.83701508399099, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_qwen_moe\"]": 281.13565809698775, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler/test_beam_search_speculative_d2h.py]": 213.0422897520475, + "triton_server/test_triton.py::test_fill_template[fill-template]": 1.6639938056468964, + "triton_server/test_triton.py::test_llama[llama]": 179.10436917375773, + "triton_server/test_triton.py::test_llmapi_unit_tests[llmapi-unit-tests]": 4.315617096610367, + "triton_server/test_triton.py::test_mistral[mistral]": 175.61003254074603, + "triton_server/test_triton.py::test_mistral_ib_streaming[mistral-ib-streaming]": 342.24799779988825, + "triton_server/test_triton.py::test_python_preproc_unit_tests[python-preproc-unit-tests]": 9.682531874626875, + "triton_server/test_triton_llm.py::test_benchmark_core_model[gptj_6b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 425.68877257127315, + "triton_server/test_triton_llm.py::test_gpt_350m_ifb[test_basic-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 103.82896339520812, + "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[e2e]": 106.43146257847548, + "triton_server/test_triton_llm.py::test_gpt_disaggregated_serving_bls[test_basic-False-1-top_k_top_p--False-True-True-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-True-tensorrt_llm_bls]": 98.34344942122698, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[False-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 268.7699463283643, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-False-ensemble]": 13.90028147213161, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 181.1615214087069, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[test_stop_words-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 16.108110164292157, + "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm]": 78.61408129800111, + "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-python-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 114.62468159664422, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 122.2498293183744, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 33.313044152222574, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 17.912641955539584, + "triton_server/test_triton_llm.py::test_whisper_large_v3_ifb[True-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-0.5-guaranteed_no_evict--24000-1-1-1-False-ensemble]": 137.97018379811198, + "triton_server/test_triton.py::test_gpt[gpt]": 182.20508953463286, + "triton_server/test_triton.py::test_gptj[gptj]": 223.35692731942981, + "triton_server/test_triton.py::test_python_multimodal_encoders_unit_tests[python-multimodal-encoders-unit-tests]": 28.360558917745948, + "triton_server/test_triton.py::test_whisper[whisper]": 204.72570630162954, + "triton_server/test_triton_llm.py::test_benchmark_core_model[llama_v2_7b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 389.1086704917252, + "triton_server/test_triton_llm.py::test_eagle_vicuna_7b_ifb[False-1-eagle--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 189.3712582429871, + "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[accuracy]": 149.95370694343, + "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 278.40986166801304, + "triton_server/test_triton_llm.py::test_llama_v2_7b_ifb[batched_inputs-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-enableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 161.46982649620622, + "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-tensorrt_llm]": 90.04027471225709, + "triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm]": 63.79091303702444, + "triton_server/test_triton_llm.py::test_medusa_vicuna_7b_ifb[False-1-medusa--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 161.17319475393742, + "triton_server/test_triton_llm.py::test_t5_small_enc_dec_ifb[test_basic-False-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap---guaranteed_no_evict--4096-1-1-1-False-tensorrt_llm_bls]": 125.47026043757796, + "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-tensorrtllm-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 119.73666133265942, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 114.50461501348764, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.529517172835767, + "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 18.59724544081837, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp": 251.41180250793695, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_ngram": 98.84480216354132, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False]": 228.78805011400254, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True]": 189.88735772500513, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 294.4758595749736, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 246.82779098799801, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 119.71112632699078, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 222.06951643901994, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention]": 307.9887097366154, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor]": 557.2529671951197, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/mamba]": 221.1838954249397, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"]": 151.7901495929982, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"]": 340.960238839034, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"]": 47.828515230008634, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 48.728463740990264, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 24.82597585298936, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 21.67635251401225, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 25.978178004006622, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 20.424516523984494, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 19.677430056006415, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 24.725896997988457, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 23.124266630999045, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 20.727860749000683, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 42.2266049849859, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_awq_quantization.py]": 19.188348875846714, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_activation_quant.py]": 20.014674406964332, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_add_rms_norm_quant.py]": 19.493550593033433, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_fused_moe.py]": 19.816725137876347, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_group_rmn_norm.py]": 23.089757694862783, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_load_balancer.py]": 19.26597039285116, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_routing.py]": 28.89103313582018, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_rotary_embedding.py]": 23.866294936975464, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_triton_linear.py]": 23.79145319806412, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/tests_lora_modules]": 659.0716862599365, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel]": 4351.49280942278, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/serial]": 58.17412027902901, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 217.26752274786122, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 222.7607175433077, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 302.9856714049997, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 223.62056540699996, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 200.86565831400003, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 189.0692499190004, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 1031.1528510910002, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 461.9252251499993, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=False]": 267.6105636049997, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]]": 71.63301057300032, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]]": 48.08178372900056, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 58.63044721000006, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]]": 85.04990262999945, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 50.014132944000266, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]]": 27.723852323000756, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]]": 20.322831061000215, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 28.123089660000005, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 48.727174462999756, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 50.479853326999546, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]]": 48.27790872099922, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 17.62364664799952, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]]": 27.62391243500042, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]]": 18.274859787999958, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_moe_host_sharer.py]": 20.292041711043566, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python": 163.95312987302896, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 287.43463359299994, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 196.23940314585343, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 216.26651366100032, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 249.59442985600026, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 261.57555265400015, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 207.06534601002932, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 155.4231541629415, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 79.62999053788371, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True]": 419.2233988319995, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=True]": 275.25382337999963, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_llama\"]": 119.63326477212831, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/test_mla_helix.py]": 976.3970404448919, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling -k \"deepseek\"]": 354.90453113700005, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 178.26867335097631, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 229.5246064819803, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 172.92849959799787, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.37839249300305, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 266.96483929408714, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.03644429100677, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 188.69488530093804, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 164.00336496997625, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 172.82999583697529, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 251.55259347101673, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False]": 139.83139117099927, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False]": 135.59422479500063, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 111.3068211893551, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False]": 183.35118844200042, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True]": 201.5803480189934, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gpt_oss\"]": 107.69878626195714, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_mixtral\"]": 41.068741515045986, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler]": 2229.254062668013, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-two_model-overlap_scheduler]": 2873.1229173520114, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 1908.5162332250038, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-two_model-overlap_scheduler]": 2359.458676673996, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False]": 222.66332006501034, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 210.72954666399164, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True]": 235.43180733395275, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False]": 186.75015839299886, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True]": 215.9784758510068, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True]": 225.26497576397378, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True]": 181.96771040401654, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 79.8115957059781, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 92.91837230202509, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram": 451.23835995496484, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True]": 393.2114061889588, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True]": 371.7093208580045, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu -m \"not post_merge\"]": 2881.170321949001, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_additional_model_outputs.py -m \"gpu2\"]": 45.69425537396455, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks]": 88.89764528500382, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync": 275.9294940570253, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 280.02051854197634, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False]": 226.95165822003037, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True]": 176.63856029702583, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False]": 213.76671027002158, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False]": 221.16686225897865, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False]": 180.42018839297816, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False]": 198.51033656601794, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 188.66204771603225, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False]": 293.0652321569505, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 179.5286056349869, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False]": 93.58281131199328, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False]": 393.84327019704506, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill": 393.43859653401887, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism]": 44.9982833910035, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/test_disagg_serving_perf_metrics.py]": 71.39680112700444, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu2\"]": 272.88288813398685, + "examples/test_ray.py::test_llm_inference_distributed_ray[pp2]": 138.11034928797744, + "examples/test_ray.py::test_llm_inference_distributed_ray[tep2]": 109.29040575103136, + "examples/test_ray.py::test_llm_inference_distributed_ray[tp2]": 72.43276466900716, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu2\"]": 235.5710942610167, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part0\"]": 341.03334485401865, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part1\"]": 341.3838682859787, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part2\"]": 179.14875201904215, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part3\"]": 180.01359257102013, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part4\"]": 215.3152888190234, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu2\"]": 71.13692150497809, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer]": 191.26553119625896, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False]": 440.4416093239561, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False]": 61.59581129811704, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False]": 46.71345825213939, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False]": 171.76501162629575, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False]": 240.30174564756453, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False]": 75.44280633982271, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True]": 127.60458957124501, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-trtllm]": 399.2673514550552, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-mpi_kvcache-90]": 566.998665202409, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-nixl_kvcache-90]": 9.834118995815516, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-ucx_kvcache-90]": 10.331790680065751, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-mpi_kvcache-90]": 23.657401267439127, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-nixl_kvcache-90]": 23.906753124669194, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-ucx_kvcache-90]": 23.80666296184063, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-mpi_kvcache-90]": 12.536427475512028, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-nixl_kvcache-90]": 12.840721806511283, + "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-ucx_kvcache-90]": 12.540847543627024, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-mpi_kvcache-90]": 18.39725266955793, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-nixl_kvcache-90]": 16.89188546873629, + "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-ucx_kvcache-90]": 16.693953413516283, + "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-nixl_kvcache-90]": 30.972941564396024, + "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-ucx_kvcache-90]": 29.168602090328932, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-mpi_kvcache-90]": 4.774993252009153, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-nixl_kvcache-90]": 4.874061470851302, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-ucx_kvcache-90]": 4.673807211220264, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-mpi_kvcache-90]": 9.332997119054198, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-nixl_kvcache-90]": 9.682227460667491, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-ucx_kvcache-90]": 9.431488024070859, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-mpi_kvcache-90]": 11.735960626974702, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-nixl_kvcache-90]": 11.684558764100075, + "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-ucx_kvcache-90]": 11.736474914476275, + "cpp/test_multi_gpu.py::test_enc_dec[t5-90]": 6.025917749851942, + "cpp/test_multi_gpu.py::test_llama_executor[llama-leader-90]": 84.81573174893856, + "cpp/test_multi_gpu.py::test_llama_executor[llama-orchestrator-90]": 162.71292873471975, + "cpp/test_multi_gpu.py::test_llama_executor_guided_decoding[llama-90]": 13.140320116654038, + "cpp/test_multi_gpu.py::test_llama_executor_logits_proc[llama-90]": 44.34581693634391, + "cpp/test_multi_gpu.py::test_trt_gpt_real_decoder[llama-90]": 113.51448027789593, + "cpp/test_multi_gpu.py::test_mpi_utils[90]": 45.64661745913327, + "cpp/test_multi_gpu.py::test_user_buffer[2proc-90]": 2.11952923797071, + "cpp/test_multi_gpu.py::test_fused_gemm_allreduce[4proc-90]": 19.49912805482745, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 134.54751868266612, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 154.7853800514713, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 135.73957254644483, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 174.52036147098988, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 152.75700911972672, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 211.04387662932277, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 129.80014774948359, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 159.481474686414, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp4-bf16-trtllm-deepseekv3_lite]]": 102.35796185210347, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 152.4078388279304, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 119.35970550915226, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 130.15719587402418, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 200.01216824213043, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus_static_eplb": 124.69750702101737, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16]": 202.9623187012039, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 139.4163328241557, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp1-bf16-trtllm-deepseekv3_lite]]": 93.75344497803599, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 1437.00570011104, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 541.5369369090186, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 471.5579449380166, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 866.0648840669892, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 594.6447642149869, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache_no_reuse-tp4-cutlass-auto]": 419.3978291969979, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 376.8179885330028, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 154.61260438500904, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 100.9895223529893, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 161.34927469299873, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 88.99735977302771, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 99.42438653996214, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[noadp-ctx_tp2pp1-gen_tp1pp1]": 151.80676046095323, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 104.44514593598433, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar]": 54.41597771999659, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache]": 374.00853590102633, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache]": 210.51360351400217, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin]": 100.25905621499987, + "disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin]": 98.90331856295234, + "disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin]": 49.15359141398221, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin]": 49.10201428201981, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin]": 206.90428009995958, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0]": 64.96553604403744, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0]": 85.28914290998364, + "test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8]": 111.94489605998388, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not MXFP8\"]": 1379.5952013240312, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_AWQ\"]": 669.7905237200321, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb]": 109.54814359301236, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu4\"]": 234.50615247199312, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 167.242011513561, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 98.52402048092335, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 199.50387456268072, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 146.5718833785504, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 94.77812515385449, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[adp-ctx_tp2pp1-gen_tp2pp1]": 156.16143955662847, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.73761726729572, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 76.1688071032986, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 74.1406664410606, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache]": 372.94147814717144, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache]": 213.26513709686697, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin]": 139.31395231373608, + "disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin]": 98.91171756386757, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware]": 49.066187175922096, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing]": 213.4513011770323, + "disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing]": 207.4394314819947, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0]": 70.3319569742307, + "test_e2e.py::test_ptp_quickstart_advanced_bs1": 112.39259800966829, + "test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b]": 128.85917745530605, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A16_MXFP4\"]": 1016.8723992183805, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W8A16\"]": 691.5421204390004, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_py_cache_transceiver_mp.py]": 2267.29654201027, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp1_4instances]": 112.71113630395848, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp2_2instances]": 116.105620782997, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu4\"]": 469.73704576998716, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp1_4instances]]": 290.26655738794943, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp2_2instances]]": 318.72108415700495, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"gpu4\"]": 77.99721742799738, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 132.69184266900993, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 158.83748530130833, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse": 153.4440853409469, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse": 82.38394147483632, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler": 71.19869335694239, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=False-overlap_scheduler=False]": 1218.607248812914, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 80.05104345735162, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 107.66202485794201, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format": 62.41016404423863, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash": 621.2223827303387, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8": 152.73669020738453, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 110.01207454688847, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0]": 35.032475878018886, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 101.00177443493158, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 83.89112774282694, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 63.88218005700037, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[no-overlap-chunked]": 168.74234423786402, + "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 61.069874382112175, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 250.20505770668387, + "test_unittests.py::test_unittests_v2[unittest/_torch/compilation]": 25.54568713111803, + "test_unittests.py::test_unittests_v2[unittest/_torch/debugger]": 19.291544676292688, + "test_unittests.py::test_unittests_v2[unittest/_torch/multimodal]": 1385.4457194129936, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_aux.py]": 18.838812944013625, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_request_id.py]": 19.28899485990405, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16]]": 31.09049210883677, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 129.92204322700854, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 235.29396068817005, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 190.64960246894043, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]": 127.35095311468467, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 220.07175700203516, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler": 75.0205802610144, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse": 72.13469798583537, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format": 34.00468918774277, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8": 300.7987077347934, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 309.6223061759956, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 247.2587843281217, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0]": 25.400752289104275, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 85.75992965977639, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 83.36212210077792, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 53.837384417653084, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[max-util-chunked]": 158.87441884400323, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[python-scheduler]": 172.01433536224067, + "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 148.76338731311262, + "test_e2e.py::test_openai_responses": 352.3726217681542, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 92.3254858199507, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 99.50422196800355, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_gemma3\"]": 243.37643587309867, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS\"]": 228.12391290487722, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel_hw_agnostic]": 585.0550106270239, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_region.py]": 20.752864133100957, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_disaggregated_params.py]": 21.85614148201421, + "test_unittests.py::test_unittests_v2[unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16]]": 33.907329918816686, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True]": 233.35579176666215, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format": 96.40042386110872, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]": 120.64296809816733, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=fp8-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 131.03063145186752, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse": 82.81574466405436, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse": 72.67876957170665, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True]": 1075.0943210227415, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 48.7322347429581, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]": 170.77184127038345, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16": 511.3651969926432, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=False-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 126.62458716612309, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False]": 109.76810259604827, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 113.73702853103168, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8]": 848.9961165660061, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 148.54536304995418, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0]": 36.3188966489397, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0]": 28.20248524507042, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 127.0615609427914, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 45.532342946040444, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[guaranteed-chunked]": 155.79833736736327, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-no-chunked]": 161.94301183428615, + "test_e2e.py::test_openai_chat_harmony": 256.37394262477756, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 94.71067717205733, + "test_e2e.py::test_trtllm_benchmark_serving[llama-3.1-model/Meta-Llama-3.1-8B]": 125.41995501983911, + "test_unittests.py::test_unittests_v2[unittest/_torch/lora]": 20.80229289084673, + "test_unittests.py::test_unittests_v2[unittest/_torch/misc]": 344.32345556328073, + "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/test_eagle3.py]": 1019.0683842310682, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/region/test_page.py]": 20.143666464835405, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer_mp.py]": 123.84979211073369, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_rank_info.py]": 18.993583410046995, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_telemetry.py]": 550.8318111081608, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse": 162.39044524217024, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse": 82.52972493879497, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available": 81.73615213809535, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM]": 149.4546632859856, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True]": 1072.4692367021926, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 77.7370796087198, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 100.55706254392862, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]": 120.40908256312832, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash": 184.22292581619695, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format": 59.57457195688039, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True]": 110.55068488512188, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 111.89929933100939, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0]": 23.80086028506048, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 70.33974885300267, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 41.7932134908624, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-chunked]": 163.14287832612172, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix_smoke": 130.64739427994937, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 70.83530108863488, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity]": 253.82326687918976, + "test_unittests.py::test_unittests_v2[unittest/_torch/sampler]": 1191.2096670051105, + "test_unittests.py::test_unittests_v2[unittest/_torch/speculative/hw_agnostic]": 1831.2627570661716, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_cache_transceiver_single_process.py]": 3324.7812275849283, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_kv_transfer.py]": 928.6214334205724, + "test_unittests.py::test_unittests_v2[unittest/disaggregated/test_perf_logger.py]": 21.391628997866064, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 152.97960573100136, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 153.88833729398903, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 121.66436125000473, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.81132309199893, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 171.62797562000924, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.72052242897917, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 154.6956266699999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 129.47093768097693, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.80305856201448, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 156.33859626899357, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 127.20496735500637, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 154.19827068597078, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 116.73455706000095, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 125.1877486439771, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 147.13845685200067, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 123.54514406202361, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.23275308904704, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 81.83421968104085, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False]": 279.30084877600893, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 957.9583136999863, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 89.2490907079773, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 135.65254550502868, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 89.36297566298163, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=FLASHINFER]": 243.73127793997992, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 75.11799066798994, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 93.10347031999845, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 45.586120243999176, + "accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill]": 215.60936970298644, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill]": 344.82283938402543, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 137.0733454680012, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 143.5359488390095, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 116.0004205229925, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 123.336180874001, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 166.7623178389913, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 118.65510969999013, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 145.0461449699942, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 124.612122652994, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 122.85179915899062, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 140.71412417298416, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 114.79413927401765, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 131.09561426599976, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 119.48417889501434, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 176.2980228930246, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 122.85712360299658, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.1533630049671, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 121.68564532196615, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 124.12095914199017, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 141.94041900796583, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 129.43495701398933, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True]": 317.6746448139893, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 447.11634525697445, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 127.43863378401147, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=FLASHINFER]": 187.84411286801333, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 124.13719056698028, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 111.11976641096408, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 140.97133619396482, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 168.74814063502708, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 233.80381705239415, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 77.1988166347146, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 79.65342007577419, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 69.40490325912833, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 76.35176690667868, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 81.16481507569551, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.47008088231087, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 83.34465102478862, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 67.5397269576788, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 89.43130768090487, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 76.1686408482492, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 72.9958230741322, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 78.06165297329426, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.27438813075423, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 80.74985931068659, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 86.10687870532274, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 82.69737496972084, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 87.65647116675973, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 82.57926809042692, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.8943274691701, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.05017046257854, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 122.5022951066494, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 72.29567366093397, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.35832703486085, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 77.41014160588384, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 104.96686546504498, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.51850606128573, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 85.44486574828625, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 71.4350159689784, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 83.38494031503797, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 153.37503067031503, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.90067131072283, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.13767189905047, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.74171372875571, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 94.56583361327648, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.35695255175233, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.06190882995725, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 95.00528683885932, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 106.75661451742053, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 115.989338260144, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 99.97725444287062, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 123.66422721371055, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 103.99547399580479, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 103.63337088748813, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 96.55534584447742, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 98.63648730143905, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 95.0242466814816, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 96.21232785284519, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 98.30433483421803, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False]": 153.95823365077376, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=False]": 63.16813847795129, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True]": 76.7680161446333, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=False]": 63.61583887413144, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=True]": 72.02205647155643, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=True]": 88.9733154848218, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 112.0710693411529, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 60.62598153203726, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 61.945570688694715, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 91.45242443680763, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 62.574849516153336, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False]": 992.8535194210708, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True]": 620.6253603063524, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False]": 808.6647146046162, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True]": 600.0486651174724, + "test_e2e.py::test_trtllm_bench_mgmn": 91.79827606678009, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 910.4367664121091, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 1242.9062750637531, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 292.60953721031547, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 175.63857275992632, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 186.52553553506732, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 161.51636945083737, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 190.38547903299332, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 131.49605344608426, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 104.33410734310746, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 99.25327169150114, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 152.5729262419045, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 89.9921805858612, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[False]": 661.3829264938831, + "accuracy/test_disaggregated_serving.py::TestLlama4ScoutInstruct::test_auto_dtype[True]": 322.62455509230494, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency]": 2162.4878128543496, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_off]": 231.63562587695196, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 89.56677887961268, + "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 67.95142366364598, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora]": 304.1238304525614, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.223822474479675, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]]": 16.374391281977296, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency]": 246.29444654472172, + "test_unittests.py::test_unittests_v2[unittest/_torch/attention/test_attention_mla.py]": 231.48642779141665, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=0]": 163.56748677929863, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=CUTLASS]": 252.98956363508478, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP]": 251.82381451921538, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2]": 211.0549951121211, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2]": 225.89990004291758, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 157.48752847802825, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 204.017874710029, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 284.236327491235, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 166.91178651992232, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 174.30949333589524, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM]": 228.28387101413682, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler]": 439.3828970207833, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-two_model-overlap_scheduler]": 385.81541442498565, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 548.5454997923225, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus_online_eplb[fp8]": 396.00846621999517, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False]": 688.351709254086, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 691.9655529111624, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP]": 410.2264897967689, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS]": 369.08950016694143, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[no_cuda_graph_overlap-cutlass]": 343.0884657520801, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep1-cutlass]": 422.295906488318, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-trtllm]": 407.53346419893205, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary]": 353.82153057679534, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 558.5696469410323, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k]": 813.6045300079277, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k]": 560.361906999955, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 468.6927351569757, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 295.37768525700085, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 391.1908592169639, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 417.4842175709782, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k]": 733.6566050739493, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k]": 787.2090990659781, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 511.71572106005624, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2908.4839029698633, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 333.19444385915995, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k]": 452.1892409371212, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 411.64431313984096, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 337.6239333390258, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k]": 945.6405130829662, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 708.4553801259026, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 473.57998897507787, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 178.40271015278995, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k]": 2171.321392316837, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_1k1k]": 233.66355675505474, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k]": 405.2331366450526, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 546.0889781760052, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 168.79803335713223, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k]": 266.5968868341297, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k]": 203.08553342614323, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 463.4555850980105, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 450.5373125029728, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k]": 521.3741318379762, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep2_1k1k]": 439.9891236410476, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp4_eagle3_1k1k]": 229.3437453189399, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2190.739206559956, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL]": 5787.14215046796, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 667.5444930130616, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 456.85187179397326, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL]": 603.7066158190137, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2627.6349721120205, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2105.2029822249897, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 238.50409000902437, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 206.6761712988373, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 219.68855385808274, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 144.7110821590759, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 283.6205424251966, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 256.5663748360239, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 85.24103148793802, + "examples/test_visual_gen.py::test_visual_gen_quickstart": 244.85972903110087, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_quant.py]": 341.68943272158504, + "test_unittests.py::test_unittests_v2[unittest/trt/model/eagle]": 35.844667091965675, + "test_unittests.py::test_unittests_v2[unittest/trt/model/test_gpt_e2e.py]": 375.8710653409362, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_api_multi_gpu.py]": 16.811496037989855, + "test_unittests.py::test_unittests_v2[unittest/trt/model_api/test_model_level_api.py]": 16.826461140066385, + "accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False]": 880.4145672637969, + "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it": 845.1492025442421, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1]": 365.7970229499042, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1]": 83.0432817041874, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm]": 185.77484482899308, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm]": 528.6534720119089, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True]": 659.9385115206242, + "examples/test_ad_guided_decoding.py::test_autodeploy_guided_decoding_main_json": 105.7096260804683, + "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple]": 64.08092588745058, + "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph]": 45.57437503896654, + "examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights": 0.8169050384312868, + "examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights": 891.9159074425697, + "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]": 245.15667858114466, + "examples/test_ray.py::test_llm_inference_async_ray": 98.74219689599704, + "test_unittests.py::test_unittests_v2[unittest/_torch/executor/test_overlap_scheduler.py]": 552.1893509819638, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py]": 34.445088005042635, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py]": 79.70088898506947, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part0\"]": 555.2823958380613, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part1\"]": 546.5310950400308, + "test_unittests.py::test_unittests_v2[unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part2\"]": 401.5109062109841, + "test_unittests.py::test_unittests_v2[unittest/llmapi/test_async_llm.py -m \"not (gpu2 or gpu4)\"]": 140.57188469800167, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8": 419.2079811580479, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_plugin": 211.37097293697298, + "accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8_gemm_swiglu_plugin": 213.21514989249408, + "accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_fp8_prequantized": 162.781315760687, + "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 332.239777687937, + "examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it]": 277.80719018913805, + "examples/test_gemma.py::test_llm_gemma_1gpu_summary_vswa[gemma-3-1b-it-other-bfloat16-8]": 93.52066849544644, + "examples/test_llama.py::test_llama_3_x_fp8_with_bf16_lora[llama-3.1-8b]": 149.5570853278041, + "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 159.4304439369589, + "test_e2e.py::test_mistral_large_hidden_vocab_size": 65.75925209745765, + "test_e2e.py::test_trtllm_bench_sanity[--streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 223.21146443486214, + "test_e2e.py::test_trtllm_bench_sanity[-extra_config-streaming-FP16-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 222.07093558087945, + "test_unittests.py::test_unittests_v2[unittest/trt/functional/test_moe.py]": 280.5644433479756, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py]": 50.19224583171308, + "test_unittests.py::test_unittests_v2[unittest/trt/quantization/test_weight_only_quant_matmul.py]": 57.290618147701025, + "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]": 428.9558831639588, + "accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype": 407.1404729729984, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_siglip\"]": 223.38703144699684, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::test_llava_next_expand_prompt_token_ids_for_mm]": 23.44378970999969, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl.py::TestQwen3VL::test_all]": 138.33508067900038, + "test_unittests.py::test_unittests_v2[unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s]": 158.215519313002, + "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype": 856.7462824780014, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_nemotron_nano_v2_vl\"]": 511.051033497999, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling -k \"modeling_vila\"]": 36.8462807270007, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_llava_next.py::TestLlavaNext::test_all]": 36.84895586700077, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen2_5vl.py::TestQwen2_5_VL::test_all]": 101.95980946499913, + "test_unittests.py::test_unittests_v2[unittest/_torch/modeling/test_modeling_qwen3vl_moe.py::TestQwen3VLMoe::test_all]": 116.45382200300082, + "examples/test_llama.py::test_llm_llama_1gpu[llama-3.1-8b-instruct-hf-fp8-enable_fp8-float16-summarization-nb:1]": 678.0718930736184, + "examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b]": 798.493692047894, + "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b]": 113.39240002073348, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 169.85340281389654, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 109.8938184119761, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 67.28465028293431, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 967.6696398910135, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf]": 1818.6560711394995, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf]": 566.2712351139635, + "test_unittests.py::test_unittests_v2[unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py]": 18.0341038107872, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py]": 24.46495814807713, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a16_linear.py]": 19.433096984401345, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_w4a8_linear.py]": 18.131888270378113, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_gemm.py]": 44.08381740376353, + "test_unittests.py::test_unittests_v2[unittest/_torch/thop/parallel/test_weight_only_quant_linear.py]": 20.011995047330856 } From 4e12ff7b75d79a4c9a02c9a43dfd4f48433e806a Mon Sep 17 00:00:00 2001 From: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:56:19 +0800 Subject: [PATCH 152/308] [TRTLLM-13015][feat] drop complex visual_gen CLI example scripts (#14632) Signed-off-by: Zhenhua Wang --- docs/source/models/visual-generation.md | 42 +- examples/visual_gen/README.md | 387 +------------- examples/visual_gen/models/wan_t2v.py | 7 +- examples/visual_gen/visual_gen_flux.py | 505 ------------------ examples/visual_gen/visual_gen_ltx2.py | 469 ---------------- .../visual_gen/visual_gen_mgmn_distributed.sh | 114 ---- examples/visual_gen/visual_gen_wan_i2v.py | 472 ---------------- examples/visual_gen/visual_gen_wan_t2v.py | 469 ---------------- .../defs/examples/test_visual_gen.py | 62 ++- 9 files changed, 74 insertions(+), 2453 deletions(-) delete mode 100755 examples/visual_gen/visual_gen_flux.py delete mode 100755 examples/visual_gen/visual_gen_ltx2.py delete mode 100644 examples/visual_gen/visual_gen_mgmn_distributed.sh delete mode 100644 examples/visual_gen/visual_gen_wan_i2v.py delete mode 100755 examples/visual_gen/visual_gen_wan_t2v.py diff --git a/docs/source/models/visual-generation.md b/docs/source/models/visual-generation.md index f528bfc7cc32..69ad11f47fec 100644 --- a/docs/source/models/visual-generation.md +++ b/docs/source/models/visual-generation.md @@ -85,32 +85,24 @@ When served via `trtllm-serve`, the following OpenAI-compatible endpoints are av VisualGen supports both **dynamic quantization** (on-the-fly at weight-loading time from BF16 checkpoints) and **static quantization** (loading pre-quantized checkpoints with embedded scales). Both modes use the [ModelOpt](https://github.com/NVIDIA/TensorRT-Model-Optimizer) `quantization_config` format. -Dynamic quantization via `--linear_type`: +Configure via `VisualGenArgs.quant_config` (YAML or programmatic): -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --linear_type trtllm-fp8-per-tensor \ - --output_path output_fp8.mp4 +```yaml +quant_config: + quant_algo: FP8 # or FP8_BLOCK_SCALES, NVFP4 + dynamic: true ``` -Supported `--linear_type` values: `default` (BF16/FP16), `trtllm-fp8-per-tensor`, `trtllm-fp8-blockwise`, `trtllm-nvfp4`. - -Programmatic usage via `VisualGenArgs.quant_config`: - ```python from tensorrt_llm import VisualGenArgs - -args = VisualGenArgs( - model="/path/to/model", - quant_config={"quant_algo": "FP8", "dynamic": True}, -) +args = VisualGenArgs(model="/path/to/model", quant_config={"quant_algo": "FP8", "dynamic": True}) ``` +Omit `quant_config` for BF16/FP16 baseline. + ### Quantized Attention -In addition to linear-layer quantization, VisualGen exposes two **attention-level** quantization presets that operate inside the attention kernel. They are configured through `AttentionConfig.quant_attention_config` (or the `--quant_attention_mode` flag in the example scripts) and are mutually exclusive with each other. +In addition to linear-layer quantization, VisualGen exposes two **attention-level** quantization presets that operate inside the attention kernel. They are configured through `AttentionConfig.quant_attention_config` and are mutually exclusive with each other. - **QK16PV8** (`CUTEDSL` backend): Keeps Q & K in BF16 and quantizes only V to FP8 (E4M3, per-tensor), thus Bmm1 will be carried out in BF16 with Bmm2 in FP8. Targets Blackwell-class GPUs (`sm_100a` / `sm_103a`) with `head_dim = 128`. - **SAGE** (`TRTLLM` backend): Quantizes Q, K, and V with per-block scaling factors. Q/K are stored as INT8 or FP8 (e4m3) and V as FP8 (e4m3); block sizes are tunable per axis (typically `(q, k, v) = (1, 4, 1)` for Wan-1.3B and `(1, 16, 1)` for larger Wan / FLUX checkpoints). Supported recipes are validated at runtime. @@ -168,15 +160,15 @@ The `teacache_thresh` parameter controls the similarity threshold. Cache-DiT is ### Multi-GPU Parallelism -6 parallelism modes can be combined: +Configured under `VisualGenArgs.parallel_config`. Modes can be combined: -- **CFG Parallelism** (`--cfg_size 2`): Splits positive/negative guidance prompts across GPUs. -- **Ulysses Parallelism** (`--ulysses_size N`): Splits the sequence dimension across GPUs for longer sequences. -- **Parallel VAE** (`--parallel_vae_size N`): Shards the final VAE decode along a spatial axis across GPUs, useful to reduce VAE latency and improve GPU utilization (Constraint: `parallel_vae_size ≤ world_size`). Currently only supported for WAN models. -- **Attention Parallel**: There are 2 methods supported to run attention parallel. Both of these methods require the attention backend to support LSE (`FA4` and `CUTEDSL`) - - - **Attention2D Parallelism** (`--attn2d_row_size N`, `--attn2d_col_size M`): Shards the sequence axis across a 2D `N x M` device mesh, all-gathering Q along rows and K/V along columns so each rank computes a sub-block of the attention matrix (total CP degree = `N * M`; not currently combinable with Ulysses). - - **Ring Attention Parallelism** (`--ring_size N`): Shards the sequence axis across a 1D ring of `N` ranks and streams K/V blocks around the ring so each rank computes its attention output without materializing the full K/V (mutually exclusive with Attention2D). -- **Tensor Parallelism** (`--tp_size N`): Splits attention heads and transformer MLPs across GPUs for faster compute and reduced memory usage. +- **CFG Parallelism** (`cfg_size: 2`): Splits positive/negative guidance prompts across GPUs. +- **Ulysses Parallelism** (`ulysses_size: N`): Splits the sequence dimension across GPUs for longer sequences. +- **Parallel VAE** (`parallel_vae_size: N`): Shards the final VAE decode along a spatial axis (constraint: `parallel_vae_size ≤ world_size`; WAN only). +- **Attention Parallel** — requires an LSE-capable attention backend (`FA4` and `CUTEDSL`): + - **Attention2D** (`attn2d_size: [N, M]`): Shards the sequence axis across an `N × M` device mesh (total CP degree = `N · M`; not combinable with Ulysses). + - **Ring Attention** (`ring_size: N`): Shards the sequence axis across a 1D ring of `N` ranks, streaming K/V blocks (mutually exclusive with Attention2D). +- **Tensor Parallelism** (`tp_size: N`): Splits attention heads and transformer MLPs across GPUs for faster compute and reduced memory usage. ## Developer Guide ### Architecture Overview diff --git a/examples/visual_gen/README.md b/examples/visual_gen/README.md index c08cd3dd871c..0991276b3c2c 100644 --- a/examples/visual_gen/README.md +++ b/examples/visual_gen/README.md @@ -1,383 +1,28 @@ # Visual Generation Examples -Quick reference for running visual generation models. -Please refer to [the VisualGen doc](https://nvidia.github.io/TensorRT-LLM/models/visual-generation.html) -about the details of the feature. +See [the VisualGen doc](https://nvidia.github.io/TensorRT-LLM/models/visual-generation.html) +for feature details. -## Directory Structure +## Layout -| Directory | Purpose | -|-----------|---------| -| [`models/`](models/) | Per-model example scripts — slim API examples (~40 lines) that focus on model-specific request construction and output processing | -| [`configs/`](configs/) | YAML configs shared by offline examples (`--extra_visual_gen_options`) and `trtllm-serve` | -| [`serve/`](serve/) | `trtllm-serve` usage, benchmarking, and client examples | +| Path | Purpose | +|---|---| +| [`quickstart_example.py`](quickstart_example.py) | Minimal VisualGen API example | +| [`models/`](models/) | Per-model example scripts | +| [`configs/`](configs/) | Shared `VisualGenArgs` YAMLs (used by `--visual_gen_args` and `trtllm-serve`) | +| [`serve/`](serve/) | `trtllm-serve` usage, benchmarking, and clients | -## Quick Start - -[`quickstart_example.py`](quickstart_example.py) — generate a video in ~30 lines (Wan T2V, 1 GPU). - -## Per-Model Examples - -Each script under `models/` demonstrates a single model with the VisualGen API. -Engine config (quantization, parallelism, TeaCache, etc.) is an optional YAML -file passed via `--extra_visual_gen_options` — the same flag that `trtllm-serve` uses. +## Usage ```bash -# Default: 1 GPU, model defaults +# Defaults +python quickstart_example.py python models/wan_t2v.py -# With a shared config for NVFP4 quantization -python models/wan_t2v.py --extra_visual_gen_options configs/wan2.2-t2v-fp4-1gpu.yaml -``` - -## Prerequisites - -```bash -# Install dependencies (from repository root) -pip install -r requirements-dev.txt -``` - - -## FLUX (Text-to-Image) - -### Basic Usage - -**FLUX.1:** - -```bash -python visual_gen_flux.py \ - --model_path black-forest-labs/FLUX.1-dev \ - --prompt "A cat sitting on a windowsill" \ - --height 1024 --width 1024 \ - --guidance_scale 3.5 \ - --output_path output.png -``` - -**With FP8 quantization:** - -```bash -python visual_gen_flux.py \ - --model_path black-forest-labs/FLUX.2-dev \ - --prompt "A cat sitting on a windowsill" \ - --linear_type trtllm-fp8-per-tensor \ - --output_path output_fp8.png -``` - -**Batch mode (multiple prompts from file):** - -```bash -python visual_gen_flux.py \ - --model_path black-forest-labs/FLUX.1-dev \ - --prompts_file prompts.txt \ - --output_dir results/ --seed 42 -``` - - -## WAN (Text-to-Video) - -### Basic Usage - -**Single GPU:** - -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --output_path output.mp4 -``` - -**With SageAttention (FP8/INT8 per-block quantized attention):** -```bash -python visual_gen_wan_t2v.py \ - --model_path ${MODEL_ROOT}/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --enable_sage_attention \ - --output_path output.mp4 -``` - -**With TeaCache:** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --enable_teacache \ - --output_path output.mp4 -``` - -### Multi-GPU Parallelism - -WAN supports two parallelism modes that can be combined: -- **CFG Parallelism**: Split positive/negative prompts across GPUs -- **Sequence Parallelism**: - - *Ulysses*: Split sequence along head dimension across GPUs; requires `ulysses_size` to divide the model's head count - - *Attention2D*: 2D mesh sequence parallelism; no head-count constraint; requires `--attention_backend FA4` - - Combining Ulysses and Attention2D is not yet supported -- **Tensor Parallelism** - - Splits attention heads and transformer MLPs across GPUs; requires `tp_size` to divide the model's head count and MLP up dimension. - - Can be combined with Ulysses and CFG - - -**Ulysses Only (2 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --cfg_size 1 --ulysses_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-1 share sequence (6 heads each) - -**CFG Only (2 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --cfg_size 2 --ulysses_size 1 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0 (positive) | GPU 1 (negative) - -**Attention2D Only (4 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend FA4 \ - --cfg_size 1 \ - --attn2d_row_size 2 --attn2d_col_size 2 \ - --output_path output.mp4 -``` -GPU Layout: Sequence equally split among GPU 0-3 - -**TP Only (2 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend FA4 \ - --cfg_size 1 --tp_size 2\ - --output_path output.mp4 +# With engine config (quant, parallelism, etc.) +python models/wan_t2v.py --visual_gen_args configs/wan2.2-t2v-fp4-1gpu.yaml ``` -GPU Layout: Attention heads and MLP projections equally split among GPU 0-1 - -**CFG + Ulysses (4 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --cfg_size 2 --ulysses_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-1 (positive, Ulysses) | GPU 2-3 (negative, Ulysses) - -**CFG + Attention2D (8 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend FA4 \ - --cfg_size 2 \ - --attn2d_row_size 2 --attn2d_col_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-3 (positive, Attention2D) | GPU 4-7 (negative, Attention2D) - -**CFG + TP (4 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --cfg_size 2 --tp_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-1 (positive, TP) | GPU 2-3 (negative, TP) - -**CFG + Ulysses + TP (8 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 480 --width 832 --num_frames 33 \ - --attention_backend TRTLLM \ - --cfg_size 2 --ulysses_size 2 \ - --tp_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-3 (positive, Ulysses + TP) | GPU 4-7 (negative, Ulysses + TP) - -**Large-Scale (64 GPUs):** -```bash -python visual_gen_wan_t2v.py \ - --model_path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --prompt "A cute cat playing piano" \ - --height 720 --width 1280 --num_frames 81 \ - --attention_backend FA4 \ - --attn2d_row_size 8 --attn2d_col_size 4 \ - --cfg_size 2 \ - --output_path output.mp4 -``` -GPU Layout: GPU 0-31 (positive, Attention2D 8×4) | GPU 32-63 (negative, Attention2D 8×4) - - -## WAN (Image-to-Video) - -```bash -python visual_gen_wan_i2v.py \ - --model_path Wan-AI/Wan2.1-I2V-14B-480P-Diffusers \ - --image_path input_image.jpg \ - --prompt "She turns around and smiles" \ - --height 480 --width 832 --num_frames 81 \ - --output_path output_i2v.mp4 -``` - - -## LTX2 (Text/Image-to-Video with Audio) - -LTX2 generates video **with audio** from text prompts or input images. -It uses a Gemma3 text encoder (provided separately via `--text_encoder_path`) -and supports BF16, FP8, and FP4 precision checkpoints. - -Please refer to tensorrt_llm/_torch/visual_gen/models/ltx2/LTX_2_CHECKPOINT_FORMAT.md for model checkpoint info. - -### Basic Usage - -**Text-to-Video (single GPU):** -```bash -python visual_gen_ltx2.py \ - --model_path ${MODEL_ROOT}/LTX-2-checkpoint/ \ - --text_encoder_path ${MODEL_ROOT}/gemma-3-12b-it \ - --prompt "A cute cat playing piano" \ - --height 720 --width 1280 --num_frames 121 \ - --steps 40 --guidance_scale 4.0 --seed 42 \ - --output_path output_t2v.mp4 -``` - -**Image-to-Video:** -```bash -python visual_gen_ltx2.py \ - --model_path ${MODEL_ROOT}/LTX-2-checkpoint/ \ - --text_encoder_path ${MODEL_ROOT}/gemma-3-12b-it \ - --prompt "A cute cat playing piano" \ - --image ${PROJECT_ROOT}/examples/visual_gen/cat_piano.png \ - --image_cond_strength 1.0 \ - --height 720 --width 1280 --num_frames 121 \ - --steps 40 --seed 42 \ - --output_path output_i2v.mp4 -``` - -### Precision Variants - -LTX2 ships checkpoints at three precision levels. Simply point `--model_path` at the -appropriate directory: - -```bash -# FP8 -python visual_gen_ltx2.py \ - --model_path ${MODEL_ROOT}/LTX-2-checkpoint/fp8/ \ - --text_encoder_path ${MODEL_ROOT}/gemma-3-12b-it \ - --prompt "A cute cat playing piano" \ - --height 720 --width 1280 --num_frames 121 \ - --output_path output_fp8.mp4 - -# FP4 -python visual_gen_ltx2.py \ - --model_path ${MODEL_ROOT}/LTX-2-checkpoint/fp4/ \ - --text_encoder_path ${MODEL_ROOT}/gemma-3-12b-it \ - --prompt "A cute cat playing piano" \ - --height 512 --width 768 --num_frames 121 \ - --output_path output_fp4.mp4 -``` - ---- - -## Common Arguments - -| Argument | FLUX | WAN | LTX2 | Default | Description | -|----------|------|-----|------|---------|-------------| -| `--model_path` | ✓ | ✓ | — | Path to model checkpoint directory | -| `--text_encoder_path` | — | ✓ | — | Path to Gemma3 text encoder | -| `--prompt` | ✓ | ✓ | — | Text prompt for generation | -| `--negative_prompt` | — | ✓ | *(built-in)* | Negative prompt | -| `--height` | ✓ | ✓ | ✓ | 1024 / 720 | Output height | -| `--width` | ✓ | ✓ | ✓ | 1024 / 1280 | Output width | -| `--num_frames` | — | ✓ | ✓ | 81 / 121 | Number of frames | -| `--frame_rate` | — | ✓ | 24.0 | Output frame rate (fps) | -| `--steps` | ✓ | ✓ | ✓ | 50 / 40 | Denoising steps | -| `--guidance_scale` | ✓ | ✓ | ✓ | 3.5 / 5.0 / 4.0 | Guidance strength | -| `--seed` | ✓ | ✓ | ✓ | 42 | Random seed | -| `--image` | — | ✓ | None | Input image for image-to-video | -| `--image_cond_strength` | — | ✓ | 1.0 | Image conditioning strength | -| `--enable_teacache` | ✓ | ✓ | — | False | Cache optimization | -| `--teacache_thresh` | ✓ | ✓ | — | 0.2 | TeaCache similarity threshold | -| `--attention_backend` | ✓ | ✓ | — | VANILLA | `VANILLA`, `TRTLLM`, or `FA4` | -| `--enable_sage_attention` | ✓ | ✓ | — | False | SageAttention (requires `TRTLLM` attention backend) | -| `--cfg_size` | — | ✓ | — | 1 | CFG parallelism | -| `--ulysses_size` | ✓ | ✓ | — | 1 | Ulysses parallelism | -| `--parallel_vae_size` | - | ✓ | — | 1 | Parallelism used for VAE | -| `--attn2d_row_size` | ✓ | ✓ | ✓ | 1 | Attention2D mesh row size | -| `--attn2d_col_size` | ✓ | ✓ | ✓ | 1 | Attention2D mesh column size | -| `--tp_size` | ✓ | ✓ | ✓ | 1 | Tensor parallelism | -| `--linear_type` | ✓ | ✓ | — | default | Quantization type | -| `--enhance_prompt` | — | ✓ | False | Gemma3 prompt enhancement | -| `--stg_scale` | — | ✓ | 0.0 | Spatiotemporal guidance scale | -| `--modality_scale` | — | ✓ | 1.0 | Cross-modal guidance scale | -| `--rescale_scale` | — | ✓ | 0.0 | Variance-preserving rescale factor | - -## Troubleshooting - -**Out of Memory:** -- Use quantization: `--linear_type trtllm-fp8-blockwise` (WAN) or `--linear_type trtllm-fp8-per-tensor` (FLUX) -- Reduce resolution or frames -- Enable TeaCache: `--enable_teacache` -- Use Ulysses parallelism with more GPUs - -**Slow Inference:** -- Enable TeaCache: `--enable_teacache` -- Use TRTLLM backend: `--attention_backend TRTLLM` -- Use multi-GPU: `--cfg_size 2` or `--ulysses_size 2` - -**Import Errors:** -- Run from repository root -- Install necessary dependencies, e.g., `pip install -r requirements-dev.txt` - -**Ulysses Errors:** -- `ulysses_size` must divide the model's head count (12 for WAN); if your GPU - count does not divide the head count, use Attention2D instead - (`--attention_backend FA4 --attn2d_row_size --attn2d_col_size `) -- Total GPUs = `cfg_size × ulysses_size` -- Sequence length must be divisible by `ulysses_size` - -**Attention2D Errors:** -- Requires `--attention_backend FA4` -- Combining with `--ulysses_size` is not yet supported -- Total GPUs = `cfg_size × attn2d_row_size × attn2d_col_size` -- Sequence length must be divisible by `attn2d_row_size × attn2d_col_size` - -**TP Errors:** -- `tp_size × ulysses_size` must divide the model's head count (12 for WAN) -- Total GPUs = `cfg_size × ulysses_size × tp_size` - -## Output Formats - -- **FLUX**: `.png` (image) -- **WAN**: `.mp4` if FFmpeg is installed, otherwise `.avi` (video) -- **LTX2**: `.mp4` (video with audio) if FFmpeg is installed, otherwise `.avi` (video) -## Serving +Install deps from the repo root: `pip install -r requirements-dev.txt`. -See [`serve/README.md`](serve/README.md) for `trtllm-serve` examples including image generation (FLUX), video generation (WAN T2V/I2V), and API endpoint reference. +Output: `.png` for image models; `.mp4` for video models when FFmpeg is installed (otherwise `.avi`). diff --git a/examples/visual_gen/models/wan_t2v.py b/examples/visual_gen/models/wan_t2v.py index 8f892d5127a4..dd829e014aea 100644 --- a/examples/visual_gen/models/wan_t2v.py +++ b/examples/visual_gen/models/wan_t2v.py @@ -54,11 +54,14 @@ def main(): visual_gen = VisualGen(model=args.model, args=extra_args) # --- Model-specific: T2V request construction --- - # Query per-model defaults (resolution, steps, guidance, seed, etc.). + # Start from per-model defaults (steps, guidance, seed, etc.) and override resolution/frames. params = visual_gen.default_params + params.height = 480 + params.width = 832 + params.num_frames = 165 output = visual_gen.generate( - inputs="A cat playing piano in a sunny room", + inputs="A cute cat playing piano", params=params, ) diff --git a/examples/visual_gen/visual_gen_flux.py b/examples/visual_gen/visual_gen_flux.py deleted file mode 100755 index f931292210b0..000000000000 --- a/examples/visual_gen/visual_gen_flux.py +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FLUX Text-to-Image generation using TensorRT-LLM Visual Generation. - -Supports both FLUX.1 and FLUX.2 models. The pipeline type is auto-detected -from the model checkpoint (model_index.json). - -Single image mode: - python visual_gen_flux.py --model_path black-forest-labs/FLUX.1-dev \ - --prompt "A cat sitting on a windowsill" --guidance_scale 3.5 - - python visual_gen_flux.py --model_path black-forest-labs/FLUX.2-dev \ - --prompt "A cat sitting on a windowsill" --guidance_scale 4.0 - - # With FP8 quantization - python visual_gen_flux.py --model_path black-forest-labs/FLUX.2-dev \ - --prompt "A cat" --linear_type trtllm-fp8-per-tensor - -Batch mode (generates multiple images from a prompts file): - python visual_gen_flux.py --model_path black-forest-labs/FLUX.1-dev \ - --prompts_file prompts.txt --output_dir results/bf16/ --seed 42 - - # With FP8 quantization - python visual_gen_flux.py --model_path black-forest-labs/FLUX.2-dev \ - --prompts_file prompts.txt --output_dir results/fp8/ \ - --linear_type trtllm-fp8-per-tensor - - # Multi-GPU with CFG + Ulysses parallelism - python visual_gen_flux.py --model_path black-forest-labs/FLUX.1-dev \ - --prompts_file prompts.txt --output_dir results/ \ - --cfg_size 2 --ulysses_size 2 -""" - -import argparse -import json -import os -import time - -from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams, logger -from tensorrt_llm.visual_gen.args import CacheDiTConfig, TeaCacheConfig - -logger.set_level("info") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="TRTLLM VisualGen - FLUX Text-to-Image Inference Example (FLUX.1 / FLUX.2)" - ) - - # Model & Input - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Local path or HuggingFace Hub model ID " - "(e.g., black-forest-labs/FLUX.1-dev, black-forest-labs/FLUX.2-dev)", - ) - parser.add_argument( - "--revision", - type=str, - default=None, - help="HuggingFace Hub revision (branch, tag, or commit SHA)", - ) - - # Single image mode - parser.add_argument( - "--prompt", type=str, default=None, help="Text prompt for single image generation" - ) - parser.add_argument( - "--output_path", - type=str, - default="output.png", - help="Path to save the output image (single image mode)", - ) - - # Batch mode - parser.add_argument( - "--prompts_file", - type=str, - default=None, - help="File with prompts (one per line) for batch generation", - ) - parser.add_argument( - "--output_dir", - type=str, - default=None, - help="Output directory for batch mode (images named 00.png, 01.png, ...)", - ) - parser.add_argument( - "--num_prompts", - type=int, - default=None, - help="Limit number of prompts from file (batch mode)", - ) - - # Generation Params - parser.add_argument("--height", type=int, default=1024, help="Image height") - parser.add_argument("--width", type=int, default=1024, help="Image width") - parser.add_argument("--steps", type=int, default=50, help="Number of denoising steps") - parser.add_argument( - "--guidance_scale", - type=float, - default=3.5, - help="Embedded guidance scale (3.5 for FLUX.1-dev, 4.0 for FLUX.2-dev)", - ) - parser.add_argument("--seed", type=int, default=42, help="Random seed") - - # Diffusion cache acceleration (TeaCache and Cache-DiT; mutually exclusive) - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument( - "--enable_teacache", action="store_true", help="Enable TeaCache acceleration" - ) - cache_group.add_argument( - "--enable_cache_dit", - action="store_true", - help=( - "Enable Cache-DiT per-block acceleration (requires the cache_dit package; " - "see https://github.com/vipshop/cache-dit). Incompatible with --enable_teacache." - ), - ) - parser.add_argument( - "--teacache_thresh", - type=float, - default=None, - help="TeaCache similarity threshold (default: 0.6 for FLUX.1, 0.2 for FLUX.2); " - "ignored when using --enable_cache_dit", - ) - parser.add_argument( - "--use_ret_steps", - action="store_true", - help="Use ret_steps mode for TeaCache. " - "Using Retention Steps will result in faster generation speed and better generation quality. " - "Ignored when using --enable_cache_dit.", - ) - - # Cache-DiT overrides (only apply with --enable_cache_dit; omitted fields use CacheDiTConfig defaults) - parser.add_argument( - "--cache_dit_fn_compute_blocks", - type=int, - default=None, - help="DBCache Fn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_bn_compute_blocks", - type=int, - default=None, - help="DBCache Bn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_warmup_steps", - type=int, - default=None, - help="DBCache max_warmup_steps (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_cached_steps", - type=int, - default=None, - help="DBCache max_cached_steps (-1 = no cap; default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_residual_threshold", - type=float, - default=None, - help="DBCache residual_diff_threshold (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_enable_taylorseer", - action="store_true", - help="Enable TaylorSeer calibrator (default: off).", - ) - parser.add_argument( - "--cache_dit_taylorseer_order", - type=int, - default=None, - choices=[1, 2, 3, 4], - help="TaylorSeer order; implies TaylorSeer on if set. Default order from CacheDiTConfig.", - ) - parser.add_argument( - "--cache_dit_scm_mask_policy", - type=str, - default=None, - help="SCM steps_mask policy name (e.g. fast, medium, slow, ultra). Omit to disable SCM.", - ) - parser.add_argument( - "--cache_dit_scm_steps_policy", - type=str, - default=None, - choices=["dynamic", "static"], - help="SCM steps_computation_policy (default: dynamic if not overridden).", - ) - - # Quantization - parser.add_argument( - "--linear_type", - type=str, - default="default", - choices=["default", "trtllm-fp8-per-tensor", "trtllm-fp8-blockwise", "trtllm-nvfp4"], - help=( - "Dynamic quantization mode for linear layers. " - "Quantizes weights on-the-fly during loading from an unquantized checkpoint." - ), - ) - - # Attention Backend - parser.add_argument( - "--attention_backend", - type=str, - default="VANILLA", - choices=["VANILLA", "TRTLLM", "FA4"], - help="Attention backend (VANILLA: PyTorch SDPA, TRTLLM: optimized kernels, " - "FA4: Flash Attention 4). " - "Note: TRTLLM falls back to VANILLA for cross-attention.", - ) - parser.add_argument( - "--enable_sage_attention", - action="store_true", - help="Enable SageAttention (per-block quantized Q/K/V). Requires TRTLLM backend.", - ) - - # Parallelism - parser.add_argument( - "--ulysses_size", - type=int, - default=1, - help="Ulysses (head-sharding) parallel size within each CFG group. " - "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", - ) - parser.add_argument( - "--tp_size", - type=int, - default=1, - help="Tensor Parallel size", - ) - parser.add_argument( - "--attn2d_row_size", - type=int, - default=1, - help="Attention2D row mesh size (Q all-gather dimension). " - "Can be set independently of --attn2d_col_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Total context parallelism degree = attn2d_row_size * attn2d_col_size. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_col_size", - type=int, - default=1, - help="Attention2D column mesh size (K/V all-gather dimension). " - "Can be set independently of --attn2d_row_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - - # CUDA graph - parser.add_argument( - "--enable_cudagraph", action="store_true", help="Enable CudaGraph acceleration" - ) - - # torch.compile - parser.add_argument( - "--disable_torch_compile", action="store_true", help="Disable TorchCompile acceleration" - ) - parser.add_argument( - "--enable_fullgraph", action="store_true", help="Enable fullgraph for TorchCompile" - ) - - # Autotune - parser.add_argument( - "--disable_autotune", action="store_true", help="Disable autotuning during warmup" - ) - - # Debug / profiling - parser.add_argument( - "--enable_layerwise_nvtx_marker", action="store_true", help="Enable layerwise NVTX markers" - ) - - args = parser.parse_args() - - if args.prompt is None and args.prompts_file is None: - parser.error("Either --prompt or --prompts_file is required") - if args.prompt is not None and args.prompts_file is not None: - parser.error("--prompt and --prompts_file are mutually exclusive") - if args.prompts_file is not None and args.output_dir is None: - parser.error("--output_dir is required when using --prompts_file") - - return args - - -def load_prompts(prompts_file, num_prompts=None): - """Load prompts from file (one per line, skip empty/comments).""" - with open(prompts_file) as f: - prompts = [line.strip() for line in f if line.strip() and not line.startswith("#")] - if num_prompts is not None: - prompts = prompts[:num_prompts] - return prompts - - -def _linear_type_to_quant_config(linear_type: str): - """Map --linear_type CLI shortcut to quant_config dict for VisualGenArgs.""" - mapping = { - "trtllm-fp8-per-tensor": {"quant_algo": "FP8", "dynamic": True}, - "trtllm-fp8-blockwise": {"quant_algo": "FP8_BLOCK_SCALES", "dynamic": True}, - "trtllm-nvfp4": {"quant_algo": "NVFP4", "dynamic": True}, - } - return mapping.get(linear_type) - - -def _teacache_config_from_args(args) -> TeaCacheConfig: - """Build TeaCacheConfig from CLI args; unset options keep Pydantic defaults.""" - kwargs: dict = {"use_ret_steps": args.use_ret_steps} - if args.teacache_thresh is not None: - kwargs["teacache_thresh"] = args.teacache_thresh - return TeaCacheConfig(**kwargs) - - -def _cache_dit_config_from_args(args) -> CacheDiTConfig: - """Subset of CacheDiTConfig from CLI; unset options keep Pydantic defaults.""" - overrides: dict = {} - if args.cache_dit_fn_compute_blocks is not None: - overrides["Fn_compute_blocks"] = args.cache_dit_fn_compute_blocks - if args.cache_dit_bn_compute_blocks is not None: - overrides["Bn_compute_blocks"] = args.cache_dit_bn_compute_blocks - if args.cache_dit_max_warmup_steps is not None: - overrides["max_warmup_steps"] = args.cache_dit_max_warmup_steps - if args.cache_dit_max_cached_steps is not None: - overrides["max_cached_steps"] = args.cache_dit_max_cached_steps - if args.cache_dit_residual_threshold is not None: - overrides["residual_diff_threshold"] = args.cache_dit_residual_threshold - if args.cache_dit_enable_taylorseer or args.cache_dit_taylorseer_order is not None: - overrides["enable_taylorseer"] = True - if args.cache_dit_taylorseer_order is not None: - overrides["taylorseer_order"] = args.cache_dit_taylorseer_order - if args.cache_dit_scm_mask_policy is not None: - overrides["scm_steps_mask_policy"] = args.cache_dit_scm_mask_policy - if args.cache_dit_scm_steps_policy is not None: - overrides["scm_steps_policy"] = args.cache_dit_scm_steps_policy - return CacheDiTConfig(**overrides) - - -def build_visual_gen_args(args) -> VisualGenArgs: - """Build VisualGenArgs from parsed CLI args.""" - if args.enable_cache_dit: - cache_kwargs = {"cache_config": _cache_dit_config_from_args(args)} - elif args.enable_teacache: - cache_kwargs = {"cache_config": _teacache_config_from_args(args)} - else: - cache_kwargs = {} - - attention_cfg: dict = {"backend": args.attention_backend} - if args.enable_sage_attention: - attention_cfg["quant_attention_config"] = { - "qk_dtype": "int8", - "q_block_size": 1, - "k_block_size": 16, - "v_block_size": 1, - } - logger.info("SageAttention: INT8 Q/K, blocks (1, 16, 1)") - - kwargs = dict( - revision=args.revision, - attention_config=attention_cfg, - **cache_kwargs, - parallel_config={ - "ulysses_size": args.ulysses_size, - "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), - "tp_size": args.tp_size, - }, - torch_compile_config={ - "enable": not args.disable_torch_compile, - "enable_fullgraph": args.enable_fullgraph, - "enable_autotune": not args.disable_autotune, - }, - cuda_graph_config={"enable": args.enable_cudagraph}, - enable_layerwise_nvtx_marker=args.enable_layerwise_nvtx_marker, - ) - quant_config = _linear_type_to_quant_config(args.linear_type) - if quant_config is not None: - kwargs["quant_config"] = quant_config - return VisualGenArgs(**kwargs) - - -def main(): - args = parse_args() - - attn2d_size = args.attn2d_row_size * args.attn2d_col_size - if attn2d_size > 1 and args.ulysses_size > 1: - raise ValueError( - "Combining --ulysses_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - - visual_gen_args = build_visual_gen_args(args) - - parallel_str = "" - if args.tp_size > 1: - parallel_str = f"TP(size={args.tp_size})" - if args.ulysses_size > 1: - parallel_str += f"Ulysses(size={args.ulysses_size})" - elif attn2d_size > 1: - parallel_str += ( - f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " - f"total={attn2d_size})" - ) - elif not parallel_str: - parallel_str = "None" - logger.info(f"Initializing VisualGen: parallelism={parallel_str}") - visual_gen = VisualGen( - model=args.model_path, - args=visual_gen_args, - ) - - try: - if args.prompts_file: - prompts = load_prompts(args.prompts_file, args.num_prompts) - os.makedirs(args.output_dir, exist_ok=True) - - logger.info(f"Batch mode: {len(prompts)} prompts -> {args.output_dir}") - logger.info(f"Resolution: {args.height}x{args.width}, Steps: {args.steps}") - - timing_records = [] - total_start = time.time() - - for i, prompt in enumerate(prompts): - logger.info(f"[{i + 1}/{len(prompts)}] {prompt[:60]}...") - start_time = time.time() - - output = visual_gen.generate( - inputs=prompt, - params=VisualGenParams( - height=args.height, - width=args.width, - num_inference_steps=args.steps, - guidance_scale=args.guidance_scale, - seed=args.seed + i, - ), - ) - - elapsed = time.time() - start_time - output_path = os.path.join(args.output_dir, f"{i:02d}.png") - output.save(output_path) - logger.info(f" Saved {output_path} ({elapsed:.1f}s)") - - timing_records.append( - { - "index": i, - "prompt": prompt, - "time": round(elapsed, 2), - "seed": args.seed + i, - } - ) - - total_elapsed = time.time() - total_start - times = [r["time"] for r in timing_records] - - timing_data = { - "images": timing_records, - "total_time": round(total_elapsed, 2), - "avg_time": round(sum(times) / len(times), 2) if times else 0, - "config": { - "model_path": args.model_path, - "linear_type": args.linear_type, - "attention_backend": args.attention_backend, - "height": args.height, - "width": args.width, - "steps": args.steps, - "guidance_scale": args.guidance_scale, - }, - } - timing_path = os.path.join(args.output_dir, "timing.json") - with open(timing_path, "w") as f: - json.dump(timing_data, f, indent=2) - - logger.info( - f"Batch complete: {len(prompts)} images in {total_elapsed:.1f}s " - f"(avg {timing_data['avg_time']:.1f}s/image)" - ) - logger.info(f"Timing saved to {timing_path}") - - else: - logger.info(f"Generating image for prompt: '{args.prompt}'") - logger.info(f"Resolution: {args.height}x{args.width}, Steps: {args.steps}") - - start_time = time.time() - - output = visual_gen.generate( - inputs=args.prompt, - params=VisualGenParams( - height=args.height, - width=args.width, - num_inference_steps=args.steps, - guidance_scale=args.guidance_scale, - seed=args.seed, - ), - ) - - logger.info(f"Generation completed in {time.time() - start_time:.2f}s") - - output.save(args.output_path) - - finally: - visual_gen.shutdown() - - -if __name__ == "__main__": - main() diff --git a/examples/visual_gen/visual_gen_ltx2.py b/examples/visual_gen/visual_gen_ltx2.py deleted file mode 100755 index cc1e8fe4f509..000000000000 --- a/examples/visual_gen/visual_gen_ltx2.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""LTX2 Text/Image-to-Video generation using TensorRT-LLM Visual Generation.""" - -import argparse -import time - -from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams, logger -from tensorrt_llm.visual_gen.args import CacheDiTConfig - -logger.set_level("info") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="TRTLLM VisualGen - LTX2 Text-to-Video with Audio Inference Example" - ) - - # Model & Input - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Path to the LTX2 checkpoint (directory containing .safetensors)", - ) - parser.add_argument( - "--text_encoder_path", - type=str, - required=True, - help="Path to the Gemma3 text encoder model directory", - ) - parser.add_argument("--prompt", type=str, required=True, help="Text prompt for generation") - parser.add_argument( - "--negative_prompt", - type=str, - default="worst quality, inconsistent motion, blurry, jittery, distorted", - help="Negative prompt to guide generation away from undesired content", - ) - parser.add_argument( - "--output_path", - type=str, - default="output.mp4", - help="Path to save the output video with audio (supports .mp4, .gif, .png)", - ) - - # Image-to-video conditioning - parser.add_argument( - "--image", - type=str, - default=None, - help="Path to input image for image-to-video conditioning", - ) - parser.add_argument( - "--image_cond_strength", - type=float, - default=1.0, - help="Conditioning strength for the input image (0.0 to 1.0, default: 1.0)", - ) - - # Generation Params - parser.add_argument("--height", type=int, default=512, help="Video height (divisible by 32)") - parser.add_argument("--width", type=int, default=768, help="Video width (divisible by 32)") - parser.add_argument( - "--num_frames", "--num-frames", type=int, default=121, help="Number of frames to generate" - ) - parser.add_argument( - "--frame_rate", type=float, default=24.0, help="Frames per second for the video" - ) - parser.add_argument( - "--steps", - "--num_inference_steps", - type=int, - default=40, - help="Number of denoising steps", - ) - parser.add_argument( - "--guidance_scale", - type=float, - default=4.0, - help="Classifier-free guidance scale", - ) - parser.add_argument( - "--guidance_rescale", - type=float, - default=0.0, - help="Guidance rescale factor to fix overexposure", - ) - parser.add_argument("--seed", type=int, default=42, help="Random seed") - parser.add_argument( - "--max_sequence_length", - type=int, - default=1024, - help="Maximum sequence length for prompt encoding", - ) - - # Multi-modal guidance (STG / modality) - parser.add_argument( - "--stg_scale", - type=float, - default=0.0, - help="Spatiotemporal guidance scale (0=disabled). Reference default: 1.0", - ) - parser.add_argument( - "--stg_blocks", - type=int, - nargs="*", - default=None, - help="Transformer block indices for STG perturbation (e.g., 29). Reference default: [29]", - ) - parser.add_argument( - "--modality_scale", - type=float, - default=1.0, - help="Cross-modal guidance scale (1=disabled). Reference default: 3.0", - ) - parser.add_argument( - "--rescale_scale", - type=float, - default=0.0, - help="Variance-preserving rescale factor (0=disabled). Reference default: 0.7", - ) - parser.add_argument( - "--guidance_skip_step", - type=int, - default=0, - help="Skip guidance every N+1 steps (0=never skip)", - ) - parser.add_argument( - "--enhance_prompt", - action="store_true", - help="Use Gemma3 to enhance the text prompt before encoding", - ) - - # Diffusion cache acceleration - parser.add_argument( - "--enable_cache_dit", - action="store_true", - help="Enable Cache-DiT per-block acceleration.", - ) - # Cache-DiT overrides (only with --enable_cache_dit; omitted fields use CacheDiTConfig defaults) - parser.add_argument( - "--cache_dit_fn_compute_blocks", - type=int, - default=None, - help="DBCache Fn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_bn_compute_blocks", - type=int, - default=None, - help="DBCache Bn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_warmup_steps", - type=int, - default=None, - help="DBCache max_warmup_steps (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_cached_steps", - type=int, - default=None, - help="DBCache max_cached_steps (-1 = no cap; default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_residual_threshold", - type=float, - default=0.16, - help=( - "DBCache residual_diff_threshold (LTX2 default 0.16; global CacheDiTConfig default is 0.24)." - ), - ) - parser.add_argument( - "--cache_dit_enable_taylorseer", - action="store_true", - help="Enable TaylorSeer calibrator (default: off).", - ) - parser.add_argument( - "--cache_dit_taylorseer_order", - type=int, - default=None, - choices=[1, 2, 3, 4], - help="TaylorSeer order; implies TaylorSeer on if set.", - ) - parser.add_argument( - "--cache_dit_scm_mask_policy", - type=str, - default=None, - help="SCM steps_mask policy (e.g. fast, medium, slow, ultra). Omit to disable SCM.", - ) - parser.add_argument( - "--cache_dit_scm_steps_policy", - type=str, - default=None, - choices=["dynamic", "static"], - help="SCM steps_computation_policy (default: dynamic if not overridden).", - ) - - # Two-stage pipeline - parser.add_argument( - "--spatial_upsampler_path", - type=str, - default="", - help=( - "Optional path to the learned LatentUpsampler checkpoint (.safetensors). " - "If omitted, VisualGen tries to discover it next to the model checkpoint. " - "When available, the pipeline uses two-stage generation: stage 1 at half " - "resolution, learned 2x upsample, stage 2 refinement." - ), - ) - parser.add_argument( - "--distilled_lora_path", - type=str, - default="", - help=( - "Optional path to the distilled LoRA checkpoint (.safetensors) for " - "stage 2 refinement. If omitted, VisualGen tries to discover it next " - "to the model checkpoint. The LoRA weights are merged into the " - "transformer for stage 2 and un-merged afterwards." - ), - ) - - # Parallelism - parser.add_argument( - "--cfg_size", - type=int, - default=1, - choices=[1, 2], - help="CFG parallel size (1 or 2). Set to 2 for CFG Parallelism.", - ) - parser.add_argument( - "--ulysses_size", - type=int, - default=1, - help="Ulysses (head-sharding) parallel size within each CFG group. " - "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", - ) - parser.add_argument( - "--tp_size", - type=int, - default=1, - help="Tensor Parallel size", - ) - parser.add_argument( - "--attn2d_row_size", - type=int, - default=1, - help="Attention2D row mesh size (Q all-gather dimension). " - "Can be set independently of --attn2d_col_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Total context parallelism degree = attn2d_row_size * attn2d_col_size. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_col_size", - type=int, - default=1, - help="Attention2D column mesh size (K/V all-gather dimension). " - "Can be set independently of --attn2d_row_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - - # CUDA graph - parser.add_argument( - "--enable_cudagraph", action="store_true", help="Enable CudaGraph acceleration" - ) - - # torch.compile - parser.add_argument( - "--disable_torch_compile", action="store_true", help="Disable TorchCompile acceleration" - ) - parser.add_argument( - "--enable_fullgraph", action="store_true", help="Enable fullgraph for TorchCompile" - ) - - # Autotune - parser.add_argument( - "--disable_autotune", action="store_true", help="Disable autotuning during warmup" - ) - - # Debug / profiling - parser.add_argument( - "--enable_layerwise_nvtx_marker", action="store_true", help="Enable layerwise NVTX markers" - ) - - # Dynamic quantization - parser.add_argument( - "--linear_type", - type=str, - default="default", - choices=["default", "trtllm-fp8-per-tensor", "trtllm-fp8-blockwise", "trtllm-nvfp4"], - help=( - "Dynamic quantization mode for linear layers. " - "Quantizes weights on-the-fly during loading from an unquantized checkpoint." - ), - ) - - # Attention Backend - parser.add_argument( - "--attention_backend", - type=str, - default="VANILLA", - choices=["VANILLA", "TRTLLM"], - help="Attention backend (VANILLA: PyTorch SDPA, TRTLLM: optimized kernels). " - "Note: TRTLLM automatically falls back to VANILLA for cross-attention.", - ) - return parser.parse_args() - - -def _linear_type_to_quant_config(linear_type: str): - """Map --linear_type CLI shortcut to quant_config dict for VisualGenArgs.""" - mapping = { - "trtllm-fp8-per-tensor": {"quant_algo": "FP8", "dynamic": True}, - "trtllm-fp8-blockwise": {"quant_algo": "FP8_BLOCK_SCALES", "dynamic": True}, - "trtllm-nvfp4": {"quant_algo": "NVFP4", "dynamic": True}, - } - return mapping.get(linear_type) - - -def _cache_dit_config_from_args(args) -> CacheDiTConfig: - """Subset of CacheDiTConfig from CLI; unset options keep Pydantic defaults.""" - overrides: dict = {} - if args.cache_dit_fn_compute_blocks is not None: - overrides["Fn_compute_blocks"] = args.cache_dit_fn_compute_blocks - if args.cache_dit_bn_compute_blocks is not None: - overrides["Bn_compute_blocks"] = args.cache_dit_bn_compute_blocks - if args.cache_dit_max_warmup_steps is not None: - overrides["max_warmup_steps"] = args.cache_dit_max_warmup_steps - if args.cache_dit_max_cached_steps is not None: - overrides["max_cached_steps"] = args.cache_dit_max_cached_steps - overrides["residual_diff_threshold"] = args.cache_dit_residual_threshold - if args.cache_dit_enable_taylorseer or args.cache_dit_taylorseer_order is not None: - overrides["enable_taylorseer"] = True - if args.cache_dit_taylorseer_order is not None: - overrides["taylorseer_order"] = args.cache_dit_taylorseer_order - if args.cache_dit_scm_mask_policy is not None: - overrides["scm_steps_mask_policy"] = args.cache_dit_scm_mask_policy - if args.cache_dit_scm_steps_policy is not None: - overrides["scm_steps_policy"] = args.cache_dit_scm_steps_policy - return CacheDiTConfig(**overrides) - - -def _build_visual_gen_args(args) -> VisualGenArgs: - """Build VisualGenArgs from parsed CLI args.""" - if args.enable_cache_dit: - logger.info("Cache DiT enabled; LTX2 will run as a one-stage pipeline.") - cache_kwargs = {"cache_config": _cache_dit_config_from_args(args)} - else: - cache_kwargs = {} - - attention_cfg: dict = {"backend": args.attention_backend} - - pipeline_config = {"text_encoder_path": args.text_encoder_path} - if args.spatial_upsampler_path: - pipeline_config["spatial_upsampler_path"] = args.spatial_upsampler_path - if args.distilled_lora_path: - pipeline_config["distilled_lora_path"] = args.distilled_lora_path - - kwargs = dict( - pipeline_config=pipeline_config, - **cache_kwargs, - attention_config=attention_cfg, - parallel_config={ - "cfg_size": args.cfg_size, - "ulysses_size": args.ulysses_size, - "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), - "tp_size": args.tp_size, - }, - torch_compile_config={ - "enable": not args.disable_torch_compile, - "enable_fullgraph": args.enable_fullgraph, - "enable_autotune": not args.disable_autotune, - }, - cuda_graph_config={"enable": args.enable_cudagraph}, - enable_layerwise_nvtx_marker=args.enable_layerwise_nvtx_marker, - ) - quant_config = _linear_type_to_quant_config(args.linear_type) - if quant_config is not None: - kwargs["quant_config"] = quant_config - return VisualGenArgs(**kwargs) - - -def main(): - args = parse_args() - - attn2d_size = args.attn2d_row_size * args.attn2d_col_size - if attn2d_size > 1 and args.ulysses_size > 1: - raise ValueError( - "Combining --ulysses_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - - if args.tp_size > 1: - raise ValueError("LTX2 does not currently support TP.") - - visual_gen_args = _build_visual_gen_args(args) - - parallel_str = "" - if args.tp_size > 1: - parallel_str = f"TP(size={args.tp_size})" - if args.ulysses_size > 1: - parallel_str += f"Ulysses(size={args.ulysses_size})" - elif attn2d_size > 1: - parallel_str += ( - f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " - f"total={attn2d_size})" - ) - elif not parallel_str: - parallel_str = "None" - logger.info( - f"Initializing VisualGen (LTX2): cfg_size={args.cfg_size}, parallelism={parallel_str}" - ) - visual_gen = VisualGen( - model=args.model_path, - args=visual_gen_args, - ) - - try: - # Run Inference - logger.info(f"Generating video with audio for prompt: '{args.prompt}'") - logger.info( - f"Resolution: {args.height}x{args.width}, " - f"Frames: {args.num_frames}, " - f"FPS: {args.frame_rate}, " - f"Steps: {args.steps}" - ) - - start_time = time.time() - - extra_params = { - "guidance_rescale": args.guidance_rescale, - "stg_scale": args.stg_scale, - "modality_scale": args.modality_scale, - "rescale_scale": args.rescale_scale, - "guidance_skip_step": args.guidance_skip_step, - "enhance_prompt": args.enhance_prompt, - } - if args.stg_blocks is not None: - extra_params["stg_blocks"] = args.stg_blocks - - params = VisualGenParams( - height=args.height, - width=args.width, - num_inference_steps=args.steps, - guidance_scale=args.guidance_scale, - max_sequence_length=args.max_sequence_length, - seed=args.seed, - num_frames=args.num_frames, - frame_rate=args.frame_rate, - negative_prompt=args.negative_prompt, - image=args.image, - image_cond_strength=args.image_cond_strength, - extra_params=extra_params, - ) - - output = visual_gen.generate(inputs=args.prompt, params=params) - - end_time = time.time() - logger.info(f"Generation completed in {end_time - start_time:.2f}s") - - output.save(args.output_path) - - finally: - # Shutdown - visual_gen.shutdown() - - -if __name__ == "__main__": - main() diff --git a/examples/visual_gen/visual_gen_mgmn_distributed.sh b/examples/visual_gen/visual_gen_mgmn_distributed.sh deleted file mode 100644 index 8e44cb949a39..000000000000 --- a/examples/visual_gen/visual_gen_mgmn_distributed.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -#SBATCH -A # parameter -#SBATCH -p # parameter -#SBATCH -t 01:00:00 -#SBATCH -N 2 -#SBATCH --ntasks-per-node=4 -#SBATCH --gpus-per-node=4 -#SBATCH -o visual-gen-distributed.out -#SBATCH -e visual-gen-distributed.err -#SBATCH -J visual-gen-distributed - -############################################################################## -# OVERVIEW: -# This script demonstrates running visual generation (e.g., Wan T2V) on SLURM -# with distributed inference support across multiple GPUs and nodes using -# Ulysses sequence parallelism. -# -# WHAT TO MODIFY: -# 1. SLURM Parameters (lines above): -# - Replace with your SLURM account name -# - Replace with your SLURM partition name -# - Adjust -N (number of nodes) to match your parallelism config -# - Adjust --ntasks-per-node and --gpus-per-node to match GPUs per node -# - Total GPUs must equal: cfg_size * ulysses_size -# -# 2. Environment Variables (set before running sbatch, or edit defaults below): -# - CONTAINER_IMAGE: Docker image or enroot .sqsh image with TensorRT-LLM installed -# - MOUNT_DIR: host directory to mount into the container (default: $HOME) -# - MOUNT_DEST: mount destination path inside the container (default: $HOME) -# - PROJECT_DIR: path to TensorRT-LLM source on the shared filesystem -# - MODEL_PATH: HuggingFace Hub model ID or local path (default: Wan-AI/Wan2.2-T2V-A14B-Diffusers) -# - OUTPUT_PATH: path to save generated video (default: output.avi) -# - ATTENTION_BACKEND: attention backend to use (default: FA4; options: VANILLA, TRTLLM, FA4) -# - CFG_SIZE: CFG parallel size (1 or 2) -# - ULYSSES_SIZE: Ulysses sequence parallel size -# - MASTER_PORT: NCCL rendezvous port (default: 29500) -# -# EXAMPLE USAGE: -# export CONTAINER_IMAGE="/path/to/tensorrt-llm.sqsh" -# export PROJECT_DIR="/path/to/TensorRT-LLM" -# export MODEL_PATH="Wan-AI/Wan2.2-T2V-A14B-Diffusers" -# export OUTPUT_PATH="output.avi" -# sbatch visual_gen_mgmn_distributed.sh -# -############################################################################## - -# --------------------------------------------------------------------------- -# Configuration — override any of these via environment before calling sbatch -# --------------------------------------------------------------------------- - -PROJECT_DIR="${PROJECT_DIR:-/path/to/TensorRT-LLM}" -CONTAINER_IMAGE="${CONTAINER_IMAGE:-/path/to/tensorrt-llm.sqsh}" -MOUNT_DIR="${MOUNT_DIR:-$HOME}" -MOUNT_DEST="${MOUNT_DEST:-$HOME}" - -MODEL_PATH="${MODEL_PATH:-Wan-AI/Wan2.2-T2V-A14B-Diffusers}" -PROMPT="${PROMPT:-A cat playing piano}" -OUTPUT_PATH="${OUTPUT_PATH:-output.avi}" - -# Generation parameters -HEIGHT="${HEIGHT:-720}" -WIDTH="${WIDTH:-1280}" -NUM_FRAMES="${NUM_FRAMES:-81}" -NUM_STEPS="${NUM_STEPS:-40}" -ATTENTION_BACKEND="${ATTENTION_BACKEND:-FA4}" - -# Parallelism -CFG_SIZE="${CFG_SIZE:-2}" -ULYSSES_SIZE="${ULYSSES_SIZE:-2}" - -# --------------------------------------------------------------------------- -# Derived values — do not edit -# --------------------------------------------------------------------------- - -NUM_GPUS=$(( SLURM_NNODES * SLURM_GPUS_PER_NODE )) - -# Determine rank-0 node hostname for NCCL rendezvous -export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) -export MASTER_PORT="${MASTER_PORT:-29500}" - -# --------------------------------------------------------------------------- -# Validate parallelism config against allocated GPU count -# --------------------------------------------------------------------------- - -EXPECTED_GPUS=$(( CFG_SIZE * ULYSSES_SIZE )) -if [ "${NUM_GPUS}" -ne "${EXPECTED_GPUS}" ]; then - echo "ERROR: NUM_GPUS=${NUM_GPUS} but cfg_size(${CFG_SIZE}) * ulysses_size(${ULYSSES_SIZE}) = ${EXPECTED_GPUS}" >&2 - exit 1 -fi - -# --------------------------------------------------------------------------- -# Build the run command -# --------------------------------------------------------------------------- - -RUN_CMD="python examples/visual_gen/visual_gen_wan_t2v.py \ - --model_path '${MODEL_PATH}' \ - --prompt '${PROMPT}' \ - --height ${HEIGHT} --width ${WIDTH} --num_frames ${NUM_FRAMES} \ - --steps ${NUM_STEPS} \ - --attention_backend ${ATTENTION_BACKEND} \ - --cfg_size ${CFG_SIZE} \ - --ulysses_size ${ULYSSES_SIZE} \ - --output_path '${OUTPUT_PATH}'" - -# --------------------------------------------------------------------------- -# Launch -# --------------------------------------------------------------------------- - -srun -l \ - --export=ALL \ - --container-image "${CONTAINER_IMAGE}" \ - --container-workdir "${PROJECT_DIR}" \ - --container-mounts=${MOUNT_DIR}:${MOUNT_DEST} \ - sh -c "${RUN_CMD}" diff --git a/examples/visual_gen/visual_gen_wan_i2v.py b/examples/visual_gen/visual_gen_wan_i2v.py deleted file mode 100644 index 3be72ed43939..000000000000 --- a/examples/visual_gen/visual_gen_wan_i2v.py +++ /dev/null @@ -1,472 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""WAN Image-to-Video generation using TensorRT-LLM Visual Generation.""" - -import argparse -import time - -from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams, logger -from tensorrt_llm.visual_gen.args import CacheDiTConfig, TeaCacheConfig - -logger.set_level("info") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="TRTLLM VisualGen - Wan Image-to-Video Inference Example (supports Wan 2.1 and Wan 2.2)" - ) - - # Model & Input - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Path to Wan I2V Diffusers model directory (2.1 or 2.2)", - ) - parser.add_argument( - "--image_path", - type=str, - required=True, - help="Path to input image for I2V conditioning", - ) - parser.add_argument( - "--last_image_path", - type=str, - default=None, - help="Optional path to last frame image for interpolation (Wan 2.1 only)", - ) - parser.add_argument("--prompt", type=str, required=True, help="Text prompt for generation") - parser.add_argument( - "--negative_prompt", - type=str, - default=None, - help="Negative prompt. Default is model-specific.", - ) - parser.add_argument( - "--output_path", - type=str, - default="output.png", - help="Path to save the output image/video frame", - ) - - # Generation Params - parser.add_argument( - "--height", type=int, default=None, help="Video height (default: auto-detect)" - ) - parser.add_argument( - "--width", type=int, default=None, help="Video width (default: auto-detect)" - ) - parser.add_argument( - "--num_frames", - type=int, - default=None, - help="Number of frames to generate (default: auto-detect)", - ) - parser.add_argument( - "--steps", - type=int, - default=None, - help="Number of denoising steps (default: auto-detect, 50 for Wan2.1 and Wan 2.2 5B, 40 for Wan2.2 A14B)", - ) - parser.add_argument( - "--guidance_scale", - type=float, - default=None, - help="Guidance scale (default: auto-detect, 5.0 for Wan2.1 and Wan 2.2 5B, 4.0 for Wan2.2 A14B)", - ) - parser.add_argument( - "--guidance_scale_2", - type=float, - default=None, - help="Second-stage guidance scale for Wan2.2 A14B two-stage denoising (default: 3.0)", - ) - parser.add_argument( - "--boundary_ratio", - type=float, - default=None, - help="Custom boundary ratio for two-stage denoising (default: auto-detect)", - ) - parser.add_argument("--seed", type=int, default=42, help="Random seed") - - # Diffusion cache acceleration (TeaCache vs Cache-DiT; mutually exclusive) - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument( - "--enable_teacache", action="store_true", help="Enable TeaCache acceleration" - ) - cache_group.add_argument( - "--enable_cache_dit", - action="store_true", - help=( - "Enable Cache-DiT per-block acceleration (requires the cache_dit package; " - "see https://github.com/vipshop/cache-dit). Incompatible with --enable_teacache." - ), - ) - parser.add_argument( - "--teacache_thresh", - type=float, - default=0.2, - help="TeaCache similarity threshold (rel_l1_thresh); ignored when using --enable_cache_dit", - ) - parser.add_argument( - "--use_ret_steps", - action="store_true", - help="Use ret_steps mode for TeaCache. " - "Using Retention Steps will result in faster generation speed and better generation quality. " - "Ignored when using --enable_cache_dit.", - ) - - # Cache-DiT overrides (only apply with --enable_cache_dit; omitted fields use CacheDiTConfig defaults) - parser.add_argument( - "--cache_dit_fn_compute_blocks", - type=int, - default=None, - help="DBCache Fn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_bn_compute_blocks", - type=int, - default=None, - help="DBCache Bn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_warmup_steps", - type=int, - default=None, - help="DBCache max_warmup_steps (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_cached_steps", - type=int, - default=None, - help="DBCache max_cached_steps (-1 = no cap; default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_residual_threshold", - type=float, - default=None, - help="DBCache residual_diff_threshold (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_enable_taylorseer", - action="store_true", - help="Enable TaylorSeer calibrator (default: off).", - ) - parser.add_argument( - "--cache_dit_taylorseer_order", - type=int, - default=None, - choices=[1, 2, 3, 4], - help="TaylorSeer order; implies TaylorSeer on if set. Default order from CacheDiTConfig.", - ) - parser.add_argument( - "--cache_dit_scm_mask_policy", - type=str, - default=None, - help="SCM steps_mask policy name (e.g. fast, medium, slow, ultra). Omit to disable SCM.", - ) - parser.add_argument( - "--cache_dit_scm_steps_policy", - type=str, - default=None, - choices=["dynamic", "static"], - help="SCM steps_computation_policy (default: dynamic if not overridden).", - ) - - # Quantization - parser.add_argument( - "--linear_type", - type=str, - default="default", - choices=["default", "trtllm-fp8-per-tensor", "trtllm-fp8-blockwise", "trtllm-nvfp4"], - help=( - "Dynamic quantization mode for linear layers. " - "Quantizes weights on-the-fly during loading from an unquantized checkpoint." - ), - ) - - # Attention Backend - parser.add_argument( - "--attention_backend", - type=str, - default="VANILLA", - choices=["VANILLA", "TRTLLM", "FA4"], - help="Attention backend (VANILLA: PyTorch SDPA, TRTLLM: optimized kernels, " - "FA4: Flash Attention 4). " - "Note: TRTLLM falls back to VANILLA for cross-attention.", - ) - - # SageAttention (requires --attention_backend TRTLLM) - parser.add_argument( - "--enable_sage_attention", - action="store_true", - help=( - "Enable SageAttention (per-block quantized Q/K/V). Requires TRTLLM backend. " - "Block layout is chosen from --model_path: (1, 4, 1) for Wan2.1, " - "(1, 16, 1) otherwise." - ), - ) - - # Parallelism - parser.add_argument( - "--cfg_size", - type=int, - default=1, - choices=[1, 2], - help="CFG parallel size (1 or 2). Set to 2 for CFG Parallelism.", - ) - parser.add_argument( - "--ulysses_size", - type=int, - default=1, - help="Ulysses (head-sharding) parallel size within each CFG group. " - "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_row_size", - type=int, - default=1, - help="Attention2D row mesh size (Q all-gather dimension). " - "Can be set independently of --attn2d_col_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Total context parallelism degree = attn2d_row_size * attn2d_col_size. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_col_size", - type=int, - default=1, - help="Attention2D column mesh size (K/V all-gather dimension). " - "Can be set independently of --attn2d_row_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--ring_size", - type=int, - default=1, - help="Ring Attention parallel size. Cannot be combined with --attn2d_row_size / --attn2d_col_size.", - ) - parser.add_argument( - "--tp_size", - type=int, - default=1, - help="TP group size", - ) - parser.add_argument( - "--parallel_vae_size", - type=int, - default=1, - help="Number of ranks used for parallel VAE. 1 disables parallel VAE.", - ) - - # CUDA graph - parser.add_argument( - "--enable_cudagraph", action="store_true", help="Enable CudaGraph acceleration" - ) - - # torch.compile - parser.add_argument( - "--disable_torch_compile", action="store_true", help="Disable TorchCompile acceleration" - ) - parser.add_argument( - "--enable_fullgraph", action="store_true", help="Enable fullgraph for TorchCompile" - ) - - # Autotune - parser.add_argument( - "--disable_autotune", action="store_true", help="Disable autotuning during warmup" - ) - - # Debug / profiling - parser.add_argument( - "--enable_layerwise_nvtx_marker", action="store_true", help="Enable layerwise NVTX markers" - ) - - return parser.parse_args() - - -def _linear_type_to_quant_config(linear_type: str): - """Map --linear_type CLI shortcut to quant_config dict for VisualGenArgs.""" - mapping = { - "trtllm-fp8-per-tensor": {"quant_algo": "FP8", "dynamic": True}, - "trtllm-fp8-blockwise": {"quant_algo": "FP8_BLOCK_SCALES", "dynamic": True}, - "trtllm-nvfp4": {"quant_algo": "NVFP4", "dynamic": True}, - } - return mapping.get(linear_type) - - -def _teacache_config_from_args(args) -> TeaCacheConfig: - """Build TeaCacheConfig from CLI args; unset options keep Pydantic defaults.""" - kwargs: dict = {"use_ret_steps": args.use_ret_steps} - if args.teacache_thresh is not None: - kwargs["teacache_thresh"] = args.teacache_thresh - return TeaCacheConfig(**kwargs) - - -def _cache_dit_config_from_args(args) -> CacheDiTConfig: - """Subset of CacheDiTConfig from CLI; unset options keep Pydantic defaults.""" - overrides: dict = {} - if args.cache_dit_fn_compute_blocks is not None: - overrides["Fn_compute_blocks"] = args.cache_dit_fn_compute_blocks - if args.cache_dit_bn_compute_blocks is not None: - overrides["Bn_compute_blocks"] = args.cache_dit_bn_compute_blocks - if args.cache_dit_max_warmup_steps is not None: - overrides["max_warmup_steps"] = args.cache_dit_max_warmup_steps - if args.cache_dit_max_cached_steps is not None: - overrides["max_cached_steps"] = args.cache_dit_max_cached_steps - if args.cache_dit_residual_threshold is not None: - overrides["residual_diff_threshold"] = args.cache_dit_residual_threshold - if args.cache_dit_enable_taylorseer or args.cache_dit_taylorseer_order is not None: - overrides["enable_taylorseer"] = True - if args.cache_dit_taylorseer_order is not None: - overrides["taylorseer_order"] = args.cache_dit_taylorseer_order - if args.cache_dit_scm_mask_policy is not None: - overrides["scm_steps_mask_policy"] = args.cache_dit_scm_mask_policy - if args.cache_dit_scm_steps_policy is not None: - overrides["scm_steps_policy"] = args.cache_dit_scm_steps_policy - return CacheDiTConfig(**overrides) - - -def _wan_needs_fine_grained_sage(model_path: str) -> bool: - """Hard-coded heuristics for determining if a WAN model needs finer-grained SageAttention K-blocks.""" - lower = model_path.lower().replace(".", "_").replace("-", "_") - # For I2V, Wan2.1 14B is also observed to require higher granularity - return "_1_3b" in lower or "wan2_1_i" in lower - - -def main(): - args = parse_args() - - attention_cfg = { - "backend": args.attention_backend, - } - if args.enable_sage_attention: - k_block_size = 4 if _wan_needs_fine_grained_sage(args.model_path) else 16 - attention_cfg["quant_attention_config"] = { - "qk_dtype": "int8", - "q_block_size": 1, - "k_block_size": k_block_size, - "v_block_size": 1, - } - logger.info(f"SageAttention: INT8 Q/K, blocks (1, {k_block_size}, 1)") - - if args.enable_cache_dit: - cache_kwargs = {"cache_config": _cache_dit_config_from_args(args)} - elif args.enable_teacache: - cache_kwargs = {"cache_config": _teacache_config_from_args(args)} - else: - cache_kwargs = {} - - attn2d_size = args.attn2d_row_size * args.attn2d_col_size - if attn2d_size > 1 and args.ulysses_size > 1: - raise ValueError( - "Combining --ulysses_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - if args.ring_size > 1 and attn2d_size > 1: - raise ValueError( - "Combining --ring_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - - parallel_str = "" - if args.tp_size > 1: - parallel_str = f"TP(size={args.tp_size})" - - if args.ulysses_size > 1 or args.ring_size > 1: - parallel_str += f"Ulysses(size={args.ulysses_size}), Ring(size={args.ring_size})" - elif attn2d_size > 1: - parallel_str += ( - f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " - f"total={attn2d_size})" - ) - elif not parallel_str: - parallel_str = "None" - - kwargs = dict( - attention_config=attention_cfg, - **cache_kwargs, - parallel_config={ - "cfg_size": args.cfg_size, - "ulysses_size": args.ulysses_size, - "ring_size": args.ring_size, - "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), - "tp_size": args.tp_size, - "parallel_vae_size": args.parallel_vae_size, - }, - torch_compile_config={ - "enable": not args.disable_torch_compile, - "enable_fullgraph": args.enable_fullgraph, - "enable_autotune": not args.disable_autotune, - }, - cuda_graph_config={"enable": args.enable_cudagraph}, - enable_layerwise_nvtx_marker=args.enable_layerwise_nvtx_marker, - ) - quant_config = _linear_type_to_quant_config(args.linear_type) - if quant_config is not None: - kwargs["quant_config"] = quant_config - - visual_gen_args = VisualGenArgs(**kwargs) - - logger.info( - f"Initializing VisualGen: " - f"cfg_size={visual_gen_args.parallel_config.cfg_size}, " - f"parallelism={parallel_str}" - ) - visual_gen = VisualGen( - model=args.model_path, - args=visual_gen_args, - ) - - try: - defaults = visual_gen.default_params - negative_prompt_log = ( - args.negative_prompt if args.negative_prompt is not None else "[model default]" - ) - height = args.height if args.height is not None else defaults.height - width = args.width if args.width is not None else defaults.width - num_frames = args.num_frames if args.num_frames is not None else defaults.num_frames - steps = args.steps if args.steps is not None else defaults.num_inference_steps - frame_rate = defaults.frame_rate - - logger.info(f"Generating video for prompt: '{args.prompt}'") - logger.info(f"Negative prompt: '{negative_prompt_log}'") - logger.info(f"Input image: {args.image_path}") - if args.last_image_path: - logger.info(f"Last frame image: {args.last_image_path}") - logger.info(f"Resolution: {height}x{width}, Frames: {num_frames}, Steps: {steps}") - - start_time = time.time() - - extra_params = {} - if args.last_image_path: - extra_params["last_image"] = args.last_image_path - if args.guidance_scale_2 is not None: - extra_params["guidance_scale_2"] = args.guidance_scale_2 - if args.boundary_ratio is not None: - extra_params["boundary_ratio"] = args.boundary_ratio - - output = visual_gen.generate( - inputs=args.prompt, - params=VisualGenParams( - height=args.height, - width=args.width, - num_inference_steps=args.steps, - guidance_scale=args.guidance_scale, - seed=args.seed, - num_frames=args.num_frames, - frame_rate=frame_rate, - negative_prompt=args.negative_prompt, - image=args.image_path, - extra_params=extra_params if extra_params else None, - ), - ) - - logger.info(f"Generation completed in {time.time() - start_time:.2f}s") - - output.save(args.output_path) - - finally: - visual_gen.shutdown() - - -if __name__ == "__main__": - main() diff --git a/examples/visual_gen/visual_gen_wan_t2v.py b/examples/visual_gen/visual_gen_wan_t2v.py deleted file mode 100755 index af77bab8c133..000000000000 --- a/examples/visual_gen/visual_gen_wan_t2v.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""WAN Text-to-Video generation using TensorRT-LLM Visual Generation.""" - -import argparse -import time - -from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams, logger -from tensorrt_llm.visual_gen.args import CacheDiTConfig, TeaCacheConfig - -logger.set_level("info") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="TRTLLM VisualGen - Wan Text-to-Video Inference Example (supports Wan 2.1 and Wan 2.2)" - ) - - # Model & Input - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Local path or HuggingFace Hub model ID (e.g., Wan-AI/Wan2.1-T2V-1.3B-Diffusers)", - ) - parser.add_argument( - "--revision", - type=str, - default=None, - help="HuggingFace Hub revision (branch, tag, or commit SHA)", - ) - parser.add_argument("--prompt", type=str, required=True, help="Text prompt for generation") - parser.add_argument( - "--negative_prompt", - type=str, - default=None, - help="Negative prompt. Default is model-specific.", - ) - parser.add_argument( - "--output_path", - type=str, - default="output.png", - help="Path to save the output image/video frame", - ) - - # Generation Params - parser.add_argument( - "--height", type=int, default=None, help="Video height (default: auto-detect)" - ) - parser.add_argument( - "--width", type=int, default=None, help="Video width (default: auto-detect)" - ) - parser.add_argument( - "--num_frames", - type=int, - default=None, - help="Number of frames to generate (default: auto-detect)", - ) - parser.add_argument( - "--steps", - type=int, - default=None, - help="Number of denoising steps (default: auto-detect, 50 for Wan2.1 and Wan 2.2 5B, 40 for Wan2.2 A14B)", - ) - parser.add_argument( - "--guidance_scale", - type=float, - default=None, - help="Guidance scale (default: auto-detect, 5.0 for Wan2.1 and Wan 2.2 5B, 4.0 for Wan2.2 A14B)", - ) - parser.add_argument( - "--guidance_scale_2", - type=float, - default=None, - help="Second-stage guidance scale for Wan2.2 A14B two-stage denoising (default: 3.0)", - ) - parser.add_argument( - "--boundary_ratio", - type=float, - default=None, - help="Custom boundary ratio for two-stage denoising (default: auto-detect)", - ) - parser.add_argument("--seed", type=int, default=42, help="Random seed") - - # Diffusion cache acceleration (TeaCache vs Cache-DiT; mutually exclusive) - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument( - "--enable_teacache", action="store_true", help="Enable TeaCache acceleration" - ) - cache_group.add_argument( - "--enable_cache_dit", - action="store_true", - help=( - "Enable Cache-DiT per-block acceleration (requires the cache_dit package; " - "see https://github.com/vipshop/cache-dit). Incompatible with --enable_teacache." - ), - ) - parser.add_argument( - "--teacache_thresh", - type=float, - default=0.2, - help="TeaCache similarity threshold (rel_l1_thresh); ignored when using --enable_cache_dit", - ) - parser.add_argument( - "--use_ret_steps", - action="store_true", - help="Use ret_steps mode for TeaCache. " - "Using Retention Steps will result in faster generation speed and better generation quality. " - "Ignored when using --enable_cache_dit.", - ) - - # Cache-DiT overrides (only apply with --enable_cache_dit; omitted fields use CacheDiTConfig defaults) - parser.add_argument( - "--cache_dit_fn_compute_blocks", - type=int, - default=None, - help="DBCache Fn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_bn_compute_blocks", - type=int, - default=None, - help="DBCache Bn_compute_blocks (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_warmup_steps", - type=int, - default=None, - help="DBCache max_warmup_steps (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_max_cached_steps", - type=int, - default=None, - help="DBCache max_cached_steps (-1 = no cap; default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_residual_threshold", - type=float, - default=None, - help="DBCache residual_diff_threshold (default: from CacheDiTConfig).", - ) - parser.add_argument( - "--cache_dit_enable_taylorseer", - action="store_true", - help="Enable TaylorSeer calibrator (default: off).", - ) - parser.add_argument( - "--cache_dit_taylorseer_order", - type=int, - default=None, - choices=[1, 2, 3, 4], - help="TaylorSeer order; implies TaylorSeer on if set. Default order from CacheDiTConfig.", - ) - parser.add_argument( - "--cache_dit_scm_mask_policy", - type=str, - default=None, - help="SCM steps_mask policy name (e.g. fast, medium, slow, ultra). Omit to disable SCM.", - ) - parser.add_argument( - "--cache_dit_scm_steps_policy", - type=str, - default=None, - choices=["dynamic", "static"], - help="SCM steps_computation_policy (default: dynamic if not overridden).", - ) - - # Quantization - parser.add_argument( - "--linear_type", - type=str, - default="default", - choices=["default", "trtllm-fp8-per-tensor", "trtllm-fp8-blockwise", "trtllm-nvfp4"], - help=( - "Dynamic quantization mode for linear layers. " - "Quantizes weights on-the-fly during loading from an unquantized checkpoint." - ), - ) - - # Attention Backend - parser.add_argument( - "--attention_backend", - type=str, - default="VANILLA", - choices=["VANILLA", "TRTLLM", "FA4"], - help="Attention backend (VANILLA: PyTorch SDPA, TRTLLM: optimized kernels, " - "FA4: Flash Attention 4). " - "Note: TRTLLM falls back to VANILLA for cross-attention.", - ) - - # SageAttention (requires --attention_backend TRTLLM) - parser.add_argument( - "--enable_sage_attention", - action="store_true", - help=( - "Enable SageAttention (per-block quantized Q/K/V). Requires TRTLLM backend. " - "Block layout is chosen from --model_path: (1, 4, 1) for Wan2.x 1.3B, " - "(1, 16, 1) otherwise." - ), - ) - - # Parallelism - parser.add_argument( - "--cfg_size", - type=int, - default=1, - choices=[1, 2], - help="CFG parallel size (1 or 2). " - "Distributes positive/negative prompts across GPUs. " - "Example: cfg_size=2 on 4 GPUs -> 2 GPUs per prompt.", - ) - parser.add_argument( - "--ulysses_size", - type=int, - default=1, - help="Ulysses (head-sharding) parallel size within each CFG group. " - "Requirements: num_heads (12) must be divisible by ulysses_size. " - "Example: ulysses_size=2 on 4 GPUs with cfg_size=2 -> " - "2 CFG groups x 2 Ulysses ranks = 4 GPUs total. " - "Cannot be combined with --attn2d_row_size / --attn2d_col_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_row_size", - type=int, - default=1, - help="Attention2D row mesh size (Q all-gather dimension). " - "Can be set independently of --attn2d_col_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Total context parallelism degree = attn2d_row_size * attn2d_col_size. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--attn2d_col_size", - type=int, - default=1, - help="Attention2D column mesh size (K/V all-gather dimension). " - "Can be set independently of --attn2d_row_size; asymmetric meshes (e.g. 1x4 or 4x1) are valid. " - "Cannot be combined with --ulysses_size (not yet implemented).", - ) - parser.add_argument( - "--ring_size", - type=int, - default=1, - help="Ring Attention parallel size. Cannot be combined with --attn2d_row_size / --attn2d_col_size.", - ) - parser.add_argument( - "--tp_size", - type=int, - default=1, - help="TP group size", - ) - parser.add_argument( - "--parallel_vae_size", - type=int, - default=1, - help="Number of ranks used for parallel VAE. 1 disables parallel VAE.", - ) - - # CUDA graph - parser.add_argument( - "--enable_cudagraph", action="store_true", help="Enable CudaGraph acceleration" - ) - - # torch.compile - parser.add_argument( - "--disable_torch_compile", action="store_true", help="Disable TorchCompile acceleration" - ) - parser.add_argument( - "--enable_fullgraph", action="store_true", help="Enable fullgraph for TorchCompile" - ) - - # Autotune - parser.add_argument( - "--disable_autotune", action="store_true", help="Disable autotuning during warmup" - ) - - # Debug / profiling - parser.add_argument( - "--enable_layerwise_nvtx_marker", action="store_true", help="Enable layerwise NVTX markers" - ) - - return parser.parse_args() - - -def _linear_type_to_quant_config(linear_type: str): - """Map --linear_type CLI shortcut to quant_config dict for VisualGenArgs.""" - mapping = { - "trtllm-fp8-per-tensor": {"quant_algo": "FP8", "dynamic": True}, - "trtllm-fp8-blockwise": {"quant_algo": "FP8_BLOCK_SCALES", "dynamic": True}, - "trtllm-nvfp4": {"quant_algo": "NVFP4", "dynamic": True}, - } - return mapping.get(linear_type) - - -def _teacache_config_from_args(args) -> TeaCacheConfig: - """Build TeaCacheConfig from CLI args; unset options keep Pydantic defaults.""" - kwargs: dict = {"use_ret_steps": args.use_ret_steps} - if args.teacache_thresh is not None: - kwargs["teacache_thresh"] = args.teacache_thresh - return TeaCacheConfig(**kwargs) - - -def _cache_dit_config_from_args(args) -> CacheDiTConfig: - """Subset of CacheDiTConfig from CLI; unset options keep Pydantic defaults.""" - overrides: dict = {} - if args.cache_dit_fn_compute_blocks is not None: - overrides["Fn_compute_blocks"] = args.cache_dit_fn_compute_blocks - if args.cache_dit_bn_compute_blocks is not None: - overrides["Bn_compute_blocks"] = args.cache_dit_bn_compute_blocks - if args.cache_dit_max_warmup_steps is not None: - overrides["max_warmup_steps"] = args.cache_dit_max_warmup_steps - if args.cache_dit_max_cached_steps is not None: - overrides["max_cached_steps"] = args.cache_dit_max_cached_steps - if args.cache_dit_residual_threshold is not None: - overrides["residual_diff_threshold"] = args.cache_dit_residual_threshold - if args.cache_dit_enable_taylorseer or args.cache_dit_taylorseer_order is not None: - overrides["enable_taylorseer"] = True - if args.cache_dit_taylorseer_order is not None: - overrides["taylorseer_order"] = args.cache_dit_taylorseer_order - if args.cache_dit_scm_mask_policy is not None: - overrides["scm_steps_mask_policy"] = args.cache_dit_scm_mask_policy - if args.cache_dit_scm_steps_policy is not None: - overrides["scm_steps_policy"] = args.cache_dit_scm_steps_policy - return CacheDiTConfig(**overrides) - - -def _wan_needs_fine_grained_sage(model_path: str) -> bool: - """Hard-coded heuristics for determining if a WAN model needs finer-grained SageAttention K-blocks.""" - lower = model_path.lower().replace(".", "_").replace("-", "_") - return "_1_3b" in lower - - -def main(): - args = parse_args() - - attn2d_size = args.attn2d_row_size * args.attn2d_col_size - if attn2d_size > 1 and args.ulysses_size > 1: - raise ValueError( - "Combining --ulysses_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - if args.ring_size > 1 and attn2d_size > 1: - raise ValueError( - "Combining --ring_size with --attn2d_row_size/--attn2d_col_size is not yet implemented." - ) - - if args.ulysses_size > 1: - num_heads = 40 - logger.info( - f"Using Ulysses sequence parallelism: " - f"{num_heads} heads / {args.ulysses_size} ranks = " - f"{num_heads // args.ulysses_size} heads per GPU" - ) - - if args.ulysses_size > 1 or args.ring_size > 1: - parallel_str = f"Ulysses(size={args.ulysses_size}), Ring(size={args.ring_size})" - elif attn2d_size > 1: - parallel_str = ( - f"Attention2D(row={args.attn2d_row_size}, col={args.attn2d_col_size}, " - f"total={attn2d_size})" - ) - else: - parallel_str = "None" - - attention_cfg = { - "backend": args.attention_backend, - } - if args.enable_sage_attention: - k_block_size = 4 if _wan_needs_fine_grained_sage(args.model_path) else 16 - attention_cfg["quant_attention_config"] = { - "qk_dtype": "int8", - "q_block_size": 1, - "k_block_size": k_block_size, - "v_block_size": 1, - } - logger.info(f"SageAttention: INT8 Q/K, blocks (1, {k_block_size}, 1)") - - if args.enable_cache_dit: - cache_kwargs = {"cache_config": _cache_dit_config_from_args(args)} - elif args.enable_teacache: - cache_kwargs = {"cache_config": _teacache_config_from_args(args)} - else: - cache_kwargs = {} - - kwargs = dict( - revision=args.revision, - attention_config=attention_cfg, - **cache_kwargs, - parallel_config={ - "cfg_size": args.cfg_size, - "ulysses_size": args.ulysses_size, - "ring_size": args.ring_size, - "attn2d_size": (args.attn2d_row_size, args.attn2d_col_size), - "tp_size": args.tp_size, - "parallel_vae_size": args.parallel_vae_size, - }, - torch_compile_config={ - "enable": not args.disable_torch_compile, - "enable_fullgraph": args.enable_fullgraph, - "enable_autotune": not args.disable_autotune, - }, - cuda_graph_config={"enable": args.enable_cudagraph}, - enable_layerwise_nvtx_marker=args.enable_layerwise_nvtx_marker, - ) - quant_config = _linear_type_to_quant_config(args.linear_type) - if quant_config is not None: - kwargs["quant_config"] = quant_config - - visual_gen_args = VisualGenArgs(**kwargs) - - logger.info( - f"Initializing VisualGen: " - f"cfg_size={visual_gen_args.parallel_config.cfg_size}, " - f"parallelism={parallel_str}" - ) - visual_gen = VisualGen( - model=args.model_path, - args=visual_gen_args, - ) - - try: - defaults = visual_gen.default_params - negative_prompt_log = ( - args.negative_prompt if args.negative_prompt is not None else "[model default]" - ) - height = args.height if args.height is not None else defaults.height - width = args.width if args.width is not None else defaults.width - num_frames = args.num_frames if args.num_frames is not None else defaults.num_frames - steps = args.steps if args.steps is not None else defaults.num_inference_steps - frame_rate = defaults.frame_rate - - logger.info(f"Generating video for prompt: '{args.prompt}'") - logger.info(f"Negative prompt: '{negative_prompt_log}'") - logger.info(f"Resolution: {height}x{width}, Frames: {num_frames}, Steps: {steps}") - - start_time = time.time() - - extra_params = {} - if args.guidance_scale_2 is not None: - extra_params["guidance_scale_2"] = args.guidance_scale_2 - if args.boundary_ratio is not None: - extra_params["boundary_ratio"] = args.boundary_ratio - - output = visual_gen.generate( - inputs=args.prompt, - params=VisualGenParams( - height=args.height, - width=args.width, - num_inference_steps=args.steps, - guidance_scale=args.guidance_scale, - seed=args.seed, - num_frames=args.num_frames, - frame_rate=frame_rate, - negative_prompt=args.negative_prompt, - extra_params=extra_params if extra_params else None, - ), - ) - - logger.info(f"Generation completed in {time.time() - start_time:.2f}s") - - output.save(args.output_path) - - finally: - visual_gen.shutdown() - - -if __name__ == "__main__": - main() diff --git a/tests/integration/defs/examples/test_visual_gen.py b/tests/integration/defs/examples/test_visual_gen.py index b91839f51c2e..1a025a0e294c 100644 --- a/tests/integration/defs/examples/test_visual_gen.py +++ b/tests/integration/defs/examples/test_visual_gen.py @@ -21,6 +21,7 @@ import random import subprocess import sys +import textwrap import time import urllib.request import zipfile @@ -685,12 +686,17 @@ def test_wan22_t2v_lpips_against_golden(tmp_path): @pytest.fixture(scope="session") def wan_trtllm_video_path(_visual_gen_deps, llm_venv, llm_root): - """Generate input video via visual_gen_wan_t2v.py and return path to trtllm_output.mp4.""" + """Generate input video via models/wan_t2v.py and return path to trtllm_output.mp4.""" return _generate_wan_video(llm_venv, llm_root, WAN_T2V_MODEL_SUBPATH, "wan") def _generate_wan_video(llm_venv, llm_root, model_subpath, output_subdir): - """Generate a video with visual_gen_wan_t2v.py for a given model checkpoint. + """Generate a video with examples/visual_gen/models/wan_t2v.py for a given checkpoint. + + The slim example hardcodes prompt/H/W/frames (matching WAN_T2V_* constants + above), so this helper only synthesizes a VisualGenArgs YAML for engine + config (parallelism / attention / cuda graph) and passes it via + ``--visual_gen_args``. Returns the path to the generated .mp4, or calls pytest.skip if the model is not found under LLM_MODELS_ROOT. @@ -707,25 +713,36 @@ def _generate_wan_video(llm_venv, llm_root, model_subpath, output_subdir): output_path = os.path.join(out_dir, VISUAL_GEN_OUTPUT_VIDEO) if os.path.isfile(output_path): return output_path - script_path = os.path.join(llm_root, "examples", "visual_gen", "visual_gen_wan_t2v.py") + + script_path = os.path.join(llm_root, "examples", "visual_gen", "models", "wan_t2v.py") assert os.path.isfile(script_path), f"Visual gen script not found: {script_path}" + + cfg_size = 2 if torch.cuda.device_count() >= 2 else 1 + visual_gen_args_yaml = os.path.join(out_dir, "visual_gen_args.yaml") + with open(visual_gen_args_yaml, "w") as f: + f.write( + textwrap.dedent( + f"""\ + attention_config: + backend: VANILLA + parallel_config: + cfg_size: {cfg_size} + ulysses_size: 1 + cuda_graph_config: + enable: false + """ + ) + ) + cmd = [ script_path, - "--height", - str(WAN_T2V_HEIGHT), - "--width", - str(WAN_T2V_WIDTH), - "--num_frames", - str(WAN_T2V_NUM_FRAMES), - "--model_path", + "--model", model_path, - "--prompt", - WAN_T2V_PROMPT, + "--visual_gen_args", + visual_gen_args_yaml, "--output_path", output_path, ] - if torch.cuda.device_count() >= 2: - cmd.extend(["--cfg_size", "2"]) venv_check_call(llm_venv, cmd) assert os.path.isfile(output_path), f"Visual gen did not produce {output_path}" return output_path @@ -756,9 +773,6 @@ def _linear_type_to_quant_config(linear_type): def _generate_ltx2_video(llm_venv, output_subdir, linear_type="default"): """Generate a video using the LTX-2 Python API directly. - Calls VisualGen / VisualGenArgs / VisualGenParams instead of shelling out - to examples/visual_gen/visual_gen_ltx2.py (which may be removed). - Returns the path to the generated .mp4, or calls pytest.skip if the model or text encoder is not found under LLM_MODELS_ROOT. """ @@ -1185,15 +1199,11 @@ def test_wan_t2v_example(_visual_gen_deps, llm_root, llm_venv): This is a core example test: it validates that the per-model example script and the shared YAML config work together as documented in the README. - Uses the pre-quantized Wan 2.2 T2V A14B NVFP4 checkpoint. - - NOTE: If a strict-duplicate test exists elsewhere (same model, same quant, - same resolution, same prompt, same script invocation), consider removing - it in favour of this one. As of this writing, the closest test is - test_vbench_dimension_score_wan22_a14b_nvfp4 which uses the same checkpoint - but invokes the *old* visual_gen_wan_t2v.py script (not models/wan_t2v.py) - with different resolution/prompt and additionally runs VBench scoring. - Not a strict duplicate. + Uses the pre-quantized Wan 2.2 T2V A14B NVFP4 checkpoint and the shared + ``configs/wan2.2-t2v-fp4-1gpu.yaml`` (NVFP4 dynamic quant). The closest + overlapping test is ``test_vbench_dimension_score_wan22_a14b_nvfp4``, + which runs the same script but with a no-quant YAML synthesized at + runtime and additionally evaluates VBench scores. """ scratch_space = conftest.llm_models_root() model_path = os.path.join(scratch_space, WAN22_A14B_NVFP4_MODEL_SUBPATH) From f4021782814f0862f71f469b4461ca4b40cac842 Mon Sep 17 00:00:00 2001 From: Pengbo Wang <221450789+pengbowang-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:57:56 +0800 Subject: [PATCH 153/308] [https://nvbugs/6117811][fix] Fix XQA IMA for invalid pages with sliding window (#14459) Signed-off-by: Pengbo Wang <221450789+pengbowang-nv@users.noreply.github.com> --- 3rdparty/fetch_content.json | 2 +- cpp/kernels/xqa/CMakeLists.txt | 9 ++++- cpp/kernels/xqa/ldgsts.cuh | 9 ++++- cpp/kernels/xqa/mha.cu | 13 +++++-- cpp/kernels/xqa/mhaUtils.cuh | 20 ++++++---- cpp/kernels/xqa/mha_sm90.cu | 25 ++++++++----- cpp/kernels/xqa/test/test.cpp | 50 ++++++++++++++++++++++++- tests/integration/test_lists/waives.txt | 3 -- 8 files changed, 103 insertions(+), 28 deletions(-) diff --git a/3rdparty/fetch_content.json b/3rdparty/fetch_content.json index 8cf04f72f8f5..e6218c2629c2 100644 --- a/3rdparty/fetch_content.json +++ b/3rdparty/fetch_content.json @@ -37,7 +37,7 @@ }, { "name": "eigen", - "git_repository": "https://github.com/libeigen/eigen", + "git_repository": "https://gitlab.com/libeigen/eigen.git", "git_tag": "3.4.0", "git_shallow": true }, diff --git a/cpp/kernels/xqa/CMakeLists.txt b/cpp/kernels/xqa/CMakeLists.txt index f9e53900cf6e..6a38ee8caac6 100644 --- a/cpp/kernels/xqa/CMakeLists.txt +++ b/cpp/kernels/xqa/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -88,6 +88,13 @@ add_custom_command( add_custom_target(xqa_sources_h DEPENDS ${XQA_SOURCES_H}) if(BUILD_XQA_TESTS) + if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + get_filename_component(TRT_LLM_ROOT_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) + add_subdirectory("${TRT_LLM_ROOT_DIR}/3rdparty" + "${CMAKE_CURRENT_BINARY_DIR}/3rdparty") + endif() + # Try to find system installed GTest first find_package(GTest QUIET) if(NOT GTest_FOUND) diff --git a/cpp/kernels/xqa/ldgsts.cuh b/cpp/kernels/xqa/ldgsts.cuh index 702cb8191db5..f7ef3045c6d8 100644 --- a/cpp/kernels/xqa/ldgsts.cuh +++ b/cpp/kernels/xqa/ldgsts.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +30,13 @@ template __device__ inline void copyAsync( void* dst, void const* src, uint32_t srcSize = size) // srcSize == 0 means filling with zeros. { + // FIXME: there are probably some race condition or compiler issue and needs further investigation. + // Without this conditional assignment the code might read global memory even if srcSize = 0. + if (srcSize == 0) + { + src = nullptr; + } + static_assert(size == 4 || size == 8 || size == 16); if constexpr (size == 16) { diff --git a/cpp/kernels/xqa/mha.cu b/cpp/kernels/xqa/mha.cu index 636cd19a4a1d..593e27a62f95 100644 --- a/cpp/kernels/xqa/mha.cu +++ b/cpp/kernels/xqa/mha.cu @@ -1709,6 +1709,9 @@ CUBIN_EXPORT __global__ #else constexpr bool rtIsReallySliding = false; constexpr uint32_t nbTotalSkipTokens = 0; +#endif +#if USE_PAGED_KV_CACHE + uint32_t const nbSkipLeadingPages = nbTotalSkipTokens / tokensPerPage; #endif uint32_t const nbSkipLeadingTiles = nbTotalSkipTokens / ctaTile.x; uint32_t const tile0NbSkipTokens = nbTotalSkipTokens % ctaTile.x; @@ -1763,10 +1766,11 @@ CUBIN_EXPORT __global__ { #if BEAM_WIDTH == 1 uint32_t const idxBeam = 0; - pageIdx = getPage(cacheList, true, idxReq, idxBeam, idxPage, nbPages); + pageIdx = getPage( + cacheList, true, idxReq, idxBeam, idxPage, nbPages, nbSkipLeadingPages); #else auto& dst = smem.kCachePages[warpIdx.x]; - loadPagesForBeamSearchAsync<1>(0U, dst, cacheList, true, idxReq, idxPage, nbPages); + loadPagesForBeamSearchAsync<1>(0U, dst, cacheList, true, idxReq, idxPage, nbPages, nbSkipLeadingPages); #endif }; uint32_t idxPageBeg = nbPagesPerCtaTile * seqIterInit + warpIdx.x * warpTile.x / tokensPerPage; @@ -2090,11 +2094,12 @@ CUBIN_EXPORT __global__ { #if BEAM_WIDTH == 1 uint32_t const idxBeam = 0; - pageIdx = getPage(cacheList, false, idxReq, idxBeam, idxPageBeg, nbPages); + pageIdx = getPage( + cacheList, false, idxReq, idxBeam, idxPageBeg, nbPages, nbSkipLeadingPages); #else auto& dst = smem.vCachePages[grpLoadV ? warpGrpIdx : warpIdx.x]; loadPagesForBeamSearchAsync( - grpLoadV ? warpIdxInGrp : 0U, dst, cacheList, false, idxReq, idxPageBeg, nbPages); + grpLoadV ? warpIdxInGrp : 0U, dst, cacheList, false, idxReq, idxPageBeg, nbPages, nbSkipLeadingPages); #endif }; uint32_t idxPageBeg = nbPagesPerCtaTile * seqIterInit + cacheVTileSeqLen * warpGrpIdx / tokensPerPage; diff --git a/cpp/kernels/xqa/mhaUtils.cuh b/cpp/kernels/xqa/mhaUtils.cuh index 96b54b92f813..8c00897beaa4 100644 --- a/cpp/kernels/xqa/mhaUtils.cuh +++ b/cpp/kernels/xqa/mhaUtils.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -298,7 +298,7 @@ __device__ inline uint32_t getCtxCacheSeqLen(BeamSearchParams const& beamSearchP template __device__ inline Vec getPage(KVCacheList const& cacheList, bool isK, - uint32_t idxReq, uint32_t idxBeam, uint32_t idxPageBeg, uint32_t nbPages) + uint32_t idxReq, uint32_t idxBeam, uint32_t idxPageBeg, uint32_t nbPages, uint32_t nbSkipLeadingPages) { auto const maxNbPagesPerSeq = cacheList.maxNbPagesPerSeq; Vec ret; @@ -307,11 +307,14 @@ __device__ inline Vec getPage(KVCacheList { uint32_t const idxPage = idxPageBeg + i; #if PAGED_KV_CACHE_LAYOUT == 1 && USE_PAGED_KV_CACHE - ret[i] = (idxPage < nbPages ? cacheList.kvCachePageList[maxNbPagesPerSeq * idxReq + idxPage] : kBAD_PAGE_INDEX); + ret[i] = ((idxPage >= nbSkipLeadingPages && idxPage < nbPages) + ? cacheList.kvCachePageList[maxNbPagesPerSeq * idxReq + idxPage] + : kBAD_PAGE_INDEX); #else - ret[i] = (idxPage < nbPages ? cacheList.kvCachePageList[beamWidth * 2 * maxNbPagesPerSeq * idxReq - + 2 * maxNbPagesPerSeq * idxBeam + maxNbPagesPerSeq * (isK ? 0U : 1U) + idxPage] - : kBAD_PAGE_INDEX); + ret[i] = ((idxPage >= nbSkipLeadingPages && idxPage < nbPages) + ? cacheList.kvCachePageList[beamWidth * 2 * maxNbPagesPerSeq * idxReq + 2 * maxNbPagesPerSeq * idxBeam + + maxNbPagesPerSeq * (isK ? 0U : 1U) + idxPage] + : kBAD_PAGE_INDEX); #endif } return ret; @@ -320,7 +323,7 @@ __device__ inline Vec getPage(KVCacheList template __device__ inline void loadPagesForBeamSearchAsync(uint32_t idxWarp, Vec, beamWidth>& dst, KVCacheList const& cacheList, bool isK, - uint32_t idxReq, uint32_t idxPageBeg, uint32_t nbPages) + uint32_t idxReq, uint32_t idxPageBeg, uint32_t nbPages, uint32_t nbSkipLeadingPages) { assert(idxWarp < nbWarps); auto const maxNbPagesPerSeq = cacheList.maxNbPagesPerSeq; @@ -333,10 +336,11 @@ __device__ inline void loadPagesForBeamSearchAsync(uint32_t idxWarp, { constexpr uint32_t nbBytes = sizeof(KVCachePageIndex); uint32_t const idxPage = idxPageBeg + idxLoadedPage; + // TODO: for beamsearch we use pagedIdx = 0 instead of kBAD_PAGE_INDEX, should unify it ldgsts::copyAsync(&dst[idxBeam][idxLoadedPage], &cacheList.kvCachePageList[beamWidth * 2 * maxNbPagesPerSeq * idxReq + 2 * maxNbPagesPerSeq * idxBeam + (isK ? 0U : maxNbPagesPerSeq) + idxPage], - idxPage < nbPages ? nbBytes : 0U); + idxPage >= nbSkipLeadingPages && idxPage < nbPages ? nbBytes : 0U); } } diff --git a/cpp/kernels/xqa/mha_sm90.cu b/cpp/kernels/xqa/mha_sm90.cu index 7f7164c9a24e..4e9bf51e00b2 100644 --- a/cpp/kernels/xqa/mha_sm90.cu +++ b/cpp/kernels/xqa/mha_sm90.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -318,9 +318,10 @@ struct KVTilePartLoader CUtensorMap const& tensorMap; #if USE_PAGED_KV_CACHE - uint32_t const nbPages; // for bound check + uint32_t const nbPages; // for bound check + uint32_t const nbSkipLeadingPages; // pages fully outside the sliding window Vec& pages; - uint32_t idxTileRef; // idxTile used to load the pages + uint32_t idxTileRef; // idxTile used to load the pages #endif uint32_t const baseOffset; @@ -328,7 +329,7 @@ struct KVTilePartLoader uint32_t idxReq, uint32_t idxHeadGrp, CUtensorMap const& tensorMap #if USE_PAGED_KV_CACHE , - uint32_t nbPages, Vec& pageBuf + uint32_t nbPages, uint32_t nbSkipLeadingPages, Vec& pageBuf #endif ); // tensorMap is for one whole page ([nbKHeads*tokensPerPage][headElems]) or whole cache @@ -760,6 +761,9 @@ CUBIN_EXPORT __global__ #else constexpr bool rtIsReallySliding = false; constexpr uint32_t nbTotalSkipTokens = 0; +#endif +#if USE_PAGED_KV_CACHE + uint32_t const nbSkipLeadingPages = nbTotalSkipTokens / tokensPerPage; #endif uint32_t const nbSkipLeadingTiles = nbTotalSkipTokens / tileSize; uint32_t const tile0NbSkipTokens = nbTotalSkipTokens % tileSize; @@ -1501,7 +1505,7 @@ CUBIN_EXPORT __global__ #else tensorMap, #endif - nbPages, smem.pages[0] + nbPages, nbSkipLeadingPages, smem.pages[0] #else tensorMap #endif @@ -1575,7 +1579,7 @@ CUBIN_EXPORT __global__ #else tensorMap, #endif - nbPages, smem.pages[1] + nbPages, nbSkipLeadingPages, smem.pages[1] #else tensorMap #endif @@ -1871,7 +1875,7 @@ CUBIN_EXPORT __global__ } #else #if GENERATE_CUBIN - static_assert("This kernel is for Hopper only"); + static_assert(false, "This kernel is for Hopper only"); #else asm volatile("trap;\n"); #endif @@ -1970,7 +1974,7 @@ __device__ inline KVTilePartLoader::KVTilePartLoader(bool isK, uint32_t nbKHeads KVCacheList const& cacheList, uint32_t idxReq, uint32_t idxHeadGrp, CUtensorMap const& tensorMap #if USE_PAGED_KV_CACHE , - uint32_t nbPages, Vec& pageBuf + uint32_t nbPages, uint32_t nbSkipLeadingPages, Vec& pageBuf #endif ) : nbKHeads{nbKHeads} @@ -1980,6 +1984,7 @@ __device__ inline KVTilePartLoader::KVTilePartLoader(bool isK, uint32_t nbKHeads , tensorMap{tensorMap} #if USE_PAGED_KV_CACHE , nbPages{nbPages} + , nbSkipLeadingPages{nbSkipLeadingPages} , pages{pageBuf} #if PAGED_KV_CACHE_LAYOUT == 1 , baseOffset{idxReq * cacheList.maxNbPagesPerSeq} @@ -2040,7 +2045,9 @@ __device__ inline void KVTilePartLoader::loadPages(uint32_t idxTile) for (uint32_t i = 0; i < nbPagesPerTile; i++) { uint32_t const idxPage = idxPageBeg + i; - auto const page = idxPage < nbPages ? cacheList.kvCachePageList[baseOffset + idxPage] : kBAD_PAGE_INDEX; + auto const page = (idxPage >= nbSkipLeadingPages && idxPage < nbPages) + ? cacheList.kvCachePageList[baseOffset + idxPage] + : kBAD_PAGE_INDEX; if (warpElectSync()) { pages[i] = page; diff --git a/cpp/kernels/xqa/test/test.cpp b/cpp/kernels/xqa/test/test.cpp index 9702d4bf6100..a3821e57d1e5 100644 --- a/cpp/kernels/xqa/test/test.cpp +++ b/cpp/kernels/xqa/test/test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,6 +44,10 @@ #define USE_SMALL_IO 1 #endif +#ifndef XQA_TEST_POISON_SLIDING_WINDOW_PREFIX_PAGES +#define XQA_TEST_POISON_SLIDING_WINDOW_PREFIX_PAGES 1 +#endif + void warmup(cudaDeviceProp const& prop, float ms, cudaStream_t stream = nullptr); bool const isTracing = []() { @@ -655,6 +659,36 @@ void runTest(uint32_t batchSize, uint32_t seqLen, bool testPerf, bool refCheck, } } +#if USE_PAGED_KV_CACHE && SLIDING_WINDOW && XQA_TEST_POISON_SLIDING_WINDOW_PREFIX_PAGES + { + constexpr KVCachePageIndex kPoisonPageIdx = static_cast(1U << 20); +#if SPEC_DEC + uint32_t const firstQSeqLen = seqLen - qSeqLen + 1; + uint32_t const seqBeg = firstQSeqLen < slidingWinSize ? 0 : firstQSeqLen - slidingWinSize; +#else + uint32_t const seqBeg = seqLen < slidingWinSize ? 0 : seqLen - slidingWinSize; +#endif + uint32_t const nbPoisonPages = std::min(seqBeg / tokensPerPage, nbPagesPerSeq); +#if PAGED_KV_CACHE_LAYOUT == 1 + for (uint32_t batch = 0; batch < batchSize; batch++) + { + std::fill_n(pageList[batch], nbPoisonPages, kPoisonPageIdx); + } +#else + for (uint32_t batch = 0; batch < batchSize; batch++) + { + for (uint32_t beam = 0; beam < beamWidth; beam++) + { + for (uint32_t kv = 0; kv < 2; kv++) + { + std::fill_n(pageList[batch][beam][kv], nbPoisonPages, kPoisonPageIdx); + } + } + } +#endif + } +#endif + // Allocate the attention sinks (per head) auto attentionSinks = ManagedMemBuf(nbQHeads); // The attention sinks ptr. @@ -1255,6 +1289,7 @@ TEST(RefCheck, llama_V2_70b_3) #endif } + #endif #else @@ -1594,3 +1629,16 @@ TEST(NVRTC, compile) } #endif #endif + +#if SLIDING_WINDOW && USE_PAGED_KV_CACHE && !IS_MLA +#if !SPEC_DEC || (!IS_SPEC_DEC_TREE && !defined(SPEC_Q_SEQ_LEN)) +TEST(RefCheck, sliding_window_invalid_prefix_pages) +{ +#if SPEC_DEC + runTest<1, HEAD_GRP_SIZE, 3>(16, 256 + 57, false, true, false, false, false, ~0U, 128); +#else + runTest<1>(16, 256 + 57, false, true, false, false, false, ~0U, 128); +#endif +} +#endif +#endif diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index da27c539bd09..32e582d22683 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -3,9 +3,6 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[Fals accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6184575) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] SKIP (https://nvbugs/6117811) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] SKIP (https://nvbugs/6117811) -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6117811) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6094102) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph SKIP (https://nvbugs/5772995) From 6b36b0e9f724d30b155221396a0222ab3ef960f2 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Mon, 1 Jun 2026 03:11:15 +0000 Subject: [PATCH 154/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- security_scanning/examples/ray_orchestrator/poetry.lock | 6 +++--- security_scanning/metadata.json | 4 ++-- security_scanning/poetry.lock | 8 ++++---- security_scanning/pyproject.toml | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index cfd8cf12a2ab..02cdf3669285 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -2279,14 +2279,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.4.1" +version = "21.4.2" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12"}, - {file = "virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2"}, + {file = "virtualenv-21.4.2-py3-none-any.whl", hash = "sha256:854210ca524a1a4d0d744734f4acbc721c3ffe163b85bbf5d56d14d5ae2f0fae"}, + {file = "virtualenv-21.4.2.tar.gz", hash = "sha256:38e6ee0a555615c0ea9da2ac7e9998fe8dc3b911dd33ad8eaad2020957653b0c"}, ] [package.dependencies] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index acbce6f5b0ee..3fafefd4a7d2 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "26c099f52ddac75a00dcaa6f172eff334f14be5f", - "timestamp": "2026-05-31T02:46:53Z" + "commit_hash": "5f5b77239ad5ccad19a944e40aee9ef7d6016101", + "timestamp": "2026-06-01T02:47:52Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index 3d8379d82af8..873a7edd0167 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -5246,14 +5246,14 @@ cli = ["click (>=5.0)"] [[package]] name = "python-multipart" -version = "0.0.29" +version = "0.0.30" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69"}, - {file = "python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904"}, + {file = "python_multipart-0.0.30-py3-none-any.whl", hash = "sha256:830964def8c90607ac5daa00514e3987815865713ade8d20febc9177ac0c3c5b"}, + {file = "python_multipart-0.0.30.tar.gz", hash = "sha256:0edfe0475c1f46ddd3ff7785a626f6118af32bdcf359bb21260367313bb32118"}, ] [[package]] @@ -7474,4 +7474,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "0eb5c9722f31b299584cd13daae11eadb5c6a5eee4090d760ee53aada0623eab" +content-hash = "26cbc35ce62c9e6822456f64742b16d54b72a783b19a2ebb08b9e2fc2a5a4bf1" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index f0c4c5c2dad3..c0cdd6e82f90 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -82,7 +82,7 @@ dependencies = [ "cuda-tile (>=1.0.1)", "nvidia-cuda-tileiras (>=13.1,<13.2)", "etcd-sdk-python (==0.0.7)", - "python-multipart (>=0.0.29,<0.0.30)", + "python-multipart (>=0.0.30,<0.0.31)", "smg-grpc-proto (>=0.4.2)", "cache-dit (>=1.3.5)" ] From 7a9c1865de6fa3dcfb20ec7e0b33ed23a5ebfb3b Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:29:59 +0800 Subject: [PATCH 155/308] [None][feat] Tune mamba config by env variables (#14730) Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../_torch/modules/mamba/ssd_combined.py | 33 ++++++++++++++----- tensorrt_llm/_torch/pyexecutor/_util.py | 6 ++-- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/_torch/modules/mamba/ssd_combined.py b/tensorrt_llm/_torch/modules/mamba/ssd_combined.py index efb02364d291..c39e7328102c 100644 --- a/tensorrt_llm/_torch/modules/mamba/ssd_combined.py +++ b/tensorrt_llm/_torch/modules/mamba/ssd_combined.py @@ -17,6 +17,7 @@ # limitations under the License. import functools +import os import torch import torch.nn.functional as F @@ -44,6 +45,29 @@ # `_plan_tmem_offsets`). Otherwise we fall back to the Triton reference kernels. _FLASHINFER_SSD_VALID_M_MODES = (64, 128) _FLASHINFER_SSD_VALID_HEAD_DIMS = (64, 128) +_USE_FLASHINFER_SSD_ENV = "TRTLLM_USE_MAMBA_FI_SSD" + + +def _use_flashinfer_ssd(): + # Default is to use Triton SSD prefill. + env_value = os.environ.get(_USE_FLASHINFER_SSD_ENV, "0") + if env_value == "0": + logger.info_once( + f"FlashInfer SSD disabled by {_USE_FLASHINFER_SSD_ENV}=0; " + "using Triton SSD prefill", + key="mamba_fi_ssd_disabled", + ) + return False + elif env_value == "1": + logger.info_once( + f"FlashInfer SSD enabled by {_USE_FLASHINFER_SSD_ENV}=1; " + "using FlashInfer SSD prefill", + key="mamba_fi_ssd_enabled", + ) + return True + else: + raise ValueError( + f"Invalid value for {_USE_FLASHINFER_SSD_ENV}: {env_value}") def _flashinfer_ssd_supported(chunk_size, dstate, headdim): @@ -437,16 +461,9 @@ def mamba_chunk_scan_combined( # FlashInfer fused CUTLASS kernel on Blackwell (SM100+); both varlen and # non-varlen route here based on cu_seqlens. Falls back to Triton when the # MMA tile constraints on (chunk_size, dstate, headdim) aren't met. - # - # DISABLED: The FlashInfer SSD kernel hardcodes state_dtype=bf16 while the - # Triton path accumulates in fp32 (states_in_fp32=True), causing silent - # precision loss during prefill. - # TODO: Re-enable once _get_flashinfer_ssd_kernel uses state_dtype=float32. - _USE_FLASHINFER_SSD_PREFILL = False dstate = B.shape[-1] headdim = x.shape[-1] - flashinfer_eligible = (_USE_FLASHINFER_SSD_PREFILL and z is None - and is_sm_100f()) + flashinfer_eligible = z is None and is_sm_100f() and _use_flashinfer_ssd() if flashinfer_eligible and _flashinfer_ssd_supported( chunk_size, dstate, headdim): return _mamba_chunk_scan_flashinfer_fwd( diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 82f6c137f3e4..f57a520c0d5a 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -1308,14 +1308,14 @@ def _create_kv_cache_manager( # Flashinfer has a SW fallback at any SM. if (stochastic_rounding and mamba_params.mamba_ssm_cache_dtype == torch.float16 - and sm < 100 or sm in (120, 121)): + and (sm < 100 or sm in (120, 121))): logger.info("Replay kernel Philox requires 100 <= sm < 120; " "using legacy MTP path for stochastic rounding support") use_replay = False - # Use replay algorithm for mamba (default is off). + # Use replay algorithm for mamba (default is on). enforce_disable_replay = os.environ.get('TRTLLM_USE_MAMBA_REPLAY', - '0') == '0' + '1') == '0' if enforce_disable_replay: logger.info( "Replay kernel is disabled by TRTLLM_USE_MAMBA_REPLAY=0") From cde996386b3358be09050c7b1356df95ef171b01 Mon Sep 17 00:00:00 2001 From: fredricz-20070104 <226039983+fredricz-20070104@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:35:49 +0800 Subject: [PATCH 156/308] [None][test] Update moe backend for ctx and acceptance length env (#14803) Signed-off-by: FredricZ-2007 <226039983+fredricz-20070104@users.noreply.github.com> --- .../aggregated/config_database_b200_nvl.yaml | 18 +++++++++--------- .../aggregated/config_database_h200_sxm.yaml | 4 ++-- .../deepseek_r1_fp4_v2_2_nodes_blackwell.yaml | 4 ++-- ...seek_r1_fp4_v2_2_nodes_grace_blackwell.yaml | 4 ++-- .../deepseek_r1_fp4_v2_blackwell.yaml | 4 ++-- .../deepseek_r1_fp4_v2_grace_blackwell.yaml | 6 +++--- .../aggregated/deepseek_r1_fp8_blackwell.yaml | 4 ++-- .../aggregated/deepseek_v32_fp4_blackwell.yaml | 2 +- .../deepseek_v32_fp4_grace_blackwell.yaml | 4 ++-- ...seek_r1_fp4_v2_2_nodes_grace_blackwell.yaml | 4 ++-- ...tx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 2 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 2 +- ...1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml | 2 +- ...48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml | 2 ++ ...ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ..._ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...tx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...x1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...tx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml | 2 +- ...tx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml | 2 +- ...ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml | 2 +- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml | 2 +- ...1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml | 2 +- 48 files changed, 66 insertions(+), 64 deletions(-) diff --git a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml index 3747dd2b0cc6..a18d0eec4323 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml @@ -68,7 +68,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -174,7 +174,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc2048_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -247,7 +247,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc32_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -286,7 +286,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc2048_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -494,7 +494,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc1024_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -532,7 +532,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc2048_gpu4 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario @@ -638,7 +638,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc512_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp4_v2 gpus: 8 match_mode: scenario @@ -675,7 +675,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc1024_gpu4 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario @@ -714,7 +714,7 @@ server_configs: backend: openai streaming: true - name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc2048_gpu4 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp4_v2 gpus: 4 match_mode: scenario diff --git a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml index 165680ce14ac..da6be0b5fccd 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml @@ -68,7 +68,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario @@ -175,7 +175,7 @@ server_configs: backend: openai streaming: true - name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc512_gpu8 - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: deepseek_r1_0528_fp8 gpus: 8 match_mode: scenario diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml index 217fb82d6e33..3b5898aff80b 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_blackwell.yaml @@ -7,7 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP16 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep16_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 16 moe_expert_parallel_size: 16 @@ -44,7 +44,7 @@ server_configs: # 8k1k configs - DEP16 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep16_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 16 moe_expert_parallel_size: 16 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index 19bafb9ffe88..3c2325f0f496 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -7,7 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -45,7 +45,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml index 268389e6c1bf..873e874eb09b 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml @@ -39,7 +39,7 @@ server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -108,7 +108,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml index 8d3623c18c34..0ae9dc8debbc 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml @@ -8,7 +8,7 @@ hardware: server_configs: # 1k1k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -109,7 +109,7 @@ server_configs: # 8k1k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -210,7 +210,7 @@ server_configs: # 1k8k configs - DEP4 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep4_mtp1_1k8k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" tensor_parallel_size: 4 moe_expert_parallel_size: 4 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml index f6509f0ffeae..3b3f9647417d 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml @@ -39,7 +39,7 @@ server_configs: # 1k1k configs - DEP8 with DEEPGEMM, MTP1 - name: "r1_fp8_dep8_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 8 @@ -108,7 +108,7 @@ server_configs: # 8k1k configs - DEP8 with DEEPGEMM, MTP1 - name: "r1_fp8_dep8_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp8" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml index 108f5f7cac52..e764ada25c18 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml @@ -39,7 +39,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "v32_fp4_dep8_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_v32_fp4" tensor_parallel_size: 8 moe_expert_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml index 77554cd50771..cd2d3cd56d73 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml @@ -39,7 +39,7 @@ server_configs: # 1k1k configs - DEP4 with CUTEDSL, MTP1 - name: "v32_fp4_dep4_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 @@ -108,7 +108,7 @@ server_configs: # 8k1k configs - DEP4 with CUTEDSL, MTP1 - name: "v32_fp4_dep4_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_v32_fp4" tensor_parallel_size: 4 moe_expert_parallel_size: 4 diff --git a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index b67170b6f52d..09f8afed86b5 100644 --- a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -7,7 +7,7 @@ hardware: server_configs: # 1k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_1k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 @@ -45,7 +45,7 @@ server_configs: # 8k1k configs - DEP8 with CUTEDSL, MTP1 - name: "r1_fp4_v2_dep8_mtp1_8k1k" - server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" model_name: "deepseek_r1_0528_fp4_v2" trust_remote_code: true tensor_parallel_size: 8 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index a1cae7e00b90..37ca84fd86e2 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 0c479ed61ce8..2eb4999a1d50 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 9014fd9b86b5..d39d01edb69b 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index c185f1c1a28a..92cae48ef671 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 2ed04b60d6f4..863bed81809d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 6f9aaf5a11a5..51a7a78e16cc 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 91c7d66ecc69..927657daa04e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index d790464b256a..ba4a156fba23 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index e0b6133ae940..b4aea34d4745 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index d267f10570a6..24794f026fd5 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index abb0195ae2fe..795a6192bf41 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 719401153151..5cc6c81a78b9 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index dd30d2b390d5..1176fd8e5982 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 7bbe7d158147..b37e2855173b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index c22d57a60b3a..091c76a214e4 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index aa0f97a859a3..b41688f89aa9 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 8b835dafdd08..a1dc23f751dc 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 87b7e8bab148..cc82fc3112de 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 4462db6324aa..1e27a99a6798 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index b517a04bc69e..657333376548 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 33b4cb397858..4709cb8b6d59 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 4b24f9aa580e..04e74a93b974 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index f0a97545cf0d..45ab3d59c928 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 89ec0ebd6acb..81796ae2fd04 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index f08e6ef8c0eb..38fc1c8e88f7 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 041ff3fb130e..34974af25222 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml index c75b02889bd3..6061cd6f79cf 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml @@ -100,6 +100,8 @@ worker_config: enable_block_reuse: false free_gpu_memory_fraction: 0.85 dtype: fp8 + moe_config: + backend: CUTEDSL cache_transceiver_config: max_tokens_in_buffer: 8320 backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index b626ef0af46c..388a98538a54 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml index 8b8814b0c968..68210344a755 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index ef5b0242b386..9c57f7c60524 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index ad0501ef145c..5e8a2dc5bfb1 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 39d7e2edb5d4..16bb6e78b923 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 42234d195b27..1e35e7e6d1a4 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 66b3869debda..8c62df6d0af7 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index b701c83eac69..bbe406f38964 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 2e9e45b7b915..b2bb6ab6fed2 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 31c2d8923c4b..9ee99dff719a 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 3e9994e21151..eef32468c274 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -35,7 +35,7 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false From 35cce747343d83c8a4f9f8fd048e29d40f14f2d7 Mon Sep 17 00:00:00 2001 From: fredricz-20070104 <226039983+fredricz-20070104@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:26:28 +0800 Subject: [PATCH 157/308] [None][test] Update precision of previous device step time (#14809) Signed-off-by: FredricZ-2007 <226039983+fredricz-20070104@users.noreply.github.com> --- tests/integration/defs/perf/test_perf_sanity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 818ccd955733..bf60df583d0d 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1320,9 +1320,7 @@ def run_cmd(self, server_idx: int) -> List[str]: start_offsets=gen_log_start_offsets, ) if device_step_time_mean is not None: - summary_line = ( - f"Average Per Iter Device Step Time (ms): {device_step_time_mean}" - ) + summary_line = f"Average Per Iter Device Step Time (ms): {device_step_time_mean:.2f}" with open(benchmark_file_path, "a") as benchmark_ctx: benchmark_ctx.write(f"\n{summary_line}\n") output = f"{output}\n{summary_line}\n" From 0b96e3a3b4069326f4f7048c936bacaae4299336 Mon Sep 17 00:00:00 2001 From: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:08:02 +0800 Subject: [PATCH 158/308] [None][infra] Waive 12 failed cases for main in post-merge 2749 (#14802) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 32e582d22683..8d20bc229c5a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -12,10 +12,14 @@ accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16 SKIP (https://nvbu accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it SKIP (https://nvbugs/6194934) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True] SKIP (https://nvbugs/6240561) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False] SKIP (https://nvbugs/6248764) +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False] SKIP (https://nvbugs/6248769) accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] SKIP (https://nvbugs/6245279) accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) +accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6248757) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp_trtllm] SKIP (https://nvbugs/6191524) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6018046) @@ -62,6 +66,7 @@ accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughpu accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput_trtllm] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5772993) accuracy/test_llm_api_pytorch.py::TestGLM4_6::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5772360) +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash SKIP (https://nvbugs/6156233) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-two_model-no_overlap_scheduler] SKIP (https://nvbugs/6211880) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[two_model] SKIP (https://nvbugs/5596343) @@ -75,6 +80,8 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) +accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] SKIP (https://nvbugs/6248837) +accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8_attn_dp] SKIP (https://nvbugs/6144270) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) @@ -112,6 +119,7 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_c accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6163033) +accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype SKIP (https://nvbugs/6076767) @@ -120,6 +128,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_on-trtllm] SKIP (https://nvbugs/6094068) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] SKIP (https://nvbugs/6177390) +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True] SKIP (https://nvbugs/6248783) accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False] SKIP (https://nvbugs/6212252) accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True] SKIP (https://nvbugs/6210714) accuracy/test_llm_api_pytorch.py::TestQwen3_5_9B::test_bf16[mtp_off] SKIP (https://nvbugs/6212250) @@ -128,6 +137,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_firs accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) +accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6248744) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6094070) cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689) @@ -358,6 +368,8 @@ unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) unittest/llmapi/test_llm_multi_gpu.py -m "gpu4 and part0" SKIP (https://nvbugs/5348958) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_phi3_lora_fused_modules_output_on_tp2_identical_to_tp1 SKIP (https://nvbugs/6109745) +unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora[None] SKIP (https://nvbugs/6248776) +unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora[cuda_graph_config0] SKIP (https://nvbugs/6248776) unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781) unittest/tools/test_layer_wise_benchmarks.py::test_performance_alignment[1] SKIP (https://nvbugs/6127669) unittest/tools/test_layer_wise_benchmarks.py::test_qwen3_next_gen_tep[1] SKIP (https://nvbugs/6153575) From d2fd17b03ca9230ac5f61fe70f6daf620b88e241 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 1 Jun 2026 15:25:41 +0800 Subject: [PATCH 159/308] [TRTLLM-12971][infra] Fix parse classname logic in timeout result (#14559) Signed-off-by: yiqingy Co-authored-by: yiqingy --- jenkins/scripts/generate_timeout_xml.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins/scripts/generate_timeout_xml.py b/jenkins/scripts/generate_timeout_xml.py index 2efd8ccb1bb7..ae608cc0d3bd 100644 --- a/jenkins/scripts/generate_timeout_xml.py +++ b/jenkins/scripts/generate_timeout_xml.py @@ -64,9 +64,7 @@ def parse_xml_classname_name_file_from_testname(testname, stage_name): ) else: classname = stage_name + "." + testname.split("::")[0].replace(".py", "").replace("/", ".") - if testname.startswith("accuracy/") or ( - testname.startswith("examples/") and "[" not in testname - ): + if testname.startswith("accuracy/"): classname = "" return classname, name, file From 3ec9e9b04cafbadb959195ceca7fd1fe1caad720 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:00:49 +0800 Subject: [PATCH 160/308] [https://nvbugs/6038228][fix] Propagate event loop errors to await_responses callers (#12735) Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 69 +++++-- tensorrt_llm/executor/base_worker.py | 96 +++++++++- .../_torch/executor/test_py_executor.py | 168 ++++++++++++++++++ .../test_event_loop_error_broadcast.py | 157 ++++++++++++++++ 4 files changed, 474 insertions(+), 16 deletions(-) create mode 100644 tests/unittest/executor/test_event_loop_error_broadcast.py diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index deabcb756e1c..739484a59b64 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -395,6 +395,15 @@ def __init__( self.response_cv = threading.Condition(self.response_lock) self.responses = {} self.result_wait_queues = {} + # If the event-loop thread (PyExecutor._event_loop_wrapper) raises + # an exception (e.g. KV cache OOM), it is stashed here and read by + # two local consumers so they can surface it instead of hanging: + # - _await_single_response re-raises it as a RuntimeError, and + # - BaseWorker.AwaitResponseHelper (the bridge between this + # engine and per-request GenerationResult queues) reads it to + # broadcast an ErrorResponse to every pending request, waking + # callers parked in queue.get() / aqueue.get(). + self._event_loop_error: Optional[BaseException] = None # kv cache events self.kv_cache_manager = self.resource_manager.resource_managers.get( @@ -806,6 +815,14 @@ def _event_loop_wrapper(self): except Exception as e: logger.error(f"Error in event loop: {e}") logger.error(traceback.format_exc()) + # Stash the original error so local consumers + # (_await_single_response and BaseWorker.AwaitResponseHelper) + # can surface it instead of letting callers hang. We do NOT + # call _handle_errors / _enqueue_responses here: they trigger + # tp_gather / allgather collectives that would deadlock when + # only this rank crashed. The is_shutdown notification in + # _executor_loop_cleanup is enough to wake local waiters. + self._event_loop_error = e raise e finally: self._executor_loop_cleanup() @@ -1743,18 +1760,29 @@ def _process_iter_stats( prev_device_step_time_ms=prev_device_step_time_ms) def _executor_loop_cleanup(self): - - for i in range(self.num_micro_batches): - self.wait_on_pp_send_handles(self.send_handles, i) - self.wait_on_pp_send_handles(self.send_schedule_handles, i) - self.wait_on_pp_send_handles(self.send_expected_batch_num_handles, - i) - + # Wake any waiters in await_responses BEFORE potentially-blocking + # work below. If wait_on_pp_send_handles hangs (e.g. after a + # crash leaves PP send handles in a bad state), the await loop + # must not hang with it. with self.response_cv: self.is_shutdown = True self.response_cv.notify_all() self.shutdown_event.set() + for i in range(self.num_micro_batches): + try: + self.wait_on_pp_send_handles(self.send_handles, i) + self.wait_on_pp_send_handles(self.send_schedule_handles, i) + self.wait_on_pp_send_handles( + self.send_expected_batch_num_handles, i) + except Exception: + # PP send handles may be in a broken state after an + # event-loop crash. Log and continue; the waiters have + # already been notified above. + logger.error( + f"Error waiting on PP send handles during cleanup: " + f"{traceback.format_exc()}") + def _pp_schedule_and_propagate(self, microbatch_id: int): """The first PP rank schedules the requests and propagates the result to all other PP ranks.""" @@ -4714,12 +4742,31 @@ def _await_single_response( with self.response_cv: def key_has_response(): - return id in self.responses.keys() + # Wake on shutdown too so that an event-loop crash + # cannot trap callers here forever (nvbug 6038228). + return id in self.responses or self.is_shutdown self.response_cv.wait_for(key_has_response, timeout=timeout) - response = self.responses[id] - self.responses.pop(id) - return response + if id in self.responses: + return self.responses.pop(id) + if self.is_shutdown: + # The event-loop thread terminated before producing a + # response for this request. Re-raise the original + # exception (if any) so callers see a meaningful error + # instead of hanging here or hitting a KeyError below. + error = self._event_loop_error + if error is not None: + raise RuntimeError( + f"Event loop terminated with error: {error}") from error + raise RuntimeError( + f"Event loop shut down before a response was received " + f"for request {id}.") + # Timed out without shutdown. Return [] to match the + # documented timeout contract used by _await_any_response and + # the other executor-API timeouts; the pre-fix code raised a + # bare KeyError on this path, which all known callers had to + # defend against anyway. + return [] def _terminate_requests(self, requests_to_terminate): # todo: support work with self.inflight_req_ids. diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index d8beef865f9b..133fdd858a85 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -359,9 +359,11 @@ def _set_iteration_result_queue(self, it_result_queue: IterationResultQueue, it_result_queue.aqueue = None def return_queue(self, client_id: int): - """ If a centralized result queue is registered (used for communication with the proxy) - send the message there. - Otherwise, push the result directly in the GenerationResult queue. + """Return the queue used to deliver responses for ``client_id``. + + If a centralized result queue is registered (used for communication + with the proxy) send the message there. Otherwise, push the result + directly in the GenerationResult queue. """ if self.result_queue is not None: return self.result_queue @@ -964,8 +966,19 @@ def responses_handler(self, responses: List[tllm.Response]): def __call__(self, timeout: Optional[float] = None) -> bool: ''' This method should be called by a ManagedThread. ''' timeout = timeout or 0.1 - responses = self.worker.engine.await_responses( - timeout=datetime.timedelta(seconds=timeout)) + try: + responses = self.worker.engine.await_responses( + timeout=datetime.timedelta(seconds=timeout)) + except Exception as e: + # Defensive: with id=None, PyExecutor.await_responses routes + # to _await_any_response, which does not raise on event-loop + # crash — it returns [] silently and we detect the crash + # via engine._event_loop_error after this block. But any + # unexpected exception out of await_responses (e.g. from a + # different engine implementation, or a future change to + # _await_any_response) is also a clear signal to broadcast + # and stop the thread. + return self._broadcast_event_loop_error(e) # filter since The _engine_response_callback may return None responses = list( filter( @@ -980,8 +993,81 @@ def __call__(self, timeout: Optional[float] = None) -> bool: color="red", category="Worker"): self.responses_handler(responses) + + # Even when await_responses returned normally (e.g. via + # _await_any_response, whose predicate already includes + # is_shutdown but does not raise), an event-loop crash leaves + # _event_loop_error stashed on the engine. Broadcast and stop the + # thread in that case too — see nvbug 6038228. + error = getattr(self.worker.engine, "_event_loop_error", None) + if error is not None: + return self._broadcast_event_loop_error(error) return True + def _broadcast_event_loop_error(self, error: BaseException) -> bool: + """Wake every pending ``GenerationResult`` after an event-loop crash. + + Inject an ``ErrorResponse`` into every pending ``GenerationResult`` + queue so callers parked in ``queue.get()`` / ``aqueue.get()`` + (``LLM.generate``, ``generate_async`` + ``aresult``, + ``trtllm-bench``) wake up with a meaningful error instead of + hanging when the PyExecutor event loop dies. + + Returns ``False`` so the calling ``ManagedThread`` exits — there + is no point polling a dead engine. + + Scope: single-process worker (the path that backs ``LLM.generate`` + and the bench async client). The IPC / proxy path tracks pending + results on a different side of the boundary and would need a + separate poison-pill on ``self.worker.result_queue``; that is left + as a follow-up consistent with the PyExecutor-side fix. + """ + error_msg = f"Event loop terminated with error: {error}" + pending_client_ids = list(self.worker._results.keys()) + if not pending_client_ids: + logger.error( + f"Event-loop error with no pending results to wake: {error}") + return False + + logger.error( + f"Broadcasting event-loop error to {len(pending_client_ids)} " + f"pending request(s): {error}") + + event_loop = None + async_queues: List[_SyncQueue] = [] + for client_id in pending_client_ids: + try: + queue = self.worker.return_queue(client_id) + except KeyError: + continue + err_resp = ErrorResponse( + client_id=client_id, + error_msg=error_msg, + request_id=client_id, + ) + try: + if isinstance(queue, _SyncQueue): + queue.put_nowait(err_resp) + async_queues.append(queue) + event_loop = event_loop or queue.loop + else: + queue.put(err_resp) + except Exception as put_error: + logger.error(f"Failed to push ErrorResponse for client_id=" + f"{client_id}: {put_error}") + continue + self.worker._pop_result(client_id) + + if async_queues: + try: + _SyncQueue.notify_many(event_loop, async_queues) + except Exception as notify_error: + logger.error( + f"Failed to notify async queues on event-loop error: " + f"{notify_error}") + + return False + def handle_for_worker(self, responses: List[tllm.Response]) -> None: ''' Return the responses to asyncio.event_loop. ''' event_loop = None diff --git a/tests/unittest/_torch/executor/test_py_executor.py b/tests/unittest/_torch/executor/test_py_executor.py index 25ef3f9afde2..2e70be239cc6 100644 --- a/tests/unittest/_torch/executor/test_py_executor.py +++ b/tests/unittest/_torch/executor/test_py_executor.py @@ -7,8 +7,11 @@ - waiting_queue management - is_shutdown state management - expected_num_active_requests tracking +- Event-loop crash propagation to await_responses callers (nvbug 6038228) """ +import threading +import time from unittest.mock import Mock import pytest @@ -383,3 +386,168 @@ def test_multiple_ctx_requests_mixed_chunks(self): context_chunk_size=50, context_remaining_length=50, estimated_reusable_tokens=10 ) assert PyExecutor._compute_scheduled_tokens([req0, req1], []) == 20 + 40 + + +# --------------------------------------------------------------------------- +# Tests for event-loop crash propagation to _await_single_response callers. +# +# nvbug 6038228: when PyExecutor._event_loop_wrapper crashed (e.g. KV cache +# OOM), the main thread parked in _await_single_response would block forever +# because is_shutdown was never set / observed by the wait predicate. The fix +# stashes the original error in self._event_loop_error, sets is_shutdown + +# notifies in _executor_loop_cleanup, and re-raises the error from +# _await_single_response so callers exit promptly with a meaningful message. +# +# We exercise the actual PyExecutor methods by binding them to a lightweight +# stub that carries only the attributes those methods touch. +# --------------------------------------------------------------------------- + + +class _ResponseStub: + """Minimal stub carrying only the state used by _await_single_response.""" + + def __init__(self): + self.response_lock = threading.Lock() + self.response_cv = threading.Condition(self.response_lock) + self.responses = {} + self.is_shutdown = False + self._event_loop_error = None + + # Bind the real production method so the test exercises real code. + _await_single_response = PyExecutor._await_single_response + + +class TestAwaitSingleResponseShutdown: + """_await_single_response must not block forever when the event loop dies.""" + + def test_returns_response_when_available(self): + """Normal path: response exists, returned and consumed.""" + stub = _ResponseStub() + stub.responses = {7: ["resp_a", "resp_b"]} + + result = stub._await_single_response(id=7, timeout=1.0) + assert result == ["resp_a", "resp_b"] + assert 7 not in stub.responses + + def test_returns_response_even_during_shutdown(self): + """If a response was enqueued before shutdown it is still returned; + the shutdown branch only fires when nothing is queued for this id.""" + stub = _ResponseStub() + stub.is_shutdown = True + stub._event_loop_error = RuntimeError("crash") + stub.responses = {7: ["resp"]} + + result = stub._await_single_response(id=7, timeout=1.0) + assert result == ["resp"] + + def test_raises_on_shutdown_with_event_loop_error(self): + """When the event loop crashed, _await_single_response surfaces the + original error as RuntimeError instead of hanging.""" + stub = _ResponseStub() + stub.is_shutdown = True + stub._event_loop_error = RuntimeError("KV cache OOM") + + with pytest.raises(RuntimeError, match="Event loop terminated"): + stub._await_single_response(id=42, timeout=1.0) + + def test_raises_on_shutdown_without_event_loop_error(self): + """Shutdown without a stored error still raises rather than blocking + — distinguishes "shutdown" from "timed out without shutdown".""" + stub = _ResponseStub() + stub.is_shutdown = True + + with pytest.raises(RuntimeError, match="Event loop shut down"): + stub._await_single_response(id=42, timeout=1.0) + + def test_returns_empty_on_timeout(self): + """Pre-fix behaviour: a bare timeout (no shutdown, no response) used + to KeyError. The fix returns an empty list to match the documented + timeout contract used elsewhere in the executor API.""" + stub = _ResponseStub() + result = stub._await_single_response(id=99, timeout=0.01) + assert result == [] + + def test_wakes_up_when_shutdown_set_from_another_thread(self): + """Real-world scenario: main thread is parked in + _await_single_response while the event-loop thread crashes and + triggers _executor_loop_cleanup, which sets is_shutdown + notifies. + The waiter must wake and re-raise.""" + stub = _ResponseStub() + original_error = RuntimeError("simulated event-loop crash") + + def crash_after_delay(): + time.sleep(0.05) + stub._event_loop_error = original_error + with stub.response_cv: + stub.is_shutdown = True + stub.response_cv.notify_all() + + crash_thread = threading.Thread(target=crash_after_delay, daemon=True) + crash_thread.start() + + with pytest.raises(RuntimeError, match="Event loop terminated"): + stub._await_single_response(id=1, timeout=5.0) + + crash_thread.join(timeout=1.0) + + +# --------------------------------------------------------------------------- +# Tests for _executor_loop_cleanup ordering (notify before PP wait). +# --------------------------------------------------------------------------- + + +class _CleanupStub: + """Stub for _executor_loop_cleanup: records the order in which the + shutdown notification and PP-handle wait happen.""" + + def __init__(self, pp_handles_raise=False): + self.response_lock = threading.Lock() + self.response_cv = threading.Condition(self.response_lock) + self.is_shutdown = False + self.shutdown_event = threading.Event() + self.num_micro_batches = 1 + self.send_handles = {} + self.send_schedule_handles = {} + self.send_expected_batch_num_handles = {} + self._pp_handles_raise = pp_handles_raise + self._events: list = [] + + original_notify = self.response_cv.notify_all + + def record_notify(): + self._events.append("notify_all") + original_notify() + + self.response_cv.notify_all = record_notify + + def wait_on_pp_send_handles(self, handles, idx): + self._events.append(f"wait_pp_{idx}") + if self._pp_handles_raise: + raise RuntimeError("PP send handle in bad state") + + _executor_loop_cleanup = PyExecutor._executor_loop_cleanup + + +class TestExecutorLoopCleanup: + """Cleanup must wake waiters BEFORE doing potentially-blocking PP work, + and a PP-handle exception must not skip the shutdown notification.""" + + def test_notify_happens_before_pp_wait(self): + stub = _CleanupStub() + stub._executor_loop_cleanup() + + assert stub._events[0] == "notify_all" + assert "wait_pp_0" in stub._events + assert stub._events.index("notify_all") < stub._events.index("wait_pp_0") + assert stub.is_shutdown is True + assert stub.shutdown_event.is_set() + + def test_pp_wait_exception_does_not_skip_notify(self): + """If wait_on_pp_send_handles raises, the shutdown notification + must still have happened (it ran first), and cleanup must not + propagate the error so the executor thread terminates cleanly.""" + stub = _CleanupStub(pp_handles_raise=True) + stub._executor_loop_cleanup() + + assert stub.is_shutdown is True + assert "notify_all" in stub._events diff --git a/tests/unittest/executor/test_event_loop_error_broadcast.py b/tests/unittest/executor/test_event_loop_error_broadcast.py new file mode 100644 index 000000000000..04851b8e651b --- /dev/null +++ b/tests/unittest/executor/test_event_loop_error_broadcast.py @@ -0,0 +1,157 @@ +"""Unit tests for AwaitResponseHelper.__call__ event-loop crash handling. + +When PyExecutor's event-loop thread dies (e.g. KV cache OOM), every +pending ``GenerationResult`` parked in ``queue.get()`` / ``aqueue.get()`` +must wake up with a meaningful ``ErrorResponse`` rather than hang +forever. See nvbug 6038228 and PR #12735. + +These tests bind the real ``AwaitResponseHelper.__call__`` / +``_broadcast_event_loop_error`` to lightweight stubs, so they need +neither GPUs nor models. +""" + +import datetime +import queue as _stdlib_queue + +from tensorrt_llm.executor.base_worker import AwaitResponseHelper +from tensorrt_llm.executor.utils import ErrorResponse + + +class _EngineStub: + """Stub for self.worker.engine: returns whatever the test plugged in.""" + + def __init__( + self, + await_responses_result=None, + await_responses_raises=None, + event_loop_error=None, + is_shutdown=False, + ): + self._await_responses_result = await_responses_result or [] + self._await_responses_raises = await_responses_raises + self._event_loop_error = event_loop_error + self.is_shutdown = is_shutdown + self.calls = 0 + + def await_responses(self, timeout: datetime.timedelta): + self.calls += 1 + if self._await_responses_raises is not None: + raise self._await_responses_raises + return list(self._await_responses_result) + + +class _ResultStub: + """Minimal GenerationResult: just exposes a queue.""" + + def __init__(self): + self.queue = _stdlib_queue.Queue() + + +class _WorkerStub: + """Stub for BaseWorker exposing only the attributes the helper touches.""" + + def __init__(self, engine, num_pending: int = 1): + self.engine = engine + self._results = {cid: _ResultStub() for cid in range(1, num_pending + 1)} + self.popped = [] + self.result_queue = None + self.postproc_queues = None + + # Echoed straight back so __call__'s filter is a no-op. + def _engine_response_callback(r): + return r + + self._engine_response_callback = _engine_response_callback + + def return_queue(self, client_id: int): + return self._results[client_id].queue + + def _pop_result(self, client_id: int): + self.popped.append(client_id) + self._results.pop(client_id, None) + + +def _make_helper(engine, num_pending: int = 1): + helper = AwaitResponseHelper.__new__(AwaitResponseHelper) + helper.worker = _WorkerStub(engine, num_pending=num_pending) + helper.handler_kind = AwaitResponseHelper.HandlerKind.unknown + helper.enable_postprocprocess_parallel = False + helper.temp_error_responses = _stdlib_queue.Queue() + return helper + + +class TestAwaitResponseHelperEventLoopError: + def test_normal_path_returns_true(self): + """No engine error and no responses: ManagedThread should keep going.""" + engine = _EngineStub(await_responses_result=[]) + helper = _make_helper(engine, num_pending=1) + + assert helper(timeout=0.01) is True + # No ErrorResponse should have been pushed. + for rs in helper.worker._results.values(): + assert rs.queue.empty() + assert helper.worker.popped == [] + + def test_broadcasts_when_await_responses_raises(self): + """Defensive: any exception out of engine.await_responses triggers broadcast.""" + original = RuntimeError("Event loop terminated with error: KV OOM") + engine = _EngineStub(await_responses_raises=original) + helper = _make_helper(engine, num_pending=2) + + assert helper(timeout=0.01) is False # ManagedThread should stop + + # Each pending GenerationResult got an ErrorResponse. + for cid in (1, 2): + err = helper.worker._results.get(cid) + assert err is None, "result should have been popped" + # popped order is iteration order over dict keys (insertion order in py3.7+) + assert sorted(helper.worker.popped) == [1, 2] + + def test_broadcasts_when_event_loop_error_set_after_empty_response(self): + """Broadcast must fire even when await_responses returns [] silently. + + ``_await_any_response`` returns ``[]`` on shutdown without raising, + but ``_event_loop_error`` is still stashed on the engine. + """ + original = RuntimeError("KV cache OOM") + engine = _EngineStub(await_responses_result=[], event_loop_error=original, is_shutdown=True) + helper = _make_helper(engine, num_pending=3) + + assert helper(timeout=0.01) is False + assert sorted(helper.worker.popped) == [1, 2, 3] + + def test_pushed_response_is_error_response_with_message(self): + """Pushed item is an ErrorResponse carrying the original error text.""" + original = RuntimeError("KV cache OOM at iteration 42") + engine = _EngineStub(event_loop_error=original, is_shutdown=True) + # Capture queue refs before they get popped from _results. + helper = _make_helper(engine, num_pending=1) + result_queue = helper.worker.return_queue(client_id=1) + + helper(timeout=0.01) + + item = result_queue.get_nowait() + assert isinstance(item, ErrorResponse) + assert item.client_id == 1 + assert "KV cache OOM" in item.error_msg + assert "Event loop terminated" in item.error_msg + + def test_no_pending_results_returns_false_quietly(self): + """Crash with no pending requests still stops the thread cleanly.""" + original = RuntimeError("crash") + engine = _EngineStub(event_loop_error=original, is_shutdown=True) + helper = _make_helper(engine, num_pending=0) + + assert helper(timeout=0.01) is False + assert helper.worker.popped == [] + + def test_broadcast_helper_idempotent_via_pop(self): + """Calling _broadcast_event_loop_error twice is safe (second is a no-op).""" + original = RuntimeError("crash") + engine = _EngineStub(event_loop_error=original, is_shutdown=True) + helper = _make_helper(engine, num_pending=2) + + assert helper._broadcast_event_loop_error(original) is False + assert sorted(helper.worker.popped) == [1, 2] + # second time around: nothing left to wake. + assert helper._broadcast_event_loop_error(original) is False From 71a188ccd8c51df0dda73fa93e9dad6989ff6820 Mon Sep 17 00:00:00 2001 From: Jian Tu <107457950+JadoTu@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:15:02 +0800 Subject: [PATCH 161/308] [TRTLLM-12288][feat] Support Nemotron-H nvfp4 ckpt on Hopper (#14775) Signed-off-by: jiant <107457950+JadoTu@users.noreply.github.com> --- .../_torch/models/modeling_nemotron_h.py | 129 ++++++- .../modules/fused_moe/fused_moe_cutlass.py | 107 +++++- .../_torch/modules/fused_moe/quantization.py | 78 ++++ .../modules/fused_moe/triton_dequant_nvfp4.py | 334 ++++++++++++++++++ tensorrt_llm/_torch/modules/linear.py | 81 +++++ .../defs/accuracy/test_llm_api_pytorch.py | 29 ++ .../test_lists/test-db/l0_dgx_h100.yml | 1 + 7 files changed, 745 insertions(+), 14 deletions(-) create mode 100644 tensorrt_llm/_torch/modules/fused_moe/triton_dequant_nvfp4.py diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h.py b/tensorrt_llm/_torch/models/modeling_nemotron_h.py index 574c74d9b190..f643eff0ef1e 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re +from contextlib import contextmanager from dataclasses import replace from typing import TYPE_CHECKING @@ -31,6 +33,7 @@ from tensorrt_llm._utils import get_sm_version from tensorrt_llm.logger import logger from tensorrt_llm.lora_helper import LoraConfig +from tensorrt_llm.models.modeling_utils import QuantAlgo # noqa: E402 from ..attention_backend import AttentionMetadata from ..distributed import AllReduce, AllReduceFusionOp, AllReduceParams @@ -39,7 +42,11 @@ from ..modules.decoder_layer import DecoderLayer from ..modules.embedding import Embedding from ..modules.fused_moe import MoEWeightLoadingMode, create_moe -from ..modules.linear import Linear, TensorParallelMode +from ..modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE +from ..modules.fused_moe.quantization import (NVFP4CutlassFusedMoEMethod, + W4A16NVFP4CutlassFusedMoEMethod) +from ..modules.linear import (Linear, NVFP4LinearMethod, TensorParallelMode, + W4A16NVFP4LinearMethod) from ..modules.mamba.mamba2_mixer import Mamba2Mixer from ..modules.mlp import MLP from ..modules.multi_stream_utils import maybe_execute_in_parallel @@ -431,10 +438,14 @@ def __init__( quant_mode = (model_config.quant_config.quant_mode if model_config.quant_config is not None else None) - self.is_nvfp4 = quant_mode is not None and quant_mode.has_nvfp4() + # We don't use the RMSNorm+NVFP4 on SM < 100 + _has_fp4_hw = get_sm_version() >= 100 + self.is_nvfp4 = (quant_mode is not None and quant_mode.has_nvfp4() + and _has_fp4_hw) # For MIXED_PRECISION models, the global quant_mode is QuantMode(0). Check per-layer # quant_config_dict to see if this specific layer is NVFP4-quantized. - if not self.is_nvfp4 and model_config.quant_config_dict is not None: + if (not self.is_nvfp4 and _has_fp4_hw + and model_config.quant_config_dict is not None): layer_prefix = f"model.layers.{layer_idx}." for key, cfg in model_config.quant_config_dict.items(): if key.startswith(layer_prefix) and cfg.quant_mode.has_nvfp4(): @@ -506,6 +517,11 @@ def __init__( ) if fuse_allreduce_norm: self.mixer.out_proj.reduce_output = False + # Hopper: route RMSNormGated to its bf16 Triton fallback + # (fused_gated_rmsnorm_quant is SM100-only). + if not _has_fp4_hw: + self.mixer.is_nvfp4 = False + self.mixer.norm.is_nvfp4 = False elif layer_type == "-": self.mixer = MLPLayer( model_config, @@ -754,6 +770,93 @@ def forward( return hidden_states +def _force_moe_backend_for_w4a16_on_hopper( + model_config: NemotronHModelConfig) -> None: + """SM<100 + NVFP4: force ``moe_backend=CUTLASS`` (only backend with the + W4A16 fallback) and disable attention FP4 output fusion. + """ + if get_sm_version() >= 100: + return + + # NVFP4 may live in global quant_config OR per-layer quant_config_dict + # (MIXED_PRECISION ckpts). + qcfg = model_config.quant_config + has_nvfp4 = qcfg is not None and qcfg.layer_quant_mode.has_nvfp4() + if not has_nvfp4 and model_config.quant_config_dict is not None: + has_nvfp4 = any(cfg.quant_mode.has_nvfp4() + for cfg in model_config.quant_config_dict.values()) + if not has_nvfp4: + return + + # o_proj.has_nvfp4 stays True under W4A16 -- the property reads quant_config. + # Use the documented env override to keep attention output in bf16. + if os.environ.get("TRTLLM_ENABLE_ATTENTION_NVFP4_OUTPUT") != "0": + logger.warning( + f"Nemotron-H SM{get_sm_version()}: TRTLLM_ENABLE_ATTENTION_NVFP4_OUTPUT=0" + ) + os.environ["TRTLLM_ENABLE_ATTENTION_NVFP4_OUTPUT"] = "0" + + if model_config.moe_backend.upper() in ('CUTLASS', 'AUTO'): + return + logger.warning( + f"Nemotron-H SM{get_sm_version()}: forcing moe_backend " + f"'{model_config.moe_backend}' -> 'CUTLASS' for W4A16 fallback") + model_config._frozen = False + model_config.moe_backend = 'CUTLASS' + model_config._frozen = True + + +@contextmanager +def _use_w4a16_for_nvfp4_on_hopper(): + """SM<100 + NVFP4: swap NVFP4 quant methods -> W4A16 fallback, + loosen MoE SM constraint, and disable MLP's fused relu2+FP4 quant. + Class-level patches; model construction is single-threaded today. + """ + if get_sm_version() >= 100: + yield + return + + original_linear = Linear.get_quant_method + original_moe = CutlassFusedMoE._get_quant_method + original_mlp_create_weights = MLP.create_weights + nvfp4_entry = CutlassFusedMoE._QUANT_SUPPORT_TABLE[QuantAlgo.NVFP4] + original_sm_constraint = nvfp4_entry["sm_constraint"] + + def _patched_linear(self, quant_config): + method = original_linear(self, quant_config) + if type(method) is NVFP4LinearMethod: + return W4A16NVFP4LinearMethod() + return method + + def _patched_moe(self): + method = original_moe(self) + if type(method) is NVFP4CutlassFusedMoEMethod: + return W4A16NVFP4CutlassFusedMoEMethod() + return method + + def _patched_mlp_create_weights(self): + # Original sets _use_fused_relu2_quant=True for NVFP4 ckpts; off here + # so MLP.forward emits bf16 (the SM100-only fused kernel never runs). + original_mlp_create_weights(self) + self._use_fused_relu2_quant = False + + # Allow SM 90 through can_implement(); existing entries preserved. + constraint_type, constraint_set = original_sm_constraint + nvfp4_entry["sm_constraint"] = (constraint_type, + frozenset(constraint_set) | {90}) + + Linear.get_quant_method = _patched_linear + CutlassFusedMoE._get_quant_method = _patched_moe + MLP.create_weights = _patched_mlp_create_weights + try: + yield + finally: + Linear.get_quant_method = original_linear + CutlassFusedMoE._get_quant_method = original_moe + MLP.create_weights = original_mlp_create_weights + nvfp4_entry["sm_constraint"] = original_sm_constraint + + @register_auto_model("NemotronHPuzzleForCausalLM") @register_auto_model("NemotronHForCausalLM") class NemotronHForCausalLM(SpecDecOneEngineForCausalLM[NemotronHModel, @@ -797,10 +900,12 @@ def __init__( } model_config._frozen = True - super().__init__( - model=NemotronHModel(model_config), - model_config=model_config, - ) + _force_moe_backend_for_w4a16_on_hopper(model_config) + with _use_w4a16_for_nvfp4_on_hopper(): + super().__init__( + model=NemotronHModel(model_config), + model_config=model_config, + ) self.model_nextn = 0 if (model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model()): @@ -832,6 +937,16 @@ def __init__( self.epilogue.extend(self.draft_model.mtp_layers) self.epilogue.append(self.spec_worker) + def __post_init__(self): + # PostInitCaller metaclass invokes __post_init__ AFTER __init__ returns, + # so our W4A16 context manager from __init__ has already exited. For + # MIXED_PRECISION checkpoints, ``apply_layerwise_quant_config`` rebinds + # per-layer ``quant_config`` to NVFP4 and then re-runs ``create_weights`` + # (see modeling_utils.py:543). Re-enter the context manager so the + # patched ``_get_quant_method`` catches that second pass. + with _use_w4a16_for_nvfp4_on_hopper(): + super().__post_init__() + @staticmethod def _normalize_puzzle_config(config): """Set global MoE defaults from block_configs for models with per-layer MoE params.""" diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py index 5b9574a10065..faa584c6570e 100755 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py @@ -21,13 +21,12 @@ from .quantization import UnquantizedFusedMoEMethod # isort: off -from .quantization import (DeepSeekFP8BlockScalesFusedMoEMethod, - FP8QDQFusedMoEMethod, MoEWeightLoadingMode, - NVFP4CutlassFusedMoEMethod, - INT8WoqPerChannelFusedMoEMethod, - W4A8MXFP4FP8CutlassFusedMoEMethod, - W4A8MXFP4MXFP8CutlassFusedMoEMethod, - WFP4A16FusedMoEMethod, WInt4AFP8FusedMoEMethod) +from .quantization import ( + DeepSeekFP8BlockScalesFusedMoEMethod, FP8QDQFusedMoEMethod, + MoEWeightLoadingMode, NVFP4CutlassFusedMoEMethod, + INT8WoqPerChannelFusedMoEMethod, W4A16NVFP4CutlassFusedMoEMethod, + W4A8MXFP4FP8CutlassFusedMoEMethod, W4A8MXFP4MXFP8CutlassFusedMoEMethod, + WFP4A16FusedMoEMethod, WInt4AFP8FusedMoEMethod) # isort: on from .routing import BaseMoeRoutingMethod @@ -445,6 +444,9 @@ def quantize_input( """ x_sf = None if self.has_any_quant: + # W4A16 NVFP4 path keeps activations hp; skip FP4 quant below. + if isinstance(self.quant_method, W4A16NVFP4CutlassFusedMoEMethod): + return x, None if self.has_fp8_qdq or self.has_w4a8_mxfp4_fp8: x, _ = torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor( x, self.fc31_input_dequant) @@ -599,6 +601,19 @@ def run_moe( Returns: final_hidden_states: Output tensor from MoE computation """ + # W4A16 NVFP4 fallback (SM<100). + if isinstance(self.quant_method, W4A16NVFP4CutlassFusedMoEMethod): + return self._run_moe_w4a16_nvfp4( + x, + token_selected_experts, + token_final_scales, + output_dtype=output_dtype, + tuner_num_tokens=tuner_num_tokens, + tuner_top_k=tuner_top_k, + moe_output=moe_output, + enable_alltoall=enable_alltoall, + ) + # SM120 + FP8 block scales: use Triton kernel (CUTLASS TMA fails on SM120 # for large token counts due to cuTensorMapEncodeTiled limitations). if self.has_deepseek_fp8_block_scales and get_sm_version() == 120: @@ -713,6 +728,84 @@ def run_moe( return final_hidden_states + def _run_moe_w4a16_nvfp4( + self, + x: torch.Tensor, + token_selected_experts: torch.Tensor, + token_final_scales: torch.Tensor, + output_dtype: Optional[torch.dtype] = None, + tuner_num_tokens: Optional[int] = None, + tuner_top_k: Optional[int] = None, + moe_output: Optional[torch.Tensor] = None, + enable_alltoall: Optional[bool] = None, + ) -> torch.Tensor: + """W4A16 fallback for NVFP4 MoE on SM<100. Active-mask dequant into + a static [E_total, N, K] bf16 workspace, then bf16 fused_moe with the + original (global) token_selected_experts. CUDA-graph capturable. + """ + assert isinstance(self.quant_method, W4A16NVFP4CutlassFusedMoEMethod) + + if enable_alltoall is None: + enable_alltoall = self.enable_alltoall + if output_dtype is None: + output_dtype = x.dtype + + # Same EP id convention as the FP8 path above: global ids (or + # ``local_n``-padded under alltoall). Clamp to local range so the + # active-mask scatter is in-bounds; non-local tokens collapse onto a + # boundary expert (1 extra dequant/rank). ``trtllm.fused_moe`` below + # still gets the original global ids -- it does its own remap. + local_n = self.expert_size_per_partition + if enable_alltoall: + local_ids = token_selected_experts.clamp(0, local_n - 1) + else: + local_ids = (token_selected_experts - self.slot_start).clamp( + 0, local_n - 1) + + w3_w1_hp, w2_hp = self.quant_method.dequant_active_experts_to_hp( + self, local_ids, output_dtype) + + # bf16 fused_moe with empty quant_scales (matches unquantized path). + result = torch.ops.trtllm.fused_moe( + x, + token_selected_experts, + token_final_scales, + w3_w1_hp, + self.w3_w1_bias, + w2_hp, + self.w2_bias, + output_dtype, + quant_scales=[], + input_sf=None, + swizzled_input_sf=False, + swiglu_alpha=self.swiglu_alpha, + swiglu_beta=self.swiglu_beta, + swiglu_limit=self.swiglu_limit, + tp_size=self.tp_size, + tp_rank=self.tp_rank, + ep_size=self.ep_size, + ep_rank=self.ep_rank, + cluster_size=self.cluster_size, + cluster_rank=self.cluster_rank, + enable_alltoall=enable_alltoall, + use_deepseek_fp8_block_scale=False, + use_w4_group_scaling=False, + use_int8_woq_per_channel=False, + use_mxfp8_act_scaling=False, + min_latency_mode=False, + use_fused_finalize=self.use_fused_finalize, + tune_max_num_tokens=self.tune_max_num_tokens, + tuner_num_tokens=tuner_num_tokens, + tuner_top_k=tuner_top_k, + activation_type=self.activation_type, + unpadded_hidden_size=self.unpadded_hidden_size, + out_tensor=moe_output, + use_dynamic_fc2_scale=False, + ) + if moe_output is not None: + return moe_output + return result[0] + def forward_chunk( self, x: Union[torch.Tensor, Fp4QuantizedTensor], diff --git a/tensorrt_llm/_torch/modules/fused_moe/quantization.py b/tensorrt_llm/_torch/modules/fused_moe/quantization.py index e6769b241646..10c09634f308 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/quantization.py +++ b/tensorrt_llm/_torch/modules/fused_moe/quantization.py @@ -2892,6 +2892,84 @@ def process_weights_after_loading(self, module: torch.nn.Module): super().process_weights_after_loading(module) +class W4A16NVFP4CutlassFusedMoEMethod(NVFP4CutlassFusedMoEMethod): + """W4A16 dequant-on-the-fly variant of NVFP4 MoE for SM<100. + + Loads an unmodified NVFP4 MoE ckpt; only load-time change is un-swizzling + per-block scales once so the per-forward dequant skips that step. + ``CutlassFusedMoE.run_moe`` dispatches here and uses an active-mask Triton + kernel (``dequant_active_experts_to_hp``) to dequant only routed experts + into a static [E_total, N, K] workspace, then runs the bf16 ``fused_moe``. + """ + + def process_weights_after_loading(self, module: torch.nn.Module): + super().process_weights_after_loading(module) + + # Scale buffer: int32-packed FP8, viewed as uint8 has shape + # [E, pad_up(N, 128), pad_up(K/sf_vec, 4)] -- the 3D layout + # block_scale_interleave_reverse accepts. + def _unswizzle_inplace(scale_param: torch.nn.Parameter): + sf_view = scale_param.data.view(float4_sf_dtype) + E, pad_rows, pad_cols = (sf_view.shape[0], sf_view.shape[1], + sf_view.shape[2]) + linear = torch.ops.trtllm.block_scale_interleave_reverse(sf_view) + scale_param.data.view(float4_sf_dtype).copy_(linear) + + _unswizzle_inplace(module.w3_w1_weight_scale) + _unswizzle_inplace(module.w2_weight_scale) + + def dequant_active_experts_to_hp( + self, + module: torch.nn.Module, + token_selected_experts: torch.Tensor, + out_dtype: torch.dtype, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Active-only dequant via Triton: static [E_total, N, K] workspace, + active-mask kernel skips dequant for experts with no routed tokens. + CUDA-graph capturable. + + Per-expert weight scale recovered as ``alpha * input_scale`` (NVFP4 + MoE loader stores alpha = amax_in*amax_w/(448*6)**2 and + input_scale = (448*6)/amax_in). + """ + from .triton_dequant_nvfp4 import (build_active_expert_mask, + dequant_nvfp4_active_triton) + + fc31_w_scale_2 = module.fc31_alpha * module.fc31_input_scale + fc2_w_scale_2 = module.fc2_alpha * module.fc2_input_scale + + sf_vec_size = module.scaling_vector_size + E_total = module.w3_w1_weight.shape[0] + + active_mask = build_active_expert_mask(token_selected_experts, E_total) + + # FP4 weights as uint8 (2 fp4/byte); per-block scales as uint8 to + # expose the unswizzled [E, N_pad, K_sf_pad] e4m3 bit layout. + w3_w1_packed = module.w3_w1_weight.view(torch.uint8) + w2_packed = module.w2_weight.view(torch.uint8) + w3_w1_scale = module.w3_w1_weight_scale.view(torch.uint8) + w2_scale = module.w2_weight_scale.view(torch.uint8) + + w3_w1_hp = dequant_nvfp4_active_triton( + w3_w1_packed, + w3_w1_scale, + fc31_w_scale_2, + active_mask, + target_dtype=out_dtype, + sf_vec_size=sf_vec_size, + ) + w2_hp = dequant_nvfp4_active_triton( + w2_packed, + w2_scale, + fc2_w_scale_2, + active_mask, + target_dtype=out_dtype, + sf_vec_size=sf_vec_size, + ) + + return w3_w1_hp, w2_hp + + class NVFP4CuteDslFusedMoEMethod(NVFP4CutlassFusedMoEMethod): def load_expert_w3_w1_weight(self, diff --git a/tensorrt_llm/_torch/modules/fused_moe/triton_dequant_nvfp4.py b/tensorrt_llm/_torch/modules/fused_moe/triton_dequant_nvfp4.py new file mode 100644 index 000000000000..da5e9d3afce7 --- /dev/null +++ b/tensorrt_llm/_torch/modules/fused_moe/triton_dequant_nvfp4.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Active-only NVFP4 weight dequant for MoE on SM<100 (used by +W4A16NVFP4CutlassFusedMoEMethod). Static shapes -> CUDA-graph capturable. + +Pipeline: scatter active_mask[E] from routing -> static (E, N/BN, K/BK) grid +-> inactive blocks early-return, active blocks dequant their tile. Downstream +fused_moe only reads rows in token_selected_experts, so leaving inactive +rows uninitialized is safe. +""" + +import torch +import triton # type: ignore[import] +import triton.language as tl # type: ignore[import] + +# E2M1 codebook (signed-magnitude nibble layout). Index 0b1000 nominally +# encodes "-0" and is treated as 0.0. Kept as a Python list so we can build +# the device tensor lazily on first use. +_E2M1_CODEBOOK = [ + 0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, + 0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0, +] # yapf: disable + +# Per-device cache so we don't reallocate the table on every call. +_E2M1_CODEBOOK_CACHE: "dict[torch.device, torch.Tensor]" = {} + + +def _get_e2m1_codebook(device: torch.device) -> torch.Tensor: + table = _E2M1_CODEBOOK_CACHE.get(device) + if table is None: + table = torch.tensor(_E2M1_CODEBOOK, dtype=torch.float32, device=device) + _E2M1_CODEBOOK_CACHE[device] = table + return table + + +def build_active_expert_mask( + token_selected_experts: torch.Tensor, num_experts: int +) -> torch.Tensor: + """Sync-free, CUDA-graph-safe active-expert mask via scalar ``scatter_`` + (the fancy-index form ``mask[ids] = 1`` is illegal under stream capture). + + Caller must pre-clamp ids into ``[0, num_experts)``. + """ + mask = torch.zeros(num_experts, dtype=torch.uint8, device=token_selected_experts.device) + mask.scatter_(0, token_selected_experts.reshape(-1).long(), 1) + return mask + + +@triton.jit +def _dequant_nvfp4_active_kernel( + # Inputs + packed_weight_ptr, + scale_ptr, + weight_scale_2_ptr, + active_mask_ptr, + e2m1_table_ptr, # [16] fp32 codebook + # Output + out_ptr, + # Strides (element counts, not bytes) + pw_stride_e, + pw_stride_n, + sc_stride_e, + sc_stride_n, + out_stride_e, + out_stride_n, + # Shapes (runtime) + N, + K, + # Compile-time + SF_VEC: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """Per-block dequant of one tile of one expert's weight.""" + pid_e = tl.program_id(0) + pid_n = tl.program_id(1) + pid_k = tl.program_id(2) + + # ---- Active-mask early exit ---- + is_active = tl.load(active_mask_ptr + pid_e) + if is_active == 0: + return + + # Output element offsets + n_offs = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) # [BLOCK_N] + k_offs = pid_k * BLOCK_K + tl.arange(0, BLOCK_K) # [BLOCK_K] + n_mask = n_offs < N + k_mask = k_offs < K + full_mask = n_mask[:, None] & k_mask[None, :] # [BLOCK_N, BLOCK_K] + + # ---- Load packed FP4 bytes ---- + # Each k position corresponds to byte k//2 with nibble shift (k%2)*4. + # Adjacent k positions share a byte: redundant loads, but cache friendly. + packed_idx = k_offs // 2 # [BLOCK_K] + nibble_shift = (k_offs % 2) * 4 + w_offs = pid_e * pw_stride_e + n_offs[:, None] * pw_stride_n + packed_idx[None, :] + packed = tl.load(packed_weight_ptr + w_offs, mask=full_mask, other=0).to(tl.int32) + nibble = (packed >> nibble_shift[None, :]) & 0xF # [BLOCK_N, BLOCK_K] + + # ---- E2M1 codebook lookup via gather ---- + # 16-element table lives in L1 / constant cache after first touch. + # nibble has shape [BLOCK_N, BLOCK_K]; the load fans out per lane and + # the hardware broadcasts duplicate indices. + val = tl.load(e2m1_table_ptr + nibble) # [BLOCK_N, BLOCK_K] fp32 + + # ---- Per-block FP8 (e4m3) scale ---- + # Each sf_vec_size consecutive k positions share one scale byte. + sf_idx = k_offs // SF_VEC # [BLOCK_K] + s_offs = pid_e * sc_stride_e + n_offs[:, None] * sc_stride_n + sf_idx[None, :] + scale_byte = tl.load(scale_ptr + s_offs, mask=full_mask, other=0) + # Reinterpret uint8 bits as fp8 e4m3, then convert to fp32. + scale_fp = scale_byte.to(tl.float8e4nv, bitcast=True).to(tl.float32) + + # ---- Per-tensor scale (scalar per expert) ---- + s2 = tl.load(weight_scale_2_ptr + pid_e) + + # ---- Combine and store ---- + out_val = val * scale_fp * s2 + o_offs = pid_e * out_stride_e + n_offs[:, None] * out_stride_n + k_offs[None, :] + tl.store(out_ptr + o_offs, out_val.to(out_ptr.dtype.element_ty), mask=full_mask) + + +def dequant_nvfp4_active_triton( + packed_weight: torch.Tensor, + scale_linear: torch.Tensor, + weight_scale_2: torch.Tensor, + active_mask: torch.Tensor, + *, + target_dtype: torch.dtype = torch.bfloat16, + sf_vec_size: int = 16, + block_n: int = 32, + block_k: int = 64, +) -> torch.Tensor: + """Triton-based active-only NVFP4 weight dequant. + + Args: + packed_weight: ``[E, N, K_packed]`` uint8 -- FP4 nibbles, two per byte. + scale_linear: ``[E, N_pad, K_sf_pad]`` uint8 -- per-block FP8 (e4m3) + scale in **linear** (un-swizzled) layout. The kernel uses tensor + strides directly, so padding on N/K_sf is fine as long as the + stride math points to the right elements. + weight_scale_2: ``[E]`` float32 -- per-tensor scale. + active_mask: ``[E]`` uint8 -- 1 for experts that need dequanting. + target_dtype: ``torch.bfloat16`` or ``torch.float16``. + sf_vec_size: NVFP4 per-block scale vector size (fixed at 16). + block_n, block_k: Triton tile shape. ``block_k`` should be a + multiple of ``sf_vec_size`` so each tile covers an integer + number of scale blocks. + + Returns: + ``[E, N, K]`` (with ``K = K_packed * 2``) in ``target_dtype``. + Tiles belonging to inactive experts are left uninitialized; they + are never read by the downstream MoE kernel. + """ + assert packed_weight.dim() == 3, "packed_weight must be 3D [E, N, K/2]" + assert sf_vec_size == 16, "NVFP4 fixed at 16-element blocks" + assert block_k % sf_vec_size == 0, ( + f"block_k={block_k} must be a multiple of sf_vec_size={sf_vec_size}" + ) + + E, N, K_packed = packed_weight.shape + K = K_packed * 2 + device = packed_weight.device + + if active_mask.dtype != torch.uint8: + active_mask = active_mask.to(torch.uint8) + + out = torch.empty(E, N, K, dtype=target_dtype, device=device) + e2m1_table = _get_e2m1_codebook(device) + + grid = (E, triton.cdiv(N, block_n), triton.cdiv(K, block_k)) + _dequant_nvfp4_active_kernel[grid]( + packed_weight, + scale_linear, + weight_scale_2, + active_mask, + e2m1_table, + out, + # strides (in elements) + packed_weight.stride(0), + packed_weight.stride(1), + scale_linear.stride(0), + scale_linear.stride(1), + out.stride(0), + out.stride(1), + # shapes + N, + K, + # constexpr + SF_VEC=sf_vec_size, + BLOCK_N=block_n, + BLOCK_K=block_k, + ) + return out + + +@triton.jit +def _dequant_nvfp4_linear_kernel( + # Inputs + packed_weight_ptr, + scale_ptr, + weight_scale_2_ptr, # pointer to one fp32 scalar (per-tensor) + e2m1_table_ptr, + # Output + out_ptr, + # Strides (in elements) + pw_stride_n, + sc_stride_n, + out_stride_n, + # Shapes (runtime) + N, + K, + # Compile-time + SF_VEC: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """Per-block dequant of one tile of a single 2D weight matrix. + + Dedicated 2D path for ``NVFP4LinearMethod``-style weights (one matrix, + one per-tensor scale, no expert dim or active mask). Codebook gather and + FP8 e4m3 -> fp32 conversion match the MoE kernel. + """ + pid_n = tl.program_id(0) + pid_k = tl.program_id(1) + + n_offs = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + k_offs = pid_k * BLOCK_K + tl.arange(0, BLOCK_K) + n_mask = n_offs < N + k_mask = k_offs < K + full_mask = n_mask[:, None] & k_mask[None, :] + + # ---- Load packed FP4 bytes ---- + packed_idx = k_offs // 2 + nibble_shift = (k_offs % 2) * 4 + w_offs = n_offs[:, None] * pw_stride_n + packed_idx[None, :] + packed = tl.load(packed_weight_ptr + w_offs, mask=full_mask, other=0).to(tl.int32) + nibble = (packed >> nibble_shift[None, :]) & 0xF + + # ---- E2M1 codebook gather ---- + val = tl.load(e2m1_table_ptr + nibble) + + # ---- Per-block FP8 (e4m3) scale ---- + sf_idx = k_offs // SF_VEC + s_offs = n_offs[:, None] * sc_stride_n + sf_idx[None, :] + scale_byte = tl.load(scale_ptr + s_offs, mask=full_mask, other=0) + scale_fp = scale_byte.to(tl.float8e4nv, bitcast=True).to(tl.float32) + + # ---- Per-tensor scale (single scalar, broadcast to the tile) ---- + s2 = tl.load(weight_scale_2_ptr) + + out_val = val * scale_fp * s2 + o_offs = n_offs[:, None] * out_stride_n + k_offs[None, :] + tl.store(out_ptr + o_offs, out_val.to(out_ptr.dtype.element_ty), mask=full_mask) + + +def dequant_nvfp4_2d_triton( + packed_weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_scale_2: torch.Tensor, + *, + target_dtype: torch.dtype = torch.bfloat16, + sf_vec_size: int = 16, + block_n: int = 32, + block_k: int = 64, +) -> torch.Tensor: + """2D (Linear) NVFP4 dequant via a dedicated Triton kernel. + + Distinct from the MoE 3D path: no expert dim, no active mask, the + per-tensor scale is a single scalar. + + Args: + packed_weight: ``[N, K_packed]`` uint8 -- FP4 nibbles, two per byte. + weight_scale: per-block FP8 (e4m3) scale in **linear** (un-swizzled) + layout. Accepted as either: + + * a flat 1-D buffer of length ``pad_up(N, 128) * pad_up(K/sf, 4)`` + (what ``NVFP4LinearMethod.create_weights`` allocates), or + * a 2-D buffer of shape ``[pad_rows, pad_cols]``. + weight_scale_2: per-tensor FP32 scale (any shape with a single + element; only ``data_ptr()`` is consumed by the kernel). + target_dtype: BF16 or FP16. + sf_vec_size: NVFP4 per-block size (16). + block_n, block_k: Triton tile shape. ``block_k`` must be a multiple + of ``sf_vec_size``. + + Returns: + ``[N, K]`` in ``target_dtype``. + """ + assert packed_weight.dim() == 2, "packed_weight must be 2D [N, K/2]" + assert sf_vec_size == 16, "NVFP4 fixed at 16-element blocks" + assert block_k % sf_vec_size == 0, ( + f"block_k={block_k} must be a multiple of sf_vec_size={sf_vec_size}" + ) + + N, K_packed = packed_weight.shape + K = K_packed * 2 + device = packed_weight.device + + # Reshape (possibly flat) scale to its 2D [pad_rows, pad_cols] form so + # the kernel can use ``scale.stride(0)`` directly. + if weight_scale.dim() == 1: + from tensorrt_llm.quantization.utils.fp4_utils import pad_up + + pad_rows = pad_up(N, 128) + pad_cols = pad_up(K // sf_vec_size, 4) + weight_scale = weight_scale.view(pad_rows, pad_cols) + elif weight_scale.dim() != 2: + raise ValueError(f"weight_scale must be 1D or 2D, got shape {tuple(weight_scale.shape)}") + + out = torch.empty(N, K, dtype=target_dtype, device=device) + e2m1_table = _get_e2m1_codebook(device) + + # The kernel reads the per-tensor scale via a single pointer load; flatten + # to ensure a contiguous, 1-D-addressable buffer regardless of caller shape. + weight_scale_2 = weight_scale_2.reshape(-1) + + grid = (triton.cdiv(N, block_n), triton.cdiv(K, block_k)) + _dequant_nvfp4_linear_kernel[grid]( + packed_weight, + weight_scale, + weight_scale_2, + e2m1_table, + out, + packed_weight.stride(0), + weight_scale.stride(0), + out.stride(0), + N, + K, + SF_VEC=sf_vec_size, + BLOCK_N=block_n, + BLOCK_K=block_k, + ) + return out diff --git a/tensorrt_llm/_torch/modules/linear.py b/tensorrt_llm/_torch/modules/linear.py index 2d3482fe1b34..84a695b1f974 100644 --- a/tensorrt_llm/_torch/modules/linear.py +++ b/tensorrt_llm/_torch/modules/linear.py @@ -1861,6 +1861,87 @@ def post_load_weights(self, module: Linear): module.rebuild_tensor_metadata) +class W4A16NVFP4LinearMethod(NVFP4LinearMethod): + """W4A16 dequant fallback for NVFP4 on SM<100. Only used by + modeling_nemotron_h. ``apply_linear_allreduce`` is inherited unchanged: + its fused path is SM>=100-gated upstream. + """ + + def post_load_weights(self, module: Linear): + # Skip parent's 32x16 weight padding (apply() accepts [N, K/2] as-is) + # and un-swizzle per-block scale once at load. + LinearMethodBase.post_load_weights(self, module) + pad_rows = fp4_utils.pad_up(module.out_features, 128) + pad_cols = fp4_utils.pad_up( + module.in_features // module.scaling_vector_size, 4) + scale_swizzled = module.weight_scale.data.view( + fp4_utils.float4_sf_dtype).reshape(pad_rows, pad_cols) + scale_linear = torch.ops.trtllm.block_scale_interleave_reverse( + scale_swizzled) + module.weight_scale.data.view(fp4_utils.float4_sf_dtype).copy_( + scale_linear.reshape(-1)) + + def apply(self, module: Linear, input: torch.Tensor, + bias: Optional[torch.Tensor]): + if isinstance(input, (Fp4QuantizedTensor, tuple)): + raise RuntimeError( + "W4A16NVFP4LinearMethod: hp input required; disable upstream " + "FP4 fusion (e.g. TRTLLM_ENABLE_ATTENTION_NVFP4_OUTPUT=0)") + + ## FP8 input from upstream FMHA pre-quant: invert by / module.inv_input_scale. + if input.dtype == torch.float8_e4m3fn: + assert module.inv_input_scale is not None, \ + "W4A16NVFP4LinearMethod: FP8 input requires static inv_input_scale" + input = (input.to(module.dtype) / module.inv_input_scale).to( + module.dtype) + + original_shape = None + if input.dim() > 2: + original_shape = input.shape + input = input.reshape(-1, input.shape[-1]) + + # NVFP4_AWQ pre_quant_scale (mirrors parent's _input_prepare branch). + if module.pre_quant_scale is not None: + assert input.dtype == module.pre_quant_scale.dtype, ( + "Input dtype and pre_quant_scale dtype must match") + input = input * module.pre_quant_scale + + from tensorrt_llm._torch.modules.fused_moe.triton_dequant_nvfp4 import \ + dequant_nvfp4_2d_triton + weight_deq = dequant_nvfp4_2d_triton( + module.weight.view(torch.uint8), + module.weight_scale, + module.weight_scale_2, + target_dtype=module.dtype, + sf_vec_size=module.scaling_vector_size, + ) + + if module.use_custom_cublas_mm: + output_buffer_kind = ( + int(BufferKind.NCCL_WINDOW) + if self.supports_nccl_symmetric_memory_window_output + and module.all_reduce is not None + and module.all_reduce.uses_nccl_symmetric_memory_window() else + int(BufferKind.DEFAULT)) + group = (module.mapping.tp_group + if output_buffer_kind == int(BufferKind.NCCL_WINDOW) + and module.mapping is not None else None) + output = torch.ops.trtllm.cublas_mm( + input, + weight_deq.t(), + bias, + out_dtype=None, + output_buffer_kind=output_buffer_kind, + group=group, + ) + else: + output = F.linear(input, weight_deq, bias) + + if original_shape is not None: + output = output.reshape(*original_shape[:-1], output.shape[-1]) + return output + + class W4A8NVFP4FP8LinearMethod(LinearMethodBase): def create_weights(self, module: Linear, in_features: int, diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index ae173a79e0e1..68f4c8edc31d 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -7141,6 +7141,35 @@ def test_nvfp4_4gpus_online_eplb(self, moe_backend): layer_updates_per_iter=2) self._run_nvfp4_4gpus_eplb(moe_backend, eplb_config, model_path) + @skip_pre_hopper + @skip_post_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @pytest.mark.skip_less_device_memory(80000) + def test_nvfp4_4gpus_hopper_w4a16(self): + """W4A16 NVFP4 dequant fallback on Hopper (SM 90), MTP draft_len=4.""" + model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + mamba_ssm_cache_dtype="float16", + free_gpu_memory_fraction=0.5, + ) + max_batch_size = 32 + cuda_graph_config = CudaGraphConfig(max_batch_size=max_batch_size, + enable_padding=True) + mtp_config = MTPDecodingConfig(max_draft_len=4) + pytorch_config = dict(cuda_graph_config=cuda_graph_config) + with LLM( + model_path, + kv_cache_config=kv_cache_config, + max_batch_size=max_batch_size, + tensor_parallel_size=4, + speculative_config=mtp_config, + **pytorch_config, + ) as llm: + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @skip_pre_hopper @pytest.mark.skip_less_mpi_world_size(4) @pytest.mark.skip_less_device_memory(40000) diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 29e0090d0c0d..34110b214d1d 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -102,6 +102,7 @@ l0_dgx_h100: - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_hopper_w4a16 - test_e2e.py::test_ptp_quickstart_advanced_bs1 - test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8] - test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b] From 70ab8fb512b797c25b63522894bd52fa9ba6560a Mon Sep 17 00:00:00 2001 From: Yuan Tong <13075180+tongyuantongyu@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:24:02 +0800 Subject: [PATCH 162/308] [TRTLLM-12596][feat] Support simple logprob format (#13972) Signed-off-by: Yuan Tong <13075180+tongyuantongyu@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/llm_request.py | 49 +++++++---- tensorrt_llm/_torch/pyexecutor/sampler.py | 58 ++++++++++--- tensorrt_llm/disaggregated_params.py | 3 +- tensorrt_llm/executor/base_worker.py | 28 ++++-- tensorrt_llm/executor/executor.py | 6 +- tensorrt_llm/executor/result.py | 52 ++++++++--- tensorrt_llm/sampling_params.py | 22 +++++ tensorrt_llm/serve/openai_protocol.py | 40 ++++++--- .../_torch/sampler/test_logits_logprobs.py | 86 +++++++++++++++++++ .../api_stability/api_stability_core.py | 2 +- .../references/completion_output.yaml | 2 +- .../references/sampling_params.yaml | 6 ++ .../completion_output.yaml | 4 +- .../test_openai_disagg_service.py | 20 +++++ 14 files changed, 307 insertions(+), 71 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index ba82ed21ae85..1758a0eff0a8 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -8,7 +8,7 @@ from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.bindings import executor as tllm_executor -from tensorrt_llm.executor.result import TokenLogprobs +from tensorrt_llm.executor.result import SimpleTokenLogprobs, TokenLogprobs from tensorrt_llm.sampling_params import LogprobMode SamplingConfig = tensorrt_llm.bindings.SamplingConfig @@ -213,19 +213,21 @@ def finalize_chunked_transfer(self): class LogProbStorage: beam_width: int = -1 - log_probs: list[TokenLogprobs] + log_probs: list[TokenLogprobs] | list[SimpleTokenLogprobs] cum_log_probs: list[float] - def _init(self, first_input: list[TokenLogprobs]): + def _init(self, first_input: list[TokenLogprobs] + | list[SimpleTokenLogprobs]): self.beam_width = len(first_input) self.log_probs = [[] for _ in range(self.beam_width)] self.cum_log_probs = [0 for _ in range(self.beam_width)] def append(self, - new_probs: list[TokenLogprobs], + new_probs: list[TokenLogprobs] | list[SimpleTokenLogprobs], cum_log_probs: Optional[list[float]] = None): """ - new_probs: [beam_width, num_tokens] + new_probs: [beam_width, num_tokens]; per-token entry is either a + ``dict[int, Logprob]`` (default) or a ``float`` (simple format). cum_log_probs: [beam_width] """ if self.beam_width == -1: @@ -236,14 +238,19 @@ def append(self, self.log_probs[beam_idx].extend(probs) if cum_log_probs is not None: self.cum_log_probs[beam_idx] = cum_log_probs[beam_idx] - else: - # FIXME: This relies on the ordering of LogProb's in the dictionary. TorchSampler ensures - # that the sampled logprob is in the first position. - self.cum_log_probs[beam_idx] += sum( - next(iter(prob.values())).logprob for prob in probs) - - def set_log_probs(self, log_probs: list[TokenLogprobs], - cum_log_probs: list[float]): + elif probs: + if isinstance(probs[0], dict): + # FIXME: This relies on the ordering of LogProb's in the dictionary. + # TorchSampler ensures that the sampled logprob is in the + # first position. + self.cum_log_probs[beam_idx] += sum( + next(iter(prob.values())).logprob for prob in probs) + else: + # Simple format: probs is SimpleTokenLogprobs (list[float]). + self.cum_log_probs[beam_idx] += sum(probs) + + def set_log_probs(self, log_probs: list[TokenLogprobs] + | list[SimpleTokenLogprobs], cum_log_probs: list[float]): """ Reset the storage and refill it with new values log_probs: [beam_width, num_tokens] @@ -268,7 +275,7 @@ class Diff: exclude_last_generation_logits: bool | None = None context_logits_list: list[torch.Tensor] = field(default_factory=list) generation_logits_list: list[torch.Tensor] = field(default_factory=list) - reset_log_probs: tuple[list[TokenLogprobs], + reset_log_probs: tuple[list[TokenLogprobs] | list[SimpleTokenLogprobs], list[float] | None] | None = None mm_embeddings: list[dict[str, Any] | None] = None mrope_position_ids: dict[str, Any] | None = None @@ -377,7 +384,8 @@ def append_generation_logits(self, generation_logits: torch.Tensor): self.diff.generation_logits_list.append(generation_logits) def append_log_probs(self, - log_probs: list[TokenLogprobs], + log_probs: list[TokenLogprobs] + | list[SimpleTokenLogprobs], cum_log_probs: Optional[list[float]] = None): if self._log_probs: self._log_probs.append(log_probs, cum_log_probs) @@ -435,8 +443,8 @@ def append_additional_generation_outputs( self.diff.additional_generation_outputs_list.append( (name, self._additional_generation_outputs[name][-1])) - def set_log_probs(self, log_probs: list[TokenLogprobs], - cum_log_probs: list[float]): + def set_log_probs(self, log_probs: list[TokenLogprobs] + | list[SimpleTokenLogprobs], cum_log_probs: list[float]): """ Set log_probs and cum_log_probs to the new values log_probs: [beam_width, num_tokens] @@ -484,7 +492,8 @@ def get_latest_logits_unexcluded(self) -> torch.Tensor | None: return storage.transpose(0, 1) @property - def log_probs(self) -> list[TokenLogprobs] | None: + def log_probs( + self) -> list[TokenLogprobs] | list[SimpleTokenLogprobs] | None: if not self._log_probs or not hasattr(self._log_probs, 'log_probs'): return None return self._log_probs.log_probs @@ -626,6 +635,7 @@ def __init__( use_chunked_generation_logits: bool = True, logits_chunk_size: int = 8, logprobs_mode: LogprobMode = LogprobMode.RAW, + logprobs_simple_format: bool = False, **kwargs): self.py_sampling_strategy: "Strategy | None" = None @@ -683,6 +693,7 @@ def __init__( self.py_num_logprobs = num_logprobs self.py_return_log_probs = return_log_probs + self.py_logprobs_simple_format = logprobs_simple_format self.py_return_context_logits = return_context_logits self.py_return_generation_logits = return_generation_logits self.py_return_logits_device_memory = return_logits_device_memory @@ -1046,6 +1057,8 @@ def executor_request_to_llm_request( agent_hierarchy=agent_hierarchy, logprobs_mode=getattr(executor_request, "py_logprobs_mode", LogprobMode.RAW), + logprobs_simple_format=getattr(executor_request, + "py_logprobs_simple_format", False), ) llm_request.py_original_end_id = getattr(executor_request, diff --git a/tensorrt_llm/_torch/pyexecutor/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler.py index 55a1c081cd2a..fc8e76b923e1 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler.py @@ -69,7 +69,7 @@ DecoderState, GptDecoderBatched, ) -from tensorrt_llm.executor.result import Logprob +from tensorrt_llm.executor.result import Logprob, SimpleTokenLogprobs, TokenLogprobs from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping @@ -2600,10 +2600,13 @@ def _store_logprobs_list_to_request( beam_width: int, count: int, num_topk_logprobs: int, - ) -> list[list[dict[int, Logprob]]]: - """Convert the LogProbsStateList object to a list of lists of dictionaries of Logprob objects + simple_format: bool = False, + ) -> list[list[dict[int, Logprob]]] | list[list[float]]: + """Convert the LogProbsStateList object to per-token logprobs. - Logprobs storage expects logprobs as a list[list[dict[int, Logprob]]] object + By default returns ``list[list[dict[int, Logprob]]]``. When + ``simple_format`` is True and ``num_topk_logprobs == 0`` the result is a + flat ``list[list[float]]`` (one logprob per generated token, per beam). args: logprobs_state_list: LogProbsStateList. Contains the topk indices, topk values, @@ -2612,17 +2615,27 @@ def _store_logprobs_list_to_request( beam_width: int. The beam width of the request. count: int. The number of tokens to store. num_topk_logprobs: int. The number of topk logprobs of each token. + simple_format: bool. If True (and num_topk_logprobs == 0), return + ``list[list[float]]`` instead of the dict format. Avoids per-token + dict allocation when only the sampled-token logprob is needed. output: - list[list[dict[int, Logprob]]]. Shape: (beam_width, count) + list[list[dict[int, Logprob]]] (default) or list[list[float]] (simple format). + Shape: (beam_width, count) """ sampled_log_probs_indices_list = logprobs_state_list.sampled_indices[req_seq_slot] sampled_log_probs_vals_list = logprobs_state_list.sampled_vals[req_seq_slot] sampled_log_probs_rank_list = logprobs_state_list.sampled_rank[req_seq_slot] - token_log_probs: list[list[dict[int, Logprob]]] if num_topk_logprobs == 0: - token_log_probs = [ + if simple_format: + token_log_probs_simple: list[list[float]] = [ + [sampled_log_probs_vals_list[beam_idx][step_idx] for step_idx in range(count)] + for beam_idx in range(beam_width) + ] + return token_log_probs_simple + + token_log_probs: list[list[dict[int, Logprob]]] = [ [ { sampled_log_probs_indices_list[beam_idx][step_idx]: Logprob( @@ -2677,7 +2690,12 @@ def handle_logprobs( assert logprobs_state_list is not None, "logprobs_state_list must be provided" assert request.py_seq_slot is not None token_log_probs = self._store_logprobs_list_to_request( - logprobs_state_list, request.py_seq_slot, beam_width, count, request.py_num_logprobs + logprobs_state_list, + request.py_seq_slot, + beam_width, + count, + request.py_num_logprobs, + simple_format=request.py_logprobs_simple_format, ) request.py_result.append_log_probs(token_log_probs) @@ -3132,12 +3150,24 @@ def _get_logprobs_from_request( if logprobs_tensor.numel() > 0: logprobs_list = request.py_result.log_probs assert logprobs_list is not None - for beam_idx, beam_logprobs in enumerate(logprobs_list): - for token_idx, token_logprobs in enumerate(beam_logprobs): - for key, value in token_logprobs.items(): - assert value.rank is not None - logprobs_tensor[beam_idx, token_idx, value.rank - 1] = value.logprob - logprobs_indices_tensor[beam_idx, token_idx, value.rank - 1] = key + + if request.py_logprobs_simple_format: + tokens = request.get_tokens() + for beam_idx, beam_logprobs in enumerate(logprobs_list): + beam_logprobs = cast(SimpleTokenLogprobs, beam_logprobs) + for token_idx, token_logprobs_simple in enumerate(beam_logprobs): + logprobs_tensor[beam_idx, token_idx, 0] = token_logprobs_simple + logprobs_indices_tensor[beam_idx, token_idx, 0] = tokens[beam_idx][ + token_idx + ] + else: + for beam_idx, beam_logprobs in enumerate(logprobs_list): + beam_logprobs = cast(TokenLogprobs, beam_logprobs) + for token_idx, token_logprobs in enumerate(beam_logprobs): + for key, value in token_logprobs.items(): + assert value.rank is not None + logprobs_tensor[beam_idx, token_idx, value.rank - 1] = value.logprob + logprobs_indices_tensor[beam_idx, token_idx, value.rank - 1] = key return logprobs_tensor_full, logprobs_indices_tensor_full def _prepare_beam_history( diff --git a/tensorrt_llm/disaggregated_params.py b/tensorrt_llm/disaggregated_params.py index 4319b5b79a99..6d6459a8afda 100644 --- a/tensorrt_llm/disaggregated_params.py +++ b/tensorrt_llm/disaggregated_params.py @@ -30,7 +30,8 @@ class DisaggregatedParams: disagg_request_id (int): The disaggregated request id, if set, both context and generation requests will use it as underlying request id. first_gen_log_probs (List): The logprobs for first_gen_tokens, produced during prefill. - Each entry is a list (one per beam) of TokenLogprobs (list of dict[int, Logprob]). + Each entry is a list (one per beam) of either ``TokenLogprobs`` (``list[dict[int, Logprob]]``, + default format) or ``SimpleTokenLogprobs`` (``list[float]``, simple format). first_gen_logits (List): The generation logits for first_gen_tokens, produced during prefill. Each entry is a torch.Tensor of shape [num_tokens, vocab_size] (one per beam/sequence). ctx_usage (Dict[str, Any]): The context usage payload to preserve exact diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 133fdd858a85..18e8a182f210 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -614,6 +614,8 @@ def _deduce_max_tokens(request: GenerationRequest, executor_request.py_num_logprobs = request.sampling_params.logprobs executor_request.py_lora_path = py_lora_path executor_request.py_logprobs_mode = request.sampling_params.logprobs_mode + executor_request.py_logprobs_simple_format = ( + request.sampling_params.logprobs_simple_format) # here we add executor_request.py_disaggregated_params= request.disaggregated_params for python cache transceiver if self._is_pytorch_backend and request.disaggregated_params is not None: @@ -1170,9 +1172,15 @@ def _compute_pytorch_prompt_logprobs( prompt_token_ids = generation_result._generation_request.prompt_token_ids[ 1:] + first_generation_token - logprobs_result = compute_logprobs(logprob_params.prompt_logprobs, None, - context_logits, None, None, - prompt_token_ids) + logprobs_result = compute_logprobs( + logprob_params.prompt_logprobs, + None, + context_logits, + None, + None, + prompt_token_ids, + simple_prompt_logprobs=logprob_params.prompt_logprobs_simple_format, + ) if generation_result._streaming: generation_result._cached_prompt_logprobs = logprobs_result.prompt @@ -1211,11 +1219,15 @@ def _get_logprobs(worker, return logprobs_result # TRT backend: compute both prompt and generation logprobs from logits - logprobs_result = compute_logprobs(logprob_params.prompt_logprobs, - logprob_params.logprobs, - response.result.context_logits, - response.result.generation_logits, - response.result.output_token_ids[0]) + logprobs_result = compute_logprobs( + logprob_params.prompt_logprobs, + logprob_params.logprobs, + response.result.context_logits, + response.result.generation_logits, + response.result.output_token_ids[0], + simple_prompt_logprobs=logprob_params.prompt_logprobs_simple_format, + simple_logprobs=logprob_params.logprobs_simple_format, + ) if logprob_params.drop_context_logits: response.clear_context_logits() diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 938acc666c20..a5f5efea6427 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -242,7 +242,11 @@ def _get_logprob_params( or self.postproc_config.num_postprocess_workers > 0, drop_generation_logits=( not request.sampling_params._need_return_generation_logits) - or self.postproc_config.num_postprocess_workers > 0) + or self.postproc_config.num_postprocess_workers > 0, + logprobs_simple_format=request.sampling_params. + logprobs_simple_format, + prompt_logprobs_simple_format=request.sampling_params. + prompt_logprobs_simple_format) return logprob_params diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index 420fb16ab3b1..2649f7aa5516 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -51,14 +51,19 @@ class Logprob: rank: Optional[int] = None -# List of token_id_to_Logprob dict for prompt or generation texts +# List of token_id_to_Logprob dict for prompt or generation texts. TokenLogprobs: TypeAlias = list[dict[int, Logprob]] +# Compact format: one logprob per token (the sampled token's logprob). +# Returned when `SamplingParams.{logprobs,prompt_logprobs}_simple_format` is set +# and the corresponding `logprobs` / `prompt_logprobs` is 0. Avoids the +# per-token `dict[int, Logprob]` allocation overhead of `TokenLogprobs`. +SimpleTokenLogprobs: TypeAlias = list[float] class LogProbsResult(NamedTuple): """Optional log probability outputs computed post runtime.""" - prompt: Optional[TokenLogprobs] = None - generation: Optional[TokenLogprobs] = None + prompt: Optional[TokenLogprobs | SimpleTokenLogprobs] = None + generation: Optional[TokenLogprobs | SimpleTokenLogprobs] = None class ResponseWrapper: @@ -102,8 +107,8 @@ class CompletionOutput: text (str): The generated output text. Defaults to "". token_ids (List[int], optional): The token ids of the generated output text. Defaults to []. cumulative_logprob (float, optional): The cumulative log probability of the generated output text. Defaults to None. - logprobs (TokenLogprobs | List[float], optional): The log probabilities of the top probability words at each position if the logprobs are requested. Defaults to None. - prompt_logprobs (TokenLogprobs, optional): The log probabilities per prompt token. Defaults to None. + logprobs (TokenLogprobs | SimpleTokenLogprobs, optional): The log probabilities of the top probability words at each position if the logprobs are requested. Defaults to None. + prompt_logprobs (TokenLogprobs | SimpleTokenLogprobs, optional): The log probabilities per prompt token. Defaults to None. finish_reason (Literal['stop', 'length', 'timeout', 'cancelled'], optional): The reason why the sequence is finished. Defaults to None. stop_reason (int, str, optional): The stop string or token id that caused the completion to stop, None if the completion finished for some other reason. Defaults to None. generation_logits (torch.Tensor, optional): The logits on the generated output token ids. Defaults to None. @@ -115,7 +120,7 @@ class CompletionOutput: Attributes: length (int): The number of generated tokens. token_ids_diff (List[int]): Newly generated token ids. - logprobs_diff (TokenLogprobs | List[float]): Logprobs of newly generated tokens. + logprobs_diff (TokenLogprobs | SimpleTokenLogprobs): Logprobs of newly generated tokens. text_diff (str): Newly generated tokens. """ index: int @@ -123,8 +128,10 @@ class CompletionOutput: token_ids: Optional[List[int]] = field(default_factory=list) cumulative_logprob: Optional[float] = None logprobs: Optional[TokenLogprobs - | List[float]] = field(default_factory=list) - prompt_logprobs: Optional[TokenLogprobs] = field(default_factory=list) + | SimpleTokenLogprobs] = field(default_factory=list) + prompt_logprobs: Optional[TokenLogprobs + | SimpleTokenLogprobs] = field( + default_factory=list) finish_reason: Optional[Literal['stop', 'length', 'timeout', 'cancelled']] = None stop_reason: Optional[Union[int, str]] = None @@ -157,7 +164,7 @@ def token_ids_diff(self) -> List[int]: return self.token_ids[self._last_token_ids_len:] @property - def logprobs_diff(self) -> TokenLogprobs | List[float]: + def logprobs_diff(self) -> TokenLogprobs | SimpleTokenLogprobs: return self.logprobs[self._last_logprobs_len:] @@ -1010,6 +1017,8 @@ def compute_logprobs( generation_logits: Optional[torch.Tensor], output_token_ids: Optional[list[int]], prompt_token_ids: Optional[list[int]] = None, + simple_prompt_logprobs: bool = False, + simple_logprobs: bool = False, ) -> LogProbsResult: """ Compute top-K logprobs from logits when engine doesn't provide them directly. @@ -1019,14 +1028,21 @@ def compute_logprobs( - Generation logprobs (from generation_logits, TRT backend): used when backend doesn't compute them in sampler (e.g., TRT). - Generation logprobs (PyTorch backend): not used; computed in sampler, not here. + When `simple_prompt_logprobs` / `simple_logprobs` is True and the corresponding + `k_*` is 0, the result is a flat ``SimpleTokenLogprobs`` (``list[float]``, + one logprob per token) instead of ``TokenLogprobs``. This avoids the + per-token dict allocation overhead when only the sampled-token logprob is + needed. + Returns: LogProbsResult, a NamedTuple containing: - - prompt: Optional[List[Dict[token_id, Logprob]]] logprobs for prompt tokens. - - generation: Optional[List[Dict[token_id, Logprob]]] logprobs for generated tokens. + - prompt: Optional[TokenLogprobs | SimpleTokenLogprobs] logprobs for prompt tokens. + - generation: Optional[TokenLogprobs | SimpleTokenLogprobs] logprobs for generated tokens. """ def _topk_logprobs(logits: torch.Tensor, top_k: int, - tokens: Optional[list[int]]) -> TokenLogprobs: + tokens: Optional[list[int]], + simple: bool) -> TokenLogprobs | SimpleTokenLogprobs: if logits.dim() == 3: # reshape from [1, T, V] to [T, V] logits = logits.squeeze(0) @@ -1040,6 +1056,13 @@ def _topk_logprobs(logits: torch.Tensor, top_k: int, # only return sampled token if top_k == 0: + if simple: + simple_results: list[float] = [] + if tokens is not None: + for t in range(logprobs.size(0)): + simple_results.append(logprobs[t, tokens[t]].item()) + return simple_results + results: TokenLogprobs = [] if tokens is not None: for t in range(logprobs.size(0)): @@ -1076,10 +1099,11 @@ def _topk_logprobs(logits: torch.Tensor, top_k: int, return results prompt_logprobs = _topk_logprobs( - context_logits, k_prompt_logprobs, prompt_token_ids + context_logits, k_prompt_logprobs, prompt_token_ids, + simple_prompt_logprobs ) if k_prompt_logprobs is not None and context_logits is not None else None generation_logprobs = _topk_logprobs( - generation_logits, k_logprobs, output_token_ids + generation_logits, k_logprobs, output_token_ids, simple_logprobs ) if k_logprobs is not None and generation_logits is not None else None return LogProbsResult(prompt=prompt_logprobs, diff --git a/tensorrt_llm/sampling_params.py b/tensorrt_llm/sampling_params.py index 910c31445e97..291abceb1c18 100644 --- a/tensorrt_llm/sampling_params.py +++ b/tensorrt_llm/sampling_params.py @@ -85,6 +85,12 @@ class LogprobParams(NamedTuple): drop_context_logits: bool = False # Drop the geneation_logits once the logprobs are computed drop_generation_logits: bool = False + # Return generation logprobs as `list[float]` instead of the default + # `list[dict[int, Logprob]]`. Only valid when logprobs == 0. + logprobs_simple_format: bool = False + # Return prompt logprobs as `list[float]` instead of the default + # `list[dict[int, Logprob]]`. Only valid when prompt_logprobs == 0. + prompt_logprobs_simple_format: bool = False class LogprobMode(StrEnum): @@ -228,8 +234,10 @@ class SamplingParams: logprobs (int, optional): Number of log probabilities to return per output token. When set to 0, return only the sampled token's log probability. When set to K>0, return top-K log probabilities + the sampled token's log probability (last entry) if it's not in the Top-K. Defaults to None. logprobs_mode (LogprobMode): The mode of log probabilities to return. Defaults to LogprobMode.RAW. + logprobs_simple_format (bool): If True (and `logprobs == 0`), return generation logprobs as a flat `list[float]` (one logprob per generated token) instead of the default `list[dict[int, Logprob]]` format. Reduces per-token allocation overhead when only the sampled-token logprob is needed. Incompatible with `logprobs is None or logprobs > 0` and with beam search. Defaults to False. prompt_logprobs (int, optional): Number of log probabilities to return per prompt token. When set to 0, return only the actual prompt token's log probability. When set to K>0, return top-K log probabilities + the actual prompt token's log probability (last entry) if it's not in the Top-K. Defaults to None. + prompt_logprobs_simple_format (bool): If True (and `prompt_logprobs == 0`), return prompt logprobs as a flat `list[float]` instead of the default `list[dict[int, Logprob]]` format. Incompatible with `prompt_logprobs is None or prompt_logprobs > 0`. Defaults to False. return_context_logits (bool): Controls if Result should contain the context logits. Defaults to False. return_generation_logits (bool): Controls if Result should contain the generation logits. Defaults to False. exclude_input_from_output (bool): Controls if output tokens in Result should include the input tokens. Defaults to True. @@ -301,6 +309,8 @@ class SamplingParams: # Keep the below fields in sync with tllme.OutputConfig logprobs: Optional[int] = None prompt_logprobs: Optional[int] = None + logprobs_simple_format: bool = False + prompt_logprobs_simple_format: bool = False return_context_logits: bool = False return_generation_logits: bool = False exclude_input_from_output: bool = True @@ -399,6 +409,18 @@ def _validate(self): check_logprobs_limit("logprobs", self.logprobs) check_logprobs_limit("prompt_logprobs", self.prompt_logprobs) + if self.logprobs_simple_format and self.logprobs != 0: + raise ValueError( + f"logprobs_simple_format=True requires logprobs == 0, got logprobs={self.logprobs}" + ) + if self.prompt_logprobs_simple_format and self.prompt_logprobs != 0: + raise ValueError( + "prompt_logprobs_simple_format=True requires prompt_logprobs == 0, got " + f"prompt_logprobs={self.prompt_logprobs}" + ) + if self.logprobs_simple_format and self.use_beam_search: + raise ValueError("logprobs_simple_format is not supported with beam search") + # NB: Static, because downstream code only holds instances of # bindings.SamplingConfig (not SamplingParams). @staticmethod diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index dbc30bd850e1..96316d2eb27d 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1183,36 +1183,54 @@ def decode_opaque_state(encoded_opaque_state: Optional[str]) -> Optional[bytes]: def _serialize_first_gen_log_probs( - first_gen_log_probs: Optional[list], ) -> Optional[List]: - """Serialize list[dict[int, Logprob]] to JSON-safe list[list[dict]].""" + first_gen_log_probs: Optional[list]) -> Optional[List]: + """Serialize ``list[dict[int, Logprob]] | list[float]`` to a JSON-safe form. + + - Default (verbose) format: each position is a ``dict[int, Logprob]`` and is + serialized as a list of ``{token_id, logprob, rank}`` dicts. + - Simple format: each position is a ``float``; passed through verbatim. + """ if first_gen_log_probs is None: return None if not isinstance(first_gen_log_probs, list): raise ValueError("first_gen_log_probs must be a list") result = [] for i, pos in enumerate(first_gen_log_probs): - if not isinstance(pos, dict): + if isinstance(pos, dict): + result.append([{ + "token_id": tid, + "logprob": lp.logprob, + "rank": lp.rank + } for tid, lp in pos.items()]) + elif isinstance(pos, (float, int)): + # Simple format: per-token sampled logprob. + result.append(float(pos)) + else: raise ValueError( - f"first_gen_log_probs[{i}] must be a dict, got {type(pos)}") - result.append([{ - "token_id": tid, - "logprob": lp.logprob, - "rank": lp.rank - } for tid, lp in pos.items()]) + f"first_gen_log_probs[{i}] must be a dict or float, got {type(pos)}" + ) return result def _deserialize_first_gen_log_probs( serialized: Optional[List], ) -> Optional[list]: - """Deserialize JSON list[list[dict]] back to list[dict[int, Logprob]].""" + """Inverse of :func:`_serialize_first_gen_log_probs`. + + Returns either ``list[dict[int, Logprob]]`` (default format) or + ``list[float]`` (simple format) depending on the serialized payload. + """ if serialized is None: return None from tensorrt_llm.executor.result import Logprob result = [] for i, pos in enumerate(serialized): + if isinstance(pos, (float, int)): + result.append(float(pos)) + continue if not isinstance(pos, list): raise ValueError( - f"first_gen_log_probs[{i}] must be a list, got {type(pos)}") + f"first_gen_log_probs[{i}] must be a list or float, got {type(pos)}" + ) token_map = {} for j, item in enumerate(pos): if not isinstance(item, dict): diff --git a/tests/unittest/_torch/sampler/test_logits_logprobs.py b/tests/unittest/_torch/sampler/test_logits_logprobs.py index dad36235aafb..c21a03054c39 100644 --- a/tests/unittest/_torch/sampler/test_logits_logprobs.py +++ b/tests/unittest/_torch/sampler/test_logits_logprobs.py @@ -361,6 +361,92 @@ def test_sampled_token_always_in_prompt_logprobs(logprobs_k: int, simple_llm: LL print(f"{'=' * 80}\n") +@pytest.mark.threadleak(enabled=False) +def test_logprobs_simple_format(simple_llm: LLM): + """When ``logprobs_simple_format=True`` and ``prompt_logprobs_simple_format=True`` + with the corresponding K==0, the per-token logprobs are returned as a flat + ``list[float]`` instead of the default ``list[dict[int, Logprob]]`` and the + numeric values match the dict-format path within tolerance.""" + + prompt = "The future of AI is" + common_kwargs = dict(max_tokens=8, temperature=0.0) + + dict_params = SamplingParams(logprobs=0, prompt_logprobs=0, **common_kwargs) + simple_params = SamplingParams( + logprobs=0, + prompt_logprobs=0, + logprobs_simple_format=True, + prompt_logprobs_simple_format=True, + **common_kwargs, + ) + + [dict_out] = list(simple_llm.generate([prompt], sampling_params=dict_params)) + [simple_out] = list(simple_llm.generate([prompt], sampling_params=simple_params)) + + dict_gen_logprobs = dict_out.outputs[0].logprobs + simple_gen_logprobs = simple_out.outputs[0].logprobs + + # Simple format must be list[float]; dict format must remain list[dict]. + assert all(isinstance(x, float) for x in simple_gen_logprobs), ( + f"Expected list[float], got element types: {[type(x) for x in simple_gen_logprobs]}" + ) + assert all(isinstance(x, dict) for x in dict_gen_logprobs) + + for token_id, lp_simple, lp_dict in zip( + dict_out.outputs[0].token_ids, simple_gen_logprobs, dict_gen_logprobs, strict=True + ): + torch.testing.assert_close( + torch.tensor(lp_simple, dtype=torch.float32), + torch.tensor(lp_dict[token_id].logprob, dtype=torch.float32), + atol=1e-4, + rtol=0, + ) + + dict_prompt_logprobs = dict_out.outputs[0].prompt_logprobs + simple_prompt_logprobs = simple_out.outputs[0].prompt_logprobs + assert all(isinstance(x, float) for x in simple_prompt_logprobs) + assert all(isinstance(x, dict) for x in dict_prompt_logprobs) + prompt_token_ids = dict_out.prompt_token_ids[1:] + dict_out.outputs[0].token_ids[:1] + for token_id, lp_simple, lp_dict in zip( + prompt_token_ids, simple_prompt_logprobs, dict_prompt_logprobs, strict=True + ): + torch.testing.assert_close( + torch.tensor(lp_simple, dtype=torch.float32), + torch.tensor(lp_dict[token_id].logprob, dtype=torch.float32), + atol=1e-4, + rtol=0, + ) + + +def test_logprobs_simple_format_validation(): + """``SamplingParams`` rejects incompatible combinations of the simple-format + flag with non-zero / unset ``logprobs`` and with beam search.""" + SamplingParams(max_tokens=4, logprobs=0, logprobs_simple_format=True) + SamplingParams(max_tokens=4, prompt_logprobs=0, prompt_logprobs_simple_format=True) + + with pytest.raises(ValueError, match=r"logprobs_simple_format=True requires logprobs == 0"): + SamplingParams(max_tokens=4, logprobs=2, logprobs_simple_format=True) + with pytest.raises(ValueError, match=r"logprobs_simple_format=True requires logprobs == 0"): + SamplingParams(max_tokens=4, logprobs=None, logprobs_simple_format=True) + with pytest.raises( + ValueError, match=r"prompt_logprobs_simple_format=True requires prompt_logprobs == 0" + ): + SamplingParams(max_tokens=4, prompt_logprobs=3, prompt_logprobs_simple_format=True) + with pytest.raises( + ValueError, match=r"prompt_logprobs_simple_format=True requires prompt_logprobs == 0" + ): + SamplingParams(max_tokens=4, prompt_logprobs=None, prompt_logprobs_simple_format=True) + with pytest.raises(ValueError, match="beam search"): + SamplingParams( + max_tokens=4, + logprobs=0, + logprobs_simple_format=True, + use_beam_search=True, + best_of=2, + n=2, + ) + + @pytest.mark.parametrize("logprobs_k", [None, 0, 3], ids=["None", "top_0", "top_3"]) @pytest.mark.parametrize("prompt_logprobs_k", [None, 0, 3], ids=["None", "top_0", "top_3"]) @pytest.mark.threadleak(enabled=False) diff --git a/tests/unittest/api_stability/api_stability_core.py b/tests/unittest/api_stability/api_stability_core.py index e1406f3148b8..604571645a89 100644 --- a/tests/unittest/api_stability/api_stability_core.py +++ b/tests/unittest/api_stability/api_stability_core.py @@ -24,7 +24,7 @@ from tensorrt_llm._torch.models.checkpoints.base_checkpoint_loader import \ BaseCheckpointLoader from tensorrt_llm.executor import GenerationResult -from tensorrt_llm.executor.result import TokenLogprobs +from tensorrt_llm.executor.result import SimpleTokenLogprobs, TokenLogprobs from tensorrt_llm.llmapi import (CalibConfig, CompletionOutput, GuidedDecodingParams, QuantConfig, RequestOutput, SamplingParams) diff --git a/tests/unittest/api_stability/references/completion_output.yaml b/tests/unittest/api_stability/references/completion_output.yaml index c30a0c308023..8c165e400781 100644 --- a/tests/unittest/api_stability/references/completion_output.yaml +++ b/tests/unittest/api_stability/references/completion_output.yaml @@ -24,7 +24,7 @@ properties: annotation: int default: inspect._empty logprobs_diff: - annotation: list[dict[int, tensorrt_llm.executor.result.Logprob]] | List[float] + annotation: list[dict[int, tensorrt_llm.executor.result.Logprob]] | list[float] default: inspect._empty text_diff: annotation: str diff --git a/tests/unittest/api_stability/references/sampling_params.yaml b/tests/unittest/api_stability/references/sampling_params.yaml index 5f4db3e7e7e2..ba59b8052f58 100644 --- a/tests/unittest/api_stability/references/sampling_params.yaml +++ b/tests/unittest/api_stability/references/sampling_params.yaml @@ -22,5 +22,11 @@ methods: logprobs_mode: annotation: LogprobMode default: LogprobMode.RAW + logprobs_simple_format: + annotation: bool + default: false + prompt_logprobs_simple_format: + annotation: bool + default: false return_annotation: None properties: {} diff --git a/tests/unittest/api_stability/references_committed/completion_output.yaml b/tests/unittest/api_stability/references_committed/completion_output.yaml index 084fbd6f7320..dae0cbf3dece 100644 --- a/tests/unittest/api_stability/references_committed/completion_output.yaml +++ b/tests/unittest/api_stability/references_committed/completion_output.yaml @@ -20,10 +20,10 @@ methods: annotation: Optional[torch.Tensor] default: null logprobs: - annotation: Optional[list[dict[int, tensorrt_llm.executor.result.Logprob]] | List[float]] + annotation: Optional[list[dict[int, tensorrt_llm.executor.result.Logprob]] | list[float]] default: null prompt_logprobs: - annotation: Optional[list[dict[int, tensorrt_llm.executor.result.Logprob]]] + annotation: Optional[list[dict[int, tensorrt_llm.executor.result.Logprob]] | list[float]] default: null return_annotation: None properties: {} diff --git a/tests/unittest/disaggregated/test_openai_disagg_service.py b/tests/unittest/disaggregated/test_openai_disagg_service.py index 78e40b772498..542a6e5c8978 100644 --- a/tests/unittest/disaggregated/test_openai_disagg_service.py +++ b/tests/unittest/disaggregated/test_openai_disagg_service.py @@ -504,6 +504,26 @@ def test_deserialize_rejects_non_list_entry(self): with pytest.raises(ValueError, match="must be a list"): _deserialize_first_gen_log_probs(["not_a_list"]) + def test_simple_format_roundtrip(self): + # Simple format: each position is a plain float (sampled-token logprob). + original = [-0.5, -1.25, -2.0] + serialized = _serialize_first_gen_log_probs(original) + # Serialized payload preserves floats verbatim so it remains JSON-safe. + assert serialized == [pytest.approx(v) for v in original] + recovered = _deserialize_first_gen_log_probs(serialized) + assert recovered == [pytest.approx(v) for v in original] + assert all(isinstance(v, float) for v in recovered) + + def test_simple_and_dict_formats_kept_disjoint(self): + # Each call uses one format; mixing within a single payload is unusual + # but the serdes round-trips them independently. + simple = _deserialize_first_gen_log_probs(_serialize_first_gen_log_probs([-0.5])) + dict_payload = _deserialize_first_gen_log_probs( + _serialize_first_gen_log_probs([{1: Logprob(logprob=-0.5, rank=1)}]) + ) + assert isinstance(simple[0], float) + assert isinstance(dict_payload[0], dict) + def test_deserialize_rejects_missing_keys(self): with pytest.raises(ValueError, match="missing required keys"): _deserialize_first_gen_log_probs([[{"token_id": 1}]]) From 2e6f602ece18c30775b1fe0a7db6d1502c5d78b9 Mon Sep 17 00:00:00 2001 From: sunnyqgg <159101675+sunnyqgg@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:30:56 +0800 Subject: [PATCH 163/308] [None][fix] Stabilize Mamba replay state update (#14509) Signed-off-by: qgai Signed-off-by: qgai Signed-off-by: sunnyqgg <159101675+sunnyqgg@users.noreply.github.com> Co-authored-by: qgai --- .../_torch/modules/mamba/mamba2_mixer.py | 39 ++- .../mamba/replay_selective_state_update.py | 22 +- tensorrt_llm/_torch/pyexecutor/_util.py | 9 + .../_torch/pyexecutor/mamba_cache_manager.py | 243 +++++++++++++++-- .../modules/mamba/test_mamba_ssm_rand_seed.py | 244 ++++++++++++++++++ 5 files changed, 525 insertions(+), 32 deletions(-) create mode 100644 tests/unittest/_torch/modules/mamba/test_mamba_ssm_rand_seed.py diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py index 1474f8f1fe2b..0c75aa58bd77 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py @@ -169,6 +169,8 @@ def __init__( # cache manager), so precompute both gate values here. sr_base = (self._stochastic_rounding_requested and self._mamba_ssm_cache_dtype == torch.float16) + # Keep replay SSM-cache writes on the same stochastic-rounding policy + # as flashinfer; the replay kernel masks stale slots before using them. self._stochastic_rounding_for_replay = sr_base self._stochastic_rounding_for_flashinfer = sr_base and self._use_flashinfer @@ -346,6 +348,8 @@ def forward( has_initial_states = mamba_metadata.has_initial_states[: num_prefills] + has_initial_states_p = has_initial_states[:num_prefills] + conv_states[state_indices_p[~has_initial_states_p]].zero_() # Fused kernel to avoid expensive .contiguous() call in causal_conv1d_fn. xbc_p_t = extract_transpose_xbc_prefill(zxbcdt, num_prefill_tokens, self.tp_d_inner, @@ -509,8 +513,22 @@ def convert_dt(): philox_kwargs = {} if use_stochastic_rounding: - philox_kwargs['rand_seed'] = torch.randint( - 0, 2**62, (1, ), device=x_d.device, dtype=torch.int64) + # Both replay and flashinfer read from the cache manager's + # persistent per-slot Philox seed buffer; replay indexes by + # cache_batch_idx, flashinfer reads slot 0 from a (1,) + # view. In-place add_(1) keeps CUDA-graph replay fresh + # without allocating any new CUDA tensors per forward. + rand_seed = layer_cache.mamba_ssm_rand_seed + assert rand_seed is not None, ( + "Mamba SSM stochastic rounding is enabled but the " + "rand_seed buffer was not allocated; check that " + "_util.py passes mamba_ssm_stochastic_rounding=True " + "to the cache manager.") + rand_seed.add_(1) + if use_replay: + philox_kwargs['rand_seed'] = rand_seed + else: + philox_kwargs['rand_seed'] = rand_seed[:1] philox_kwargs['philox_rounds'] = self._philox_rounds if use_replay: @@ -602,10 +620,19 @@ def convert_dt(): # Non-MTP decode only runs through flashinfer, no replay path. use_stochastic_rounding = self._stochastic_rounding_for_flashinfer if use_stochastic_rounding: - ssu_kwargs['rand_seed'] = torch.randint(0, - 2**62, (1, ), - device=x_d.device, - dtype=torch.int64) + # Fetch the persistent (cache_size,) Philox seed buffer + # from the cache manager and pass slot 0 as a (1,) view to + # flashinfer. No per-call CUDA tensor allocation; the + # in-place add_(1) is CUDA-graph-friendly. + rand_seed = (attn_metadata.kv_cache_manager. + get_mamba_ssm_rand_seed()) + assert rand_seed is not None, ( + "Mamba SSM stochastic rounding is enabled but the " + "rand_seed buffer was not allocated; check that " + "_util.py passes mamba_ssm_stochastic_rounding=True " + "to the cache manager.") + rand_seed.add_(1) + ssu_kwargs['rand_seed'] = rand_seed[:1] ssu_kwargs['philox_rounds'] = self._philox_rounds self.selective_state_update_func( diff --git a/tensorrt_llm/_torch/modules/mamba/replay_selective_state_update.py b/tensorrt_llm/_torch/modules/mamba/replay_selective_state_update.py index c8c6a655929b..9c510f8f4956 100644 --- a/tensorrt_llm/_torch/modules/mamba/replay_selective_state_update.py +++ b/tensorrt_llm/_torch/modules/mamba/replay_selective_state_update.py @@ -497,7 +497,7 @@ def _replay_state_update_kernel( # Each Philox call produces 4 random ints. We call randint4x on # quarter-sized dstate offsets and join+reshape to get the full # (M, dstate) random tensor — 4x fewer PRNG rounds. - rand_seed = tl.load(rand_seed_ptr) + rand_seed = tl.load(rand_seed_ptr + cache_batch_idx) base_rand = cache_batch_idx * stride_state_batch + pid_h * stride_state_head offs_n_q = tl.arange(0, BLOCK_SIZE_DSTATE // 4) rand_offsets_q = ( @@ -671,8 +671,11 @@ def replay_selective_state_update( z: (batch, T, nheads, dim) optional silu gate. dt_bias: (nheads, dim) optional, with stride(-1)==0 (tie_hdim). state_batch_indices: (batch,) optional cache slot mapping. - rand_seed: optional single-element int64 CUDA tensor for Philox PRNG seed. - When provided, state is stochastically rounded to fp16 on store. + rand_seed: optional (cache_size,) int64 CUDA tensor of per-cache-slot + Philox PRNG seeds. The caller bumps this tensor in-place for each + replay invocation so CUDA graph replay still gets fresh draws; the + kernel indexes it by cache_batch_idx. When provided, state is + stochastically rounded to fp16 on store. When None, standard deterministic rounding is used. philox_rounds: number of Philox PRNG rounds (default 10). launch_with_pdl: enable external PDL (conv1d → precompute chain). @@ -743,6 +746,19 @@ def replay_selective_state_update( assert old_dA_cumsum.shape == (cache_size, 2, nheads, T) assert cache_buf_idx.shape == (cache_size,) assert prev_num_accepted_tokens.shape == (cache_size,) + if rand_seed is not None: + assert rand_seed.dtype == torch.int64, ( + f"rand_seed dtype must be int64, got {rand_seed.dtype}" + ) + assert rand_seed.dim() == 1, ( + f"rand_seed must be a 1D tensor; got shape {tuple(rand_seed.shape)}" + ) + if rand_seed.shape[0] == 1 and cache_size > 1: + rand_seed = rand_seed.expand(cache_size).contiguous() + assert rand_seed.shape[0] >= cache_size, ( + f"rand_seed must have length 1 or >= cache_size ({cache_size}); " + f"got shape {tuple(rand_seed.shape)}" + ) tie_hdim = ( A.stride(-1) == 0 diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index f57a520c0d5a..addf98611597 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -1324,6 +1324,14 @@ def _create_kv_cache_manager( logger.info( "Replay kernel is not changed since TRTLLM_USE_MAMBA_REPLAY=1") + # Stochastic-rounding seeds must live on the cache manager (not be + # re-created with torch.randint per forward) whenever SR can fire + # on the fp16 SSM cache. This mirrors the predicate the mixer uses + # internally (`_stochastic_rounding_for_flashinfer` / + # `_stochastic_rounding_for_replay`) so allocation matches consumption. + mamba_ssm_stochastic_rounding = (stochastic_rounding + and mamba_params.mamba_ssm_cache_dtype + == torch.float16) kv_cache_manager = kv_cache_manager_cls( # mamba cache parameters mamba_params.state_size, @@ -1353,6 +1361,7 @@ def _create_kv_cache_manager( execution_stream=execution_stream, model_type="nemotron_hybrid", use_replay_state_update=use_replay, + mamba_ssm_stochastic_rounding=mamba_ssm_stochastic_rounding, ) elif is_qwen3_hybrid(config): if max_beam_width > 1: diff --git a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py index 50edbee51db7..b4eebeb1a25c 100644 --- a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @@ -56,6 +56,79 @@ def get_tensor_size_bytes(tensor): return 0 +# Mamba SSM stochastic-rounding Philox seed plumbing. +# +# Both the replay kernel and flashinfer's selective_state_update consume a +# `rand_seed` int64 tensor. Historically the non-replay paths created this +# tensor via `torch.randint(..., (1,))` on every decode step, which (a) is +# non-deterministic across runs, (b) allocates a fresh CUDA tensor per call +# and is therefore unfriendly to CUDA-graph capture, and (c) has no notion +# of cache-slot identity (so slot reuse cannot rotate the stream). +# +# The functions below produce per-slot int64 seeds via SplitMix64 finalization +# of (counter, slot, rank). Adjacent inputs yield uncorrelated outputs so +# consecutive cache slots and consecutive request counters do not leave +# structural fingerprints in the Philox input stream. All outputs live in +# (0, 2**62) so they avoid Philox's degenerate seed=0 case while staying +# within int64. +_MAMBA_SSM_SEED_MASK = (1 << 62) - 1 +_MAMBA_SSM_UINT64_MASK = (1 << 64) - 1 +_MAMBA_SSM_SEED_BASE = 0x6A09E667F3BCC908 +_MAMBA_SSM_SEED_MIX_COUNTER = 0x2545F4914F6CDD1D +_MAMBA_SSM_SEED_MIX_SLOT = 0x1B873593CC9E2D51 +_MAMBA_SSM_SEED_MIX_RANK = 0x9E3779B97F4A7C15 + + +def _splitmix64(x: int) -> int: + """SplitMix64 finalizer; pure function, no torch.""" + x = (x + 0x9E3779B97F4A7C15) & _MAMBA_SSM_UINT64_MASK + x ^= (x >> 30) + x = (x * 0xBF58476D1CE4E5B9) & _MAMBA_SSM_UINT64_MASK + x ^= (x >> 27) + x = (x * 0x94D049BB133111EB) & _MAMBA_SSM_UINT64_MASK + x ^= (x >> 31) + return x & _MAMBA_SSM_UINT64_MASK + + +def _compute_deterministic_mamba_seed(counter: int, slot: int, + rank_offset: int) -> int: + """Deterministic int64 seed in (0, 2**62) from (counter, slot, rank). + + Pure function (no RNG, no torch.randint). Identical inputs across + process invocations produce identical outputs, which is what the + acceptance criteria require for cross-run reproducibility. + """ + folded = (_MAMBA_SSM_SEED_BASE + counter * _MAMBA_SSM_SEED_MIX_COUNTER + + slot * _MAMBA_SSM_SEED_MIX_SLOT + + rank_offset * _MAMBA_SSM_SEED_MIX_RANK) + folded &= _MAMBA_SSM_UINT64_MASK + value = _splitmix64(folded) & _MAMBA_SSM_SEED_MASK + if value == 0: + value = 1 + return value + + +def _allocate_mamba_seed_buffer(cache_size: int, rank_offset: int, + device: torch.device) -> torch.Tensor: + """Allocate a (cache_size,) int64 CUDA buffer of deterministic seeds. + + counter=0 at allocation time; per-slot reset on fresh request assignment + bumps a host-side counter and rewrites only the freshly-assigned slot. + """ + slot_seeds = [ + _compute_deterministic_mamba_seed(0, i, rank_offset) + for i in range(cache_size) + ] + return torch.tensor(slot_seeds, dtype=torch.int64, device=device) + + +def _mamba_rank_offset(mapping: Mapping) -> int: + """Distinct seed offset per (tp_rank, pp_rank) so independent ranks + don't draw identical streams when not coordinated.""" + return (mapping.tp_rank * 1_000_003 + mapping.pp_rank * 1_000_033 + + mapping.rank * 1_009) + + def use_cpp_mamba_cache_manager() -> bool: """Check if C++ MambaCacheManager should be used. @@ -277,8 +350,9 @@ class SpeculativeState(State): - Legacy: caches full intermediate SSM states (intermediate_ssm) - Replay: compact double-buffered cache (old_x, old_B, old_dt, old_dA_cumsum) """ - _SHARED_FIELDS = frozenset( - {"prev_num_accepted_tokens", "cache_buf_idx"}) + _SHARED_FIELDS = frozenset({ + "prev_num_accepted_tokens", "cache_buf_idx", "mamba_ssm_rand_seed" + }) intermediate_conv_window: torch.Tensor # always allocated @@ -290,6 +364,10 @@ class SpeculativeState(State): # 0 means temporal saved state is actually the last state, not two back. prev_num_accepted_tokens: torch.Tensor | None = None # (cache,) int — shared across layers cache_buf_idx: torch.Tensor | None = None # (cache,) int32 — shared across layers + # Per-cache-slot Philox seeds. Replay bumps them in-place per launch so + # CUDA graph replay uses fresh SR draws without allocating RNG tensors. + # (cache,) int64 - shared across layers + mamba_ssm_rand_seed: torch.Tensor | None = None old_x: torch.Tensor | None = None # (layers, cache, T, nheads, dim) old_B: torch.Tensor | None = None # (layers, cache, 2, T, ngroups, dstate) # Processed dt: softplus(raw_dt + dt_bias), clamped to dt_limit. @@ -313,12 +391,23 @@ def __init__( speculative_num_draft_tokens: Optional[int] = None, model_type: str = "nemotron_hybrid", use_replay_state_update: bool = False, + mamba_ssm_stochastic_rounding: bool = False, ) -> None: self.mamba_ssm_cache_dtype = ssm_cache_dtype self.speculative_num_draft_tokens = speculative_num_draft_tokens self.spec_state_size = spec_state_size self._use_replay_state_update = use_replay_state_update + # When True, allocate the per-slot Philox seed buffer even outside + # the replay path so the non-replay flashinfer SR kernel reads a + # persistent deterministic seed instead of a per-call torch.randint. + self._mamba_ssm_stochastic_rounding = mamba_ssm_stochastic_rounding + self._seed_rank_offset = _mamba_rank_offset(mapping) + # Host-side counter bumped per fresh cache-slot assignment. Combined + # with slot index and rank offset to produce reproducible per-slot + # seed values. Starts at 0 so the post-init "reset" stream is + # disjoint from the counter=0 stream used at allocation time. + self._seed_request_counter = 0 # get tp size tp_size = 1 if mapping.enable_attention_dp else mapping.tp_size @@ -379,6 +468,17 @@ def __init__( device=device, ) + # Per-slot Philox seeds. Allocated whenever stochastic rounding can + # fire, even outside the replay path and even when MTP/spec is off, + # so all consumers (replay kernel, MTP non-replay flashinfer, + # non-MTP flashinfer) read a persistent deterministic seed from the + # cache manager instead of calling torch.randint per forward. + self._mamba_ssm_rand_seed: Optional[torch.Tensor] = None + if (self._use_replay_state_update + or self._mamba_ssm_stochastic_rounding): + self._mamba_ssm_rand_seed = _allocate_mamba_seed_buffer( + max_batch_size, self._seed_rank_offset, device) + # create state container if speculative_num_draft_tokens is not None: T = speculative_num_draft_tokens + 1 @@ -393,6 +493,10 @@ def __init__( # SSM speculative cache — path-specific tensors spec_kwargs = {} + # Share the manager-level seed buffer through SpeculativeState + # so the MTP path can still read it via layer_cache. + if self._mamba_ssm_rand_seed is not None: + spec_kwargs['mamba_ssm_rand_seed'] = self._mamba_ssm_rand_seed if self._use_replay_state_update: assert n_groups % tp_size == 0, \ "replay state update requires n_groups divisible by tp_size" @@ -535,6 +639,16 @@ def _prepare_mamba_cache_blocks(self, request_ids: List[int]): if (isinstance(self.mamba_cache, self.SpeculativeState) and self._use_replay_state_update): self.mamba_cache.prev_num_accepted_tokens[block] = 0 + if self._mamba_ssm_rand_seed is not None: + # Deterministic per-slot rotation on fresh assignment. + # `block` is pulled from mamba_cache_free_blocks, which + # excludes _padding_slot by construction (see __init__), + # so padding sentinels never reach this branch. + self._seed_request_counter += 1 + self._mamba_ssm_rand_seed[block] = ( + _compute_deterministic_mamba_seed( + self._seed_request_counter, block, + self._seed_rank_offset)) def prepare_resources(self, scheduled_batch: ScheduledRequests): context_ids = [ @@ -621,6 +735,16 @@ def mamba_layer_cache(self, def get_mamba_ssm_cache_dtype(self) -> torch.dtype: return self.mamba_ssm_cache_dtype + def get_mamba_ssm_rand_seed(self) -> Optional[torch.Tensor]: + """Return the persistent (cache_size,) int64 Philox seed buffer or + None when stochastic rounding is not active for this manager. + + Used by mamba2_mixer non-MTP paths that don't hold a SpeculativeState. + Callers must bump in-place (`.add_(1)` or slice-and-add) and pass a + view that matches the consuming kernel's expected shape. + """ + return self._mamba_ssm_rand_seed + @property def use_replay_state_update(self) -> bool: return self._use_replay_state_update @@ -642,6 +766,7 @@ def _drop(tensor): prev_num_accepted_tokens=_drop( self.mamba_cache.prev_num_accepted_tokens), cache_buf_idx=_drop(self.mamba_cache.cache_buf_idx), + mamba_ssm_rand_seed=_drop(self.mamba_cache.mamba_ssm_rand_seed), old_x=_drop(self.mamba_cache.old_x), old_B=_drop(self.mamba_cache.old_B), old_dt=_drop(self.mamba_cache.old_dt), @@ -714,6 +839,7 @@ def __init__( speculative_num_draft_tokens: Optional[int] = None, model_type: str = "nemotron_hybrid", use_replay_state_update: bool = False, + mamba_ssm_stochastic_rounding: bool = False, ) -> None: max_num_sequences = max_batch_size * mapping.pp_size self._use_cpp = use_cpp_mamba_cache_manager() @@ -752,6 +878,7 @@ def __init__( speculative_num_draft_tokens=speculative_num_draft_tokens, model_type=model_type, use_replay_state_update=use_replay_state_update, + mamba_ssm_stochastic_rounding=mamba_ssm_stochastic_rounding, ) def get_max_resource_count(self) -> int: @@ -800,6 +927,13 @@ def get_ssm_states(self, layer_idx: int) -> torch.Tensor: def get_mamba_ssm_cache_dtype(self) -> torch.dtype: return self._impl.get_mamba_ssm_cache_dtype() + def get_mamba_ssm_rand_seed(self) -> Optional[torch.Tensor]: + """Delegate to the underlying Python manager. The C++ manager does + not allocate this buffer because it does not support speculative + decoding (and the SR-on-non-replay bug only fires under MTP).""" + getter = getattr(self._impl, 'get_mamba_ssm_rand_seed', None) + return getter() if getter is not None else None + @property def use_replay_state_update(self) -> bool: return getattr(self._impl, 'use_replay_state_update', False) @@ -893,6 +1027,7 @@ def __init__( model_type: str = "nemotron_hybrid", is_draft: bool = False, use_replay_state_update: bool = False, + mamba_ssm_stochastic_rounding: bool = False, # Per-pool configurations forwarded to the C++ KVCacheManager ctor. # Lets a single manager host pools with mixed shapes (e.g. Gemma4 # hybrid attention). See KVCacheManager.__init__. @@ -926,6 +1061,7 @@ def __init__( if spec_config is not None else None), model_type=model_type, use_replay_state_update=use_replay_state_update, + mamba_ssm_stochastic_rounding=mamba_ssm_stochastic_rounding, ) # initialize kv cache manager @@ -1033,6 +1169,7 @@ def __init__( is_estimating_kv_cache: bool = False, is_draft: bool = False, use_replay_state_update: bool = False, + mamba_ssm_stochastic_rounding: bool = False, model_type: str = "nemotron_hybrid", **kwargs, ) -> None: @@ -1072,6 +1209,14 @@ def __init__( # accessors (get_mamba_ssm_cache_dtype, use_replay_state_update) work # on ranks with no local mamba layers. self._use_replay_state_update = use_replay_state_update + # Same allocation gate as PythonMambaCacheManager: the rand_seed + # buffer must exist whenever SR can fire, not only on the replay path. + self._mamba_ssm_stochastic_rounding = mamba_ssm_stochastic_rounding + self._seed_rank_offset = _mamba_rank_offset(mapping) + # Host-side counter bumped per fresh context-slot assignment; combined + # with the slot index and rank offset to produce reproducible per-slot + # seed values without any torch.randint. + self._seed_request_counter = 0 self.ssm_state_dtype = mamba_ssm_cache_dtype if self.local_num_mamba_layers == 0: @@ -1327,6 +1472,7 @@ def shutdown(self): self.intermediate_state_indices = None self.prev_num_accepted_tokens = None self.cache_buf_idx = None + self.mamba_ssm_rand_seed = None self.old_x = None self.old_B = None self.old_dt = None @@ -1395,17 +1541,41 @@ def _prepare_resources(self, scheduled_batch: ScheduledRequests): if self._pending_state_transfers: logger.info(f"Need to transfer mamba state blocks") self._setup_state_indices() - # Reset replay double-buffer state for fresh context blocks. A reused - # block (prefix-cache hit or block recycled across requests) may carry - # stale prev_num_accepted_tokens / cache_buf_idx values from a prior - # owner; the replay kernel reads these on the first decode step. - if self._use_replay_state_update and self.prev_num_accepted_tokens is not None: - num_contexts = len(scheduled_batch.context_requests) - if num_contexts > 0: - ctx_slots = self.cuda_state_indices[:num_contexts].long() + num_contexts = len(scheduled_batch.context_requests) + if num_contexts > 0: + ctx_slots = self.cuda_state_indices[:num_contexts].long() + if self._use_replay_state_update and self.prev_num_accepted_tokens is not None: self.prev_num_accepted_tokens[ctx_slots] = 0 - # don't care which half of doulbe-buffer is using - # self.cache_buf_idx[ctx_slots] = 0 + self.cache_buf_idx[ctx_slots] = 0 + if self.old_x is not None: + self.old_x[:, ctx_slots] = 0 + if self.old_B is not None: + self.old_B[:, ctx_slots] = 0 + if self.old_dt is not None: + self.old_dt[:, ctx_slots] = 0 + if self.old_dA_cumsum is not None: + self.old_dA_cumsum[:, ctx_slots] = 0 + # Deterministic per-context-slot seed rotation. Runs whenever + # the seed buffer exists, including the non-replay SR path. + # Bump the host counter once per batch and write one new seed + # per fresh context slot from a pure function of + # (counter, slot, rank). No torch.randint involved. + if self.mamba_ssm_rand_seed is not None: + self._seed_request_counter += 1 + counter = self._seed_request_counter + rank_offset = self._seed_rank_offset + host_slots = ctx_slots.cpu().tolist() + new_seeds = [ + _compute_deterministic_mamba_seed(counter, slot, + rank_offset) + for slot in host_slots + ] + seed_tensor = torch.tensor( + new_seeds, + dtype=torch.int64, + device=self.mamba_ssm_rand_seed.device, + ) + self.mamba_ssm_rand_seed[ctx_slots] = seed_tensor def prepare_resources(self, scheduled_batch: ScheduledRequests): super().prepare_resources(scheduled_batch) @@ -1528,6 +1698,12 @@ def mamba_layer_cache( if self.spec_config is not None: layer_offset = self.mamba_layer_offsets[layer_idx] spec_kwargs = {} + # Per-cache-slot Philox seed buffer is shared across replay and + # non-replay MTP paths. The mixer asserts non-None on both + # branches when SR is enabled, so pass it through whenever it + # exists — not just on the replay branch. + if self.mamba_ssm_rand_seed is not None: + spec_kwargs['mamba_ssm_rand_seed'] = self.mamba_ssm_rand_seed if self._use_replay_state_update: # Per-layer slices for the replay kernel; shared 1D tensors # (cache_buf_idx, prev_num_accepted_tokens) are passed @@ -1720,24 +1896,40 @@ def _setup_replay_buffers(self, spec_config) -> None: ``prev_num_accepted_tokens`` by these block indices, so the buffers must match the pool extent rather than ``max_batch_size``. """ + # Replay tensors require spec_config + replay path enabled. The + # rand_seed buffer is separable from replay and must also be + # allocated for non-replay SR so the flashinfer path has a + # persistent deterministic seed source. + self.prev_num_accepted_tokens = None + self.cache_buf_idx = None + self.mamba_ssm_rand_seed = None + self.old_x = None + self.old_B = None + self.old_dt = None + self.old_dA_cumsum = None + + if (not self._use_replay_state_update + and not self._mamba_ssm_stochastic_rounding): + return + + cache_size = self.all_ssm_states.shape[1] + device = self.all_ssm_states.device + # Always-available deterministic seed buffer when SR (or replay) + # is on. Works for non-MTP runs because we don't depend on + # spec_config to allocate it. + self.mamba_ssm_rand_seed = _allocate_mamba_seed_buffer( + cache_size, self._seed_rank_offset, device) + if spec_config is None or not self._use_replay_state_update: - # Replay tensors are sized by the recurrent-state pool block count and - # are allocated in _setup_replay_buffers after _setup_states(). - self.prev_num_accepted_tokens = None - self.cache_buf_idx = None - self.old_x = None - self.old_B = None - self.old_dt = None - self.old_dA_cumsum = None + # Without spec_config or replay we still keep the seed buffer + # (above) so the non-MTP flashinfer SR path has a persistent + # rand_seed source. return T = spec_config.max_draft_len + 1 num_local_mamba_layers = self.local_num_mamba_layers - # all_ssm_states: [num_local_mamba_layers, num_blocks_in_pool, ...] - cache_size = self.all_ssm_states.shape[1] nheads, head_dim, d_state = self.ssm_state_shape n_groups_per_rank = self._n_groups_per_rank - device = self.all_ssm_states.device # Shared across layers (consumed by the replay kernel via slot index). self.prev_num_accepted_tokens = torch.zeros(cache_size, @@ -1784,3 +1976,8 @@ def use_replay_state_update(self) -> bool: def get_mamba_ssm_cache_dtype(self) -> torch.dtype: return self.ssm_state_dtype + + def get_mamba_ssm_rand_seed(self) -> Optional[torch.Tensor]: + """Return the persistent (cache_size,) int64 Philox seed buffer or + None when stochastic rounding is not active for this manager.""" + return getattr(self, 'mamba_ssm_rand_seed', None) diff --git a/tests/unittest/_torch/modules/mamba/test_mamba_ssm_rand_seed.py b/tests/unittest/_torch/modules/mamba/test_mamba_ssm_rand_seed.py new file mode 100644 index 000000000000..f94a5f7a28cf --- /dev/null +++ b/tests/unittest/_torch/modules/mamba/test_mamba_ssm_rand_seed.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the Mamba SSM stochastic-rounding Philox seed plumbing. + +The Mamba SSM SR path previously generated `rand_seed` tensors via +`torch.randint(..., (1,))` on every decode forward. The cache manager now +owns a persistent per-cache-slot int64 buffer that is deterministically +initialized and rewritten on fresh request assignment. These tests pin the +contract: pure-function seed generation, deterministic allocation, and +per-slot reset without `torch.randint`. +""" + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( + PythonMambaCacheManager, + _allocate_mamba_seed_buffer, + _compute_deterministic_mamba_seed, + _mamba_rank_offset, +) +from tensorrt_llm.mapping import Mapping + + +def test_deterministic_seed_is_pure_function(): + s1 = _compute_deterministic_mamba_seed(counter=7, slot=3, rank_offset=42) + s2 = _compute_deterministic_mamba_seed(counter=7, slot=3, rank_offset=42) + assert s1 == s2 + assert 0 < s1 < (1 << 62) + + +def test_deterministic_seed_distinct_inputs_distinct_outputs(): + s_a = _compute_deterministic_mamba_seed(1, 0, 0) + s_b = _compute_deterministic_mamba_seed(1, 1, 0) + s_c = _compute_deterministic_mamba_seed(1, 0, 1) + s_d = _compute_deterministic_mamba_seed(2, 0, 0) + # All four (counter, slot, rank_offset) keys yield distinct seeds. + assert len({s_a, s_b, s_c, s_d}) == 4 + + +def test_rank_offset_distinct_per_rank(): + o0 = _mamba_rank_offset(Mapping(world_size=4, tp_size=4, pp_size=1, rank=0)) + o1 = _mamba_rank_offset(Mapping(world_size=4, tp_size=4, pp_size=1, rank=1)) + o2 = _mamba_rank_offset(Mapping(world_size=4, tp_size=4, pp_size=1, rank=2)) + o3 = _mamba_rank_offset(Mapping(world_size=4, tp_size=4, pp_size=1, rank=3)) + assert len({o0, o1, o2, o3}) == 4 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_allocated_buffer_is_reproducible_and_nonzero(): + device = torch.device("cuda") + buf1 = _allocate_mamba_seed_buffer(8, rank_offset=5, device=device) + buf2 = _allocate_mamba_seed_buffer(8, rank_offset=5, device=device) + assert buf1.dtype == torch.int64 + assert buf1.shape == (8,) + assert torch.equal(buf1, buf2) + assert (buf1 > 0).all().item() + # All eight slots should differ; collisions would defeat per-slot + # variance in the replay kernel. + assert buf1.unique().numel() == 8 + + +def _make_python_manager(*, sr: bool, replay: bool, max_batch_size: int = 4): + """Build a tiny PythonMambaCacheManager with MTP/spec enabled.""" + mapping = Mapping(world_size=1, tp_size=1, pp_size=1, rank=0) + return PythonMambaCacheManager( + d_state=4, + d_conv=2, + num_heads=2, + n_groups=1, + head_dim=2, + num_layers=1, + max_batch_size=max_batch_size, + spec_state_size=max_batch_size, + mapping=mapping, + dtype=torch.float16, + ssm_cache_dtype=torch.float16, + layer_mask=[True], + speculative_num_draft_tokens=1, + model_type="nemotron_hybrid", + use_replay_state_update=replay, + mamba_ssm_stochastic_rounding=sr, + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_buffer_allocated_when_sr_only_no_replay(): + # The regression: SR enabled but replay off must still produce a + # persistent seed buffer so mamba2_mixer's non-replay branch can read it. + mgr = _make_python_manager(sr=True, replay=False) + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + assert seed_buf.dtype == torch.int64 + assert seed_buf.device.type == "cuda" + # Replay-only buffers are not allocated. + assert mgr.mamba_cache.old_x is None + assert mgr.mamba_cache.intermediate_ssm is not None # legacy SSM cache + # SpeculativeState exposes the same buffer (same Python identity). + assert mgr.mamba_cache.mamba_ssm_rand_seed is seed_buf + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_buffer_absent_when_neither_sr_nor_replay(): + mgr = _make_python_manager(sr=False, replay=False) + assert mgr.get_mamba_ssm_rand_seed() is None + assert mgr.mamba_cache.mamba_ssm_rand_seed is None + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_per_slot_reset_is_deterministic_and_uses_no_randint(): + mgr = _make_python_manager(sr=True, replay=False, max_batch_size=8) + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + pre = seed_buf.clone() + mgr._prepare_mamba_cache_blocks([1001]) + block = mgr.mamba_cache_index[1001] + post = seed_buf.clone() + # Exactly the freshly-assigned slot is rewritten. + diff_mask = pre != post + assert diff_mask.sum().item() == 1 + assert diff_mask[block].item() + # Rewrite is a deterministic function of (counter, slot, rank_offset). + expected = _compute_deterministic_mamba_seed( + mgr._seed_request_counter, block, mgr._seed_rank_offset + ) + assert post[block].item() == expected + assert expected > 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_padding_sentinel_does_not_churn_seeds(): + # CUDA-graph padding sentinels alias to the shared _padding_slot and + # must not rotate the seed entry of live real requests. + mgr = _make_python_manager(sr=True, replay=False, max_batch_size=8) + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID + + sentinel_id = CUDA_GRAPH_DUMMY_REQUEST_ID + pre = seed_buf.clone() + counter_pre = mgr._seed_request_counter + mgr.add_dummy_requests([sentinel_id]) + post = seed_buf.clone() + # Sentinel assignment must leave the buffer (and the host counter) + # untouched; otherwise the seed buffer would churn every graph capture. + assert torch.equal(pre, post) + assert mgr._seed_request_counter == counter_pre + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_replay_path_still_allocates_seed_buffer(): + # Backward-compatibility: the replay path used to allocate the seed + # buffer; the new wiring must not regress that. + mgr = _make_python_manager(sr=False, replay=True) + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + assert mgr.mamba_cache.mamba_ssm_rand_seed is seed_buf + assert mgr.mamba_cache.old_x is not None # replay buffers also exist + + +def _build_cpp_hybrid(*, spec_config, use_replay: bool, sr: bool, max_batch_size: int = 4): + """Construct a CppMambaHybridCacheManager with one mamba + one attention + layer. Mirrors test_mamba_cache_manager._build_hybrid_with_mamba_layer + but parameterizes the replay / SR flags so we can exercise the + non-replay MTP SR layer-cache hand-off path.""" + from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import CppMambaHybridCacheManager + from tensorrt_llm._torch.pyexecutor.resource_manager import CacheTypeCpp + from tensorrt_llm.llmapi.llm_args import KvCacheConfig + + mamba_mask = [True, False] + attn_mask = [False, True] + mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + kv_cache_config = KvCacheConfig(max_tokens=512, enable_block_reuse=False) + return CppMambaHybridCacheManager( + mamba_d_state=8, + mamba_d_conv=4, + mamba_num_heads=4, + mamba_n_groups=1, + mamba_head_dim=8, + mamba_num_layers=1, + mamba_layer_mask=mamba_mask, + mamba_cache_dtype=torch.float16, + mamba_ssm_cache_dtype=torch.float16, + kv_cache_config=kv_cache_config, + kv_cache_type=CacheTypeCpp.SELF, + num_layers=1, + num_kv_heads=4, + head_dim=64, + tokens_per_block=32, + max_seq_len=128, + max_batch_size=max_batch_size, + mapping=mapping, + spec_config=spec_config, + layer_mask=attn_mask, + use_replay_state_update=use_replay, + mamba_ssm_stochastic_rounding=sr, + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_cpp_hybrid_non_replay_mtp_layer_cache_carries_rand_seed(): + """Regression: CppMambaHybridCacheManager.mamba_layer_cache() with + spec_config != None AND _use_replay_state_update == False AND + mamba_ssm_stochastic_rounding == True must still surface + mamba_ssm_rand_seed on the returned SpeculativeState. + + The mixer's non-replay MTP SR branch (mamba2_mixer.py) reads + `layer_cache.mamba_ssm_rand_seed` and asserts non-None. Iter5 review + caught the regression where the seed was only forwarded inside the + replay branch of mamba_layer_cache; this test pins both paths.""" + from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig + + spec_config = MTPDecodingConfig(max_draft_len=2) + mgr = _build_cpp_hybrid(spec_config=spec_config, use_replay=False, sr=True) + # Manager-level buffer must exist (SR is on). + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + assert seed_buf.dtype == torch.int64 + # Layer cache exposes the same buffer (Python identity equality). + layer_cache = mgr.mamba_layer_cache(0) + assert layer_cache is not None + assert layer_cache.mamba_ssm_rand_seed is mgr.mamba_ssm_rand_seed + # And the SpeculativeState is on the non-replay legacy branch. + assert layer_cache.intermediate_ssm is not None + assert layer_cache.old_x is None + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_cpp_hybrid_replay_mtp_layer_cache_still_carries_rand_seed(): + """Backward-compat: the replay branch must keep forwarding the seed + buffer through mamba_layer_cache.""" + from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig + + spec_config = MTPDecodingConfig(max_draft_len=2) + mgr = _build_cpp_hybrid(spec_config=spec_config, use_replay=True, sr=True) + seed_buf = mgr.get_mamba_ssm_rand_seed() + assert seed_buf is not None + layer_cache = mgr.mamba_layer_cache(0) + assert layer_cache is not None + assert layer_cache.mamba_ssm_rand_seed is mgr.mamba_ssm_rand_seed + # Replay-specific compact buffers are populated. + assert layer_cache.old_x is not None + assert layer_cache.cache_buf_idx is not None + assert layer_cache.prev_num_accepted_tokens is not None From 2f9b85a4df28081698f035992e18277329d13288 Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:52:28 +0800 Subject: [PATCH 164/308] [None][feat] Upgrade NIXL to v1.0.1 and UCX to 1.21 (#14436) Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- docker/common/install_nixl.sh | 2 +- docker/common/install_ucx.sh | 4 +-- jenkins/L0_Test.groovy | 33 ++++++++++++++++++- jenkins/current_image_tags.properties | 10 +++--- requirements-dev.txt | 2 +- tests/unittest/disaggregated/test_agent.py | 7 ++-- .../others/test_kv_cache_transceiver.py | 1 + 7 files changed, 47 insertions(+), 12 deletions(-) diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 905ff3ee87ce..f14398ced80c 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -4,7 +4,7 @@ set -ex GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" CUDA_PATH="/usr/local/cuda" -NIXL_VERSION="0.9.0" +NIXL_VERSION="v1.0.1" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH diff --git a/docker/common/install_ucx.sh b/docker/common/install_ucx.sh index 4a40679c804f..038033e26ba0 100644 --- a/docker/common/install_ucx.sh +++ b/docker/common/install_ucx.sh @@ -1,8 +1,8 @@ #!/bin/bash set -ex -UCX_VERSION="v1.20.x" -UCX_COMMIT="f656dbdf93e72e60b5d6ca78b9e3d9e744e789bd" +UCX_VERSION="v1.21.x" +UCX_COMMIT="167a4c6a311d9a42e30a37dcc01b8a3e73ea2826" UCX_INSTALL_PATH="/usr/local/ucx/" CUDA_PATH="/usr/local/cuda" UCX_REPO="https://github.com/openucx/ucx.git" diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a8572fe55a77..f457914a5277 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3359,7 +3359,24 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO | tar -C "${llmSrc}/tensorrt_llm" -xv """ withEnv(["MYPY_REQUIRE_BINDINGS=1"]) { - sh "cd ${llmSrc} && python3 -m pre_commit run type-check --all-files || (cat /root/.cache/pre-commit/pre-commit.log && /bin/false)" + // Strip the wheel's tensorrt_llm/libs (and its ucx/ subdir) out of + // LD_LIBRARY_PATH for this stage. The tar above populated + // ${llmSrc}/tensorrt_llm/{libs,bindings.*.so} from the wheel, so the + // source tree now has its own copies. With the wheel libs path on + // LD_LIBRARY_PATH, bindings.so's DT_NEEDED resolves libth_common.so + // to /tensorrt_llm/libs/ while _common.py explicitly loads + // /tensorrt_llm/libs/libth_common.so via torch.classes.load_library + // — two different absolute paths register the same Torch op twice + // and PyTorch aborts. Removing the wheel paths lets DT_NEEDED fall + // back to bindings.so's RUNPATH ($ORIGIN/libs = /tensorrt_llm/libs/), + // matching the explicit load. This restores pre-#722cbdd071 behavior + // for type-check only; UCX/NIXL kv_cache_transceiver tests above + // still see the new LD_LIBRARY_PATH. + sh """ + TRTLLM_WHEEL_LIBS=\$(pip3 show tensorrt_llm | awk -F': ' '/^Location:/ { print \$2 }')/tensorrt_llm/libs + export LD_LIBRARY_PATH=\$(echo "\$LD_LIBRARY_PATH" | tr ':' '\\n' | grep -vxF "\$TRTLLM_WHEEL_LIBS" | grep -vxF "\$TRTLLM_WHEEL_LIBS/ucx" | paste -sd:) + cd ${llmSrc} && python3 -m pre_commit run type-check --all-files || (cat /root/.cache/pre-commit/pre-commit.log && /bin/false) + """ } } } @@ -4435,6 +4452,20 @@ def launchTestJobs(pipeline, testFilter) } echo "###### Run LLMAPI tests Start ######" + // Resolve the real tensorrt_llm install location after pip install, + // and expose UCX shared libraries shipped inside the wheel + // (tensorrt_llm/libs/ucx/*.so and libtensorrt_llm_ucx_wrapper.so) + // so dlopen can find them at test runtime. + // Use `pip3 show` (metadata only) instead of `import tensorrt_llm`, + // because importing executes tensorrt_llm/__init__.py which prints a + // version banner to stdout and would pollute the captured path. + def trtllmLibsDir = sh( + script: "pip3 show tensorrt_llm | grep \"Location\" | awk -F\":\" '{ gsub(/ /, \"\", \$2); print \$2\"/tensorrt_llm/libs\"}'", + returnStdout: true, + ).replaceAll("\\s","") + libEnv += ["LD_LIBRARY_PATH+trtllm_ucx=${trtllmLibsDir}/ucx"] + libEnv += ["LD_LIBRARY_PATH+trtllm_libs=${trtllmLibsDir}"] + def config = VANILLA_CONFIG if (cpu_arch == AARCH64_TRIPLE) { config = LINUX_AARCH64_CONFIG diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 7eff9f5484eb..a8590bc254c5 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-x86_64-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202605060827-13616 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-sbsa-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202605060827-13616 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py310-trt10.15.1.29-skip-tritondevel-202605060827-13616 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py312-trt10.15.1.29-skip-tritondevel-202605060827-13616 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.15.1.29-skip-tritondevel-202605060827-13616 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-x86_64-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202605251043-14436 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.02-py3-sbsa-ubuntu24.04-trt10.15.1.29-skip-tritondevel-202605251043-14436 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py310-trt10.15.1.29-skip-tritondevel-202605251043-14436 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-rocky8-x86_64-rocky8-py312-trt10.15.1.29-skip-tritondevel-202605251043-14436 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.1.0-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.15.1.29-skip-tritondevel-202605251043-14436 diff --git a/requirements-dev.txt b/requirements-dev.txt index b94c600a07eb..6a6995a47afa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -41,7 +41,7 @@ opentelemetry-semantic-conventions-ai>=0.4.1 fuzzywuzzy==0.18.0 aiperf==0.6.0 nanobind>=2.9.0 -nixl==0.9.0 +nixl-cu13==1.0.1 cupti-python>=13.0,<13.2 nvidia-cuda-cupti>=13.0,<13.2 cxxfilt diff --git a/tests/unittest/disaggregated/test_agent.py b/tests/unittest/disaggregated/test_agent.py index 134c1ea1a40f..496cd0e7c8ec 100644 --- a/tests/unittest/disaggregated/test_agent.py +++ b/tests/unittest/disaggregated/test_agent.py @@ -55,9 +55,12 @@ class MemoryManager: allocated_memory: list[torch.Tensor] = field(default_factory=list) def allocate_memory( - self, size: int, name: str, memory_type=MemoryType.VRAM, device_id: int = 0 + self, size: int, name: str, memory_type: str = "VRAM", device_id: int = 0 ) -> RegMemoryDescs: - device = torch.device(f"cuda:{device_id}" if memory_type == MemoryType.VRAM else "cpu") + # `memory_type` is the string used by RegMemoryDescs (see base/agent.py:81); + # do not compare against `MemoryType.VRAM`, which is overridden to a C++ enum + # when the C++ binding is available. + device = torch.device(f"cuda:{device_id}" if memory_type == "VRAM" else "cpu") # Allocate memory block using torch.Tensor and track it block = torch.zeros(size, dtype=torch.uint8, device=device) diff --git a/tests/unittest/others/test_kv_cache_transceiver.py b/tests/unittest/others/test_kv_cache_transceiver.py index cf9d544a9ce0..1a38fd8b8d51 100644 --- a/tests/unittest/others/test_kv_cache_transceiver.py +++ b/tests/unittest/others/test_kv_cache_transceiver.py @@ -261,6 +261,7 @@ def create_hybrid_cache_manager(mapping, mamba_layer_mask=mamba_layer_mask, mamba_cache_dtype=mamba_conv_dtype, mamba_ssm_cache_dtype=mamba_ssm_dtype, + is_disagg=True, kv_cache_config=KvCacheConfig( max_tokens=256, enable_block_reuse=False, From 441eaae20d5cb45835da210744359c285f127327 Mon Sep 17 00:00:00 2001 From: tianyuz-nv Date: Mon, 1 Jun 2026 18:47:46 +0800 Subject: [PATCH 165/308] [None][feat] Refactor DWDP from CUDA IPC to CUDA VMM + MNNVL composite VA (#14453) Signed-off-by: tianyuz-nv Co-authored-by: dongxuy04 Co-authored-by: Jianbo Hu --- .../slurm/benchmark/disaggr_torch_dwdp.slurm | 11 + .../slurm/benchmark/start_worker_dwdp.sh | 29 + .../slurm/benchmark/submit_dwdp.py | 200 ++- .../_torch/custom_ops/cute_dsl_custom_ops.py | 372 ++---- ...ntiguous_gather_grouped_gemm_act_fusion.py | 558 +------- ...contiguous_grouped_gemm_finalize_fusion.py | 528 +------- tensorrt_llm/_torch/modules/dwdp/__init__.py | 86 ++ tensorrt_llm/_torch/modules/dwdp/page_pool.py | 400 ++++++ tensorrt_llm/_torch/modules/dwdp/setup.py | 1187 +++++++++++++++++ tensorrt_llm/_torch/modules/dwdp/specs.py | 439 ++++++ tensorrt_llm/_torch/modules/dwdp/transport.py | 646 +++++++++ tensorrt_llm/_torch/modules/dwdp/vmm.py | 596 +++++++++ .../_torch/modules/dwdp/weight_buffer.py | 742 +++++++++++ .../_torch/modules/dwdp/weight_manager.py | 519 +++++++ .../modules/fused_moe/configurable_moe.py | 38 +- .../modules/fused_moe/fused_moe_cute_dsl.py | 180 +-- .../_torch/modules/fused_moe/interface.py | 59 +- .../_torch/modules/fused_moe/moe_scheduler.py | 7 +- tensorrt_llm/_torch/pyexecutor/dwdp.py | 816 +++-------- .../_torch/pyexecutor/py_executor_creator.py | 30 +- tensorrt_llm/llmapi/llm_args.py | 8 +- tensorrt_llm/mapping.py | 89 +- .../test_dwdp_disaggregated_serving.py | 257 +++- .../test_lists/qa/llm_function_core.txt | 2 + .../integration/test_lists/test-db/l0_a10.yml | 4 + .../test-db/l0_gb200_multi_gpus.yml | 2 + tests/integration/test_lists/waives.txt | 1 - ...ntiguous_gather_grouped_gemm_act_fusion.py | 338 +---- ...contiguous_grouped_gemm_finalize_fusion.py | 189 +-- .../dwdp/test_dwdp_fixup_moe_backends.py | 342 +++++ .../_torch/modules/dwdp/test_dwdp_manager.py | 372 ++++++ .../_torch/modules/dwdp/test_dwdp_mapping.py | 190 +++ .../modules/dwdp/test_dwdp_peer_ranges.py | 171 +++ .../_torch/thop/parallel/test_cute_dsl_moe.py | 88 +- 34 files changed, 6859 insertions(+), 2637 deletions(-) create mode 100644 tensorrt_llm/_torch/modules/dwdp/__init__.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/page_pool.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/setup.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/specs.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/transport.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/vmm.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/weight_buffer.py create mode 100644 tensorrt_llm/_torch/modules/dwdp/weight_manager.py create mode 100644 tests/unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py create mode 100644 tests/unittest/_torch/modules/dwdp/test_dwdp_manager.py create mode 100644 tests/unittest/_torch/modules/dwdp/test_dwdp_mapping.py create mode 100644 tests/unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm index 145c3099267b..8835399d6c50 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm @@ -155,6 +155,17 @@ mpi_worker_config_file=${full_logdir}/mpi_worker_config.yaml if [ -f "${mpi_worker_config_base_file}" ]; then replace_placeholder "${mpi_worker_config_base_file}" "${all_nodes_str}" "${mpi_worker_config_file}" fi + +# Per-worker hostfile / gpu_map for srun --distribution=arbitrary. +# submit_dwdp.py writes *_base.txt with ; rewrite them +# here once we know the real hostnames. Each base file becomes a runtime +# file with the same name minus the ``_base`` suffix. +for base_file in "${full_logdir}"/hostfile_*_base.txt "${full_logdir}"/gpu_map_*_base.txt; do + [ -f "${base_file}" ] || continue + runtime_file="${base_file%_base.txt}.txt" + replace_placeholder "${base_file}" "${all_nodes_str}" "${runtime_file}" +done + client_cmds_base_file=${full_logdir}/client_cmds_base.sh client_cmds_file=${full_logdir}/client_cmds.sh replace_placeholder "${client_cmds_base_file}" "${all_nodes_str}" "${client_cmds_file}" diff --git a/examples/disaggregated/slurm/benchmark/start_worker_dwdp.sh b/examples/disaggregated/slurm/benchmark/start_worker_dwdp.sh index b0fe5bf88443..31a1757b0c25 100644 --- a/examples/disaggregated/slurm/benchmark/start_worker_dwdp.sh +++ b/examples/disaggregated/slurm/benchmark/start_worker_dwdp.sh @@ -18,6 +18,35 @@ unset UCX_TLS echo "SLURM_PROCID: ${SLURM_PROCID}, hostname: $(hostname)" +# NOTE: do NOT export CUDA_VISIBLE_DEVICES from this script. +# +# Restricting CUDA visibility to a single GPU (via CUDA driver isolation) +# breaks DWDP's intra-node peer GPU discovery: +# - VA composite cuMemMap imports of peer GPUs' MNNVL fabric handles +# - UCX cuda_ipc / cuda_copy intra-node transports for CTX->GEN KV +# - PyTorch torch.cuda.device_count() peer enumeration +# All of these need the process to *see* peer GPUs on the same node, +# even if it only computes on one of them. +# +# Empirically (R1/T2/T3/T4 vs T5 on dwdp3 dg=4): exporting +# ``CUDA_VISIBLE_DEVICES=`` blows TTFT std up 3x and drops +# per-CTX-GPU throughput by 15%, with TPOT unchanged. Letting +# trtllm-serve auto-pick the device from SLURM_LOCALID restores Phase +# D's full perf. +# +# For audit, log which GPU SLURM would have given this rank. With our +# compact-packing allocate_gpus, the natural mapping is: +# gpu_id = (gpu_map_mpi_worker.txt[rank][2]) if gpu_map exists +# == SLURM_LOCALID (always, by construction) +# so we log it but don't export. +gpu_map_file="${log_dir}/gpu_map_mpi_worker.txt" +if [ -f "${gpu_map_file}" ]; then + expected_gpu=$(awk -v p="${SLURM_PROCID}" '$1==p {print $3; exit}' "${gpu_map_file}") + echo "rank-to-gpu (arbitrary-dist path): SLURM_PROCID=${SLURM_PROCID} LOCALID=${SLURM_LOCALID} expected_gpu=${expected_gpu}" +else + echo "rank-to-gpu (block-dist path): SLURM_PROCID=${SLURM_PROCID} LOCALID=${SLURM_LOCALID}" +fi + if [ "${SLURM_PROCID}" -lt "${num_ctx_gpus}" ]; then worker_role="CTX" worker_env_var=${ctx_worker_env_var} diff --git a/examples/disaggregated/slurm/benchmark/submit_dwdp.py b/examples/disaggregated/slurm/benchmark/submit_dwdp.py index cc0529f52fad..43975ee418b2 100644 --- a/examples/disaggregated/slurm/benchmark/submit_dwdp.py +++ b/examples/disaggregated/slurm/benchmark/submit_dwdp.py @@ -208,6 +208,72 @@ def submit_dwdp_job(config, log_dir, dry_run): with open(os.path.join(log_dir, "allocations.json"), "w") as f: json.dump(allocations, f, indent=2) + # Placement summary: emitted up-front for audit/debug. Surfaces both the + # SLURM launcher path that will be used (block vs arbitrary, decided by + # divisibility of num_ctx_gpus by gpus_per_node) and where every CTX + # server, GEN server, and DWDP group physically lands. Cross-node DWDP + # groups and cross-node GEN TP allreduces are explicitly called out so + # they aren't a silent surprise when reading benchmark logs. + _num_ctx_gpus = ctx_num * ctx_world_size + _is_divisible_layout = _num_ctx_gpus % gpus_per_node == 0 + _num_dwdp_groups = ctx_num // dwdp_size if dwdp_size > 0 else 1 + print("=" * 72) + print("[DWDP launch summary]") + print( + f" Total GPU: {_num_ctx_gpus + gen_num * gen_world_size} " + f"Total nodes: {total_nodes} " + f"Launcher path: {'block' if _is_divisible_layout else 'arbitrary'} " + f"(num_ctx_gpus % gpus_per_node = {_num_ctx_gpus % gpus_per_node})" + ) + print(f" CTX servers: {ctx_num} (TP={ctx_world_size} each)") + print(f" GEN servers: {gen_num} (TP={gen_world_size} each)") + _per_node = {} + for _role in ("CTX", "GEN"): + for _sid in sorted(allocations.get(_role, {}).keys()): + for _host, _gpus in allocations[_role][_sid]["nodes"].items(): + _per_node.setdefault(_host, []).append((_role, _sid, _gpus)) + print(" Per-node placement:") + for _host in sorted(_per_node.keys()): + _ctxs = [(s, g) for r, s, g in _per_node[_host] if r == "CTX"] + _gens = [(s, g) for r, s, g in _per_node[_host] if r == "GEN"] + _parts = [] + if _ctxs: + _parts.append( + f"CTX {[s for s, _ in _ctxs]} (GPU {sorted(sum([g for _, g in _ctxs], []))})" + ) + if _gens: + _parts.append( + f"GEN {[s for s, _ in _gens]} (GPU {sorted(sum([g for _, g in _gens], []))})" + ) + _shared = " [CTX/GEN sharing node]" if _ctxs and _gens else "" + print(f" {_host}: {' + '.join(_parts)}{_shared}") + if _num_dwdp_groups and dwdp_size: + print(f" DWDP groups (dwdp_size={dwdp_size}, num_groups={_num_dwdp_groups}):") + for _g in range(_num_dwdp_groups): + _members = list(range(_g * dwdp_size, (_g + 1) * dwdp_size)) + _member_nodes = set() + for _c in _members: + if _c in allocations.get("CTX", {}): + _member_nodes.update(allocations["CTX"][_c]["nodes"].keys()) + _span = ( + "intra-node" + if len(_member_nodes) == 1 + else f"cross-node ({len(_member_nodes)} nodes via MNNVL fabric)" + ) + print(f" group {_g} = CTX {_members} {_span}") + if gen_num and gen_world_size > 1: + _gen_nodes = set() + for _sid in allocations.get("GEN", {}): + _gen_nodes.update(allocations["GEN"][_sid]["nodes"].keys()) + if len(_gen_nodes) == 1: + print(f" GEN TP={gen_world_size}: intra-node (NVLink allreduce)") + else: + print( + f" GEN TP={gen_world_size}: cross-node " + f"({len(_gen_nodes)} nodes; allreduce uses MNNVL fabric)" + ) + print("=" * 72) + server_config = convert_allocations_to_server_config(allocations) with open(os.path.join(log_dir, "server_config_base.yaml"), "w") as f: yaml.dump(server_config, f) @@ -231,19 +297,7 @@ def submit_dwdp_job(config, log_dir, dry_run): mpi_config_base_path, ) - ctx_node_list = [] - for sid in sorted(allocations.get("CTX", {}).keys()): - for node in allocations["CTX"][sid]["nodes"]: - if node not in ctx_node_list: - ctx_node_list.append(node) - gen_node_list = [] - for sid in sorted(allocations.get("GEN", {}).keys()): - for node in allocations["GEN"][sid]["nodes"]: - if node not in gen_node_list: - gen_node_list.append(node) - mpi_nodelist = ctx_node_list + gen_node_list total_mpi_tasks = ctx_num * ctx_world_size + gen_num * gen_world_size - mpi_num_nodes = len(mpi_nodelist) num_ctx_gpus = ctx_num * ctx_world_size worker_env_var = env_config.get("worker_env_var", "") ctx_worker_env_var = env_config.get("ctx_worker_env_var", "") @@ -255,28 +309,106 @@ def submit_dwdp_job(config, log_dir, dry_run): f" {gen_worker_env_var}" if gen_worker_env_var else "" ) - cmd = [ - "srun -l", - f"--nodelist {','.join(mpi_nodelist)}", - f"-N {mpi_num_nodes}", - f"--ntasks {total_mpi_tasks}", - f"--ntasks-per-node {gpus_per_node}", - f"--container-image {env_config['container_image']}", - f"--container-name {container_name}", - f"--container-mounts {container_mount_str}", - "--no-container-mount-home --mpi=pmix --overlap", - f"bash {os.path.join(script_dir, 'start_worker_dwdp.sh')}", - mpi_config_path, - str(slurm_config["numa_bind"]).lower(), - log_dir, - str(profiling_config["nsys_on"]).lower(), - f"'{profiling_config['ctx_profile_range']}'", - f"'{profiling_config['gen_profile_range']}'", - str(num_ctx_gpus), - f"'{dwdp_ctx_worker_env_var}'", - f"'{dwdp_gen_worker_env_var}'", - f"&> {log_dir}/3_output_workers.log &", - ] + # Dual-path launcher: pick block vs arbitrary distribution based on + # whether ``num_ctx_gpus`` aligns with ``gpus_per_node``. + # + # * Divisible (num_ctx_gpus % gpus_per_node == 0): + # CTX servers fill whole nodes, GEN starts on a fresh node, no + # CTX/GEN node sharing. Use the legacy block-distribution srun + # (``--nodelist + -N + --ntasks-per-node``). start_worker_dwdp.sh + # sets CUDA_VISIBLE_DEVICES from SLURM_LOCALID since rank-to-gpu + # mapping is implicit from block dist. + # + # * Non-divisible: + # CTX and GEN may share a physical node (compact packing). + # Block distribution can't express the resulting per-node task + # counts (e.g. 4+2 / 2+4), so we fall back to SLURM_HOSTFILE + + # --distribution=arbitrary, plus per-rank gpu_map_*.txt for + # start_worker_dwdp.sh to look up CUDA_VISIBLE_DEVICES by + # SLURM_PROCID. This path mirrors PR #13888's compact_packing + # mechanism upstream. + # + # Empirically (see refactor PR perf table), the block path runs + # 14-15% faster per-CTX-GPU than the arbitrary path on the same + # physical placement; the arbitrary path's cost is inherent to + # SLURM/PMIX behaviour under --distribution=arbitrary and not + # specific to this launcher. So we use it only when needed. + is_divisible_layout = num_ctx_gpus % gpus_per_node == 0 + + if is_divisible_layout: + # ---- Block-distribution path (perf-optimal) ---- + ctx_node_list = [] + for sid in sorted(allocations.get("CTX", {}).keys()): + for node in allocations["CTX"][sid]["nodes"]: + if node not in ctx_node_list: + ctx_node_list.append(node) + gen_node_list = [] + for sid in sorted(allocations.get("GEN", {}).keys()): + for node in allocations["GEN"][sid]["nodes"]: + if node not in gen_node_list and node not in ctx_node_list: + gen_node_list.append(node) + mpi_nodelist = ctx_node_list + gen_node_list + mpi_num_nodes = len(mpi_nodelist) + + cmd = [ + "srun -l", + f"--nodelist {','.join(mpi_nodelist)}", + f"-N {mpi_num_nodes}", + f"--ntasks {total_mpi_tasks}", + f"--ntasks-per-node {gpus_per_node}", + f"--container-image {env_config['container_image']}", + f"--container-name {container_name}", + f"--container-mounts {container_mount_str}", + "--no-container-mount-home --mpi=pmix --overlap", + f"bash {os.path.join(script_dir, 'start_worker_dwdp.sh')}", + mpi_config_path, + str(slurm_config["numa_bind"]).lower(), + log_dir, + str(profiling_config["nsys_on"]).lower(), + f"'{profiling_config['ctx_profile_range']}'", + f"'{profiling_config['gen_profile_range']}'", + str(num_ctx_gpus), + f"'{dwdp_ctx_worker_env_var}'", + f"'{dwdp_gen_worker_env_var}'", + f"&> {log_dir}/3_output_workers.log &", + ] + else: + # ---- Arbitrary-distribution path (functional for non-divisible) ---- + hostfile_base = os.path.join(log_dir, "hostfile_mpi_worker_base.txt") + gpu_map_base = os.path.join(log_dir, "gpu_map_mpi_worker_base.txt") + hostfile_runtime = os.path.join(log_dir, "hostfile_mpi_worker.txt") + with open(hostfile_base, "w") as hf, open(gpu_map_base, "w") as gm: + rank = 0 + for server_type in ("CTX", "GEN"): + for server_id in sorted(allocations.get(server_type, {}).keys()): + inst = allocations[server_type][server_id] + for host, gpus in inst["nodes"].items(): + for gpu in gpus: + hf.write(f"{host}\n") + gm.write(f"{rank} {host} {gpu}\n") + rank += 1 + + cmd = [ + f"SLURM_HOSTFILE={hostfile_runtime}", + "srun -l", + f"--ntasks {total_mpi_tasks}", + "--distribution=arbitrary", + f"--container-image {env_config['container_image']}", + f"--container-name {container_name}", + f"--container-mounts {container_mount_str}", + "--no-container-mount-home --mpi=pmix --overlap", + f"bash {os.path.join(script_dir, 'start_worker_dwdp.sh')}", + mpi_config_path, + str(slurm_config["numa_bind"]).lower(), + log_dir, + str(profiling_config["nsys_on"]).lower(), + f"'{profiling_config['ctx_profile_range']}'", + f"'{profiling_config['gen_profile_range']}'", + str(num_ctx_gpus), + f"'{dwdp_ctx_worker_env_var}'", + f"'{dwdp_gen_worker_env_var}'", + f"&> {log_dir}/3_output_workers.log &", + ] start_server_cmds.append(" ".join(cmd)) # Generate start server commands diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py index 013a30523e78..7f1efabeff99 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py @@ -235,23 +235,23 @@ class GatherGroupedGemmInputsHelper(GroupedGemmInputsHelper): - permuted_idx_to_expanded_idx specifies the gather pattern - Shape inference uses permuted_idx_to_expanded_idx size instead of a size - Input layout (positions 1, 3, 4 are lists for multi-B support): - 0: a - tensor, original input activation - 1: b_list - list of tensors, weight tensors - 2: a_sf - tensor, scale factor for a - 3: b_sf_list - list of tensors, scale factors for b - 4: alpha_list - list of tensors, per-expert scaling factors - 5: tile_idx_to_group_idx - tensor, tile to expert mapping - 6: tile_idx_to_mn_limit - tensor, tile M/N limits - 7: permuted_idx_to_expanded_idx - tensor, token permutation mapping - 8: num_non_exiting_tiles - tensor, number of valid tiles - 9: global_sf - tensor, global scale factor + Input tensor layout: + 0: a - Original input activation (not permuted) + 1: b - Weight tensor + 2: a_sf - Scale factor for a + 3: b_sf - Scale factor for b + 4: alpha - Per-expert scaling factor + 5: tile_idx_to_group_idx - Tile to expert mapping + 6: tile_idx_to_mn_limit - Tile M/N limits + 7: permuted_idx_to_expanded_idx - Token permutation mapping + 8: num_non_exiting_tiles - Number of valid tiles + 9: global_sf - Global scale factor """ # Override: use permuted_idx_to_expanded_idx for shape inference IDX_PERMUTED_IDX_TO_EXPANDED_IDX = 7 IDX_SHAPE_INFER = IDX_PERMUTED_IDX_TO_EXPANDED_IDX - def inputs_pre_hook(self, inputs: List) -> List: + def inputs_pre_hook(self, inputs: List[torch.Tensor]) -> List[torch.Tensor]: """Pre-hook for gather-based activation fusion kernel. Generates: @@ -259,22 +259,9 @@ def inputs_pre_hook(self, inputs: List) -> List: - tile_idx_to_mn_limit - permuted_idx_to_expanded_idx (for gather operation) - num_non_exiting_tiles - - Input layout (positions 1, 3, 4 are lists): - 0: a - tensor - 1: b_list - list of tensors - 2: a_sf - tensor - 3: b_sf_list - list of tensors - 4: alpha_list - list of tensors - 5: tile_idx_to_group_idx - tensor - 6: tile_idx_to_mn_limit - tensor - 7: permuted_idx_to_expanded_idx - tensor - 8: num_non_exiting_tiles - tensor - 9: global_sf - tensor """ - a, b_list, a_sf, b_sf_list, alpha_list, tile_idx_to_group_idx, \ - tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, \ - num_non_exiting_tiles, global_sf = inputs + a, b, a_sf, b_sf, alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, \ + permuted_idx_to_expanded_idx, num_non_exiting_tiles, global_sf = inputs # Verify permuted_idx_to_expanded_idx index matches the class constant assert inputs[ self. @@ -306,7 +293,7 @@ def inputs_pre_hook(self, inputs: List) -> List: local_num_experts=self.num_local_experts, tile_tokens_dim=self.tile_size, ) - return (a, b_list, a_sf, b_sf_list, alpha_list, tile_idx_to_group_idx, + return (a, b, a_sf, b_sf, alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, num_non_exiting_tiles, global_sf) @@ -1686,7 +1673,7 @@ def get_valid_tactics( a, b, *_ = inputs b_list = b if isinstance(b, (list, tuple)) else [b] m, k = a.size(0), a.size(1) * 2 - l = sum(bi.size(0) for bi in b_list) + l = sum(bi.size(0) for bi in b_list) # noqa: E741 n = b_list[0].size(1) mma_tiler_mn_candidates = [(self.tile_size, 128), @@ -1758,7 +1745,7 @@ def forward(self, inputs: List[torch.Tensor], assert alpha.dim() == 1 m, k = a.size(0), a.size(1) * 2 - l, n = b.size(0), b.size(1) + l, n = b.size(0), b.size(1) # noqa: E741 scale_k = k // self.scaling_vector_size assert m % self.tile_size == 0 assert k % (self.scaling_vector_size * 4) == 0 @@ -1943,8 +1930,7 @@ def __init__(self, local_expert_offset: int, tile_size: int, output_dtype: torch.dtype, - scaling_vector_size: int = 16, - b_tensor_l_sizes: Optional[Tuple[int, ...]] = None): + scaling_vector_size: int = 16): super().__init__() self.num_experts = num_experts self.top_k = top_k @@ -1955,7 +1941,6 @@ def __init__(self, assert output_dtype == torch.bfloat16 self.output_dtype = output_dtype self.scaling_vector_size = scaling_vector_size - self.b_tensor_l_sizes = b_tensor_l_sizes if (sm_version := get_sm_version()) not in (100, 103): raise ValueError( @@ -1976,7 +1961,6 @@ def unique_id(self): self.tile_size, self.output_dtype, self.scaling_vector_size, - self.b_tensor_l_sizes, ) def get_valid_tactics( @@ -1985,12 +1969,9 @@ def get_valid_tactics( profile: OptimizationProfile, **kwargs, ) -> List[Tuple[int, int]]: - a, b_list, *_ = inputs - if not isinstance(b_list, (list, tuple)): - raise TypeError("weight must be a list of tensors") + a, b, *_ = inputs m, k = a.size(0), a.size(1) * 2 - l = sum(bi.size(0) for bi in b_list) - n = b_list[0].size(1) + l, n = b.size(0), b.size(1) # noqa: E741 mma_tiler_mn_candidates = [(self.tile_size, 128), (self.tile_size, 256)] @@ -2056,45 +2037,29 @@ def get_tuning_config(self) -> TuningConfig: def forward(self, inputs: List[torch.Tensor], tactic: Optional[tuple]) -> torch.Tensor: - a, b_list, a_sf, b_sf_list, alpha_list, c, tile_idx_to_group_idx, tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, num_non_exiting_tiles, token_final_scales = inputs - if not isinstance(b_list, (list, tuple)): - raise TypeError("weight must be a list of tensors") - if not isinstance(b_sf_list, (list, tuple)): - raise TypeError("weight_scale must be a list of tensors") - if not isinstance(alpha_list, (list, tuple)): - raise TypeError("alpha must be a list of tensors") - assert len(b_list) == len(b_sf_list) == len(alpha_list) - b_tensor_l_sizes = tuple(bi.size(0) for bi in b_list) - - b0 = b_list[0] - b_sf0 = b_sf_list[0] - alpha0 = alpha_list[0] + a, b, a_sf, b_sf, alpha, c, tile_idx_to_group_idx, tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, num_non_exiting_tiles, token_final_scales = inputs assert a.dtype == torch.float4_e2m1fn_x2 assert a.dim() == 2 - assert b0.dtype == torch.float4_e2m1fn_x2 - assert b0.dim() == 3 + assert b.dtype == torch.float4_e2m1fn_x2 + assert b.dim() == 3 assert a_sf.dtype == torch.uint8 assert a_sf.dim() == 1 - assert b_sf0.dtype == torch.uint8 - assert b_sf0.dim() == 3 - assert alpha0.dtype == torch.float32 - assert alpha0.dim() == 1 + assert b_sf.dtype == torch.uint8 + assert b_sf.dim() == 3 + assert alpha.dtype == torch.float32 + assert alpha.dim() == 1 m, k = a.size(0), a.size(1) * 2 - sum(bi.size(0) for bi in b_list) - n = b0.size(1) + l, n = b.size(0), b.size(1) # noqa: E741 scale_k = k // self.scaling_vector_size assert m % self.tile_size == 0 assert k % (self.scaling_vector_size * 4) == 0 - assert b0.size(2) * 2 == k + assert b.size(2) * 2 == k assert a_sf.size(0) == m * scale_k - for bi, bsfi, ai in zip(b_list, b_sf_list, alpha_list): - assert bi.size(1) == n - assert bi.size(2) * 2 == k - assert bsfi.size(0) == bi.size(0) - assert bsfi.size(1) == n - assert bsfi.size(2) == scale_k - assert ai.size(0) == bi.size(0) + assert b_sf.size(0) == l + assert b_sf.size(1) == n + assert b_sf.size(2) == scale_k + assert alpha.size(0) == l assert c.dtype == self.output_dtype assert c.dim() == 2 @@ -2118,10 +2083,20 @@ def forward(self, inputs: List[torch.Tensor], a.data_ptr(), cute.AddressSpace.gmem, assumed_align=32) + b_ptr = make_ptr(cutlass.Float4E2M1FN, + b.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=32) a_sf_ptr = make_ptr(cutlass.Float8E4M3FN, a_sf.data_ptr(), cute.AddressSpace.gmem, assumed_align=16) + b_sf_ptr = make_ptr(cutlass.Float8E4M3FN, + b_sf.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=16) + alpha_ptr = make_ptr(cutlass.Float32, alpha.data_ptr(), + cute.AddressSpace.gmem) tile_idx_to_group_idx_ptr = make_ptr( cutlass.Int32, tile_idx_to_group_idx.data_ptr(), cute.AddressSpace.gmem) @@ -2142,20 +2117,6 @@ def forward(self, inputs: List[torch.Tensor], cute.AddressSpace.gmem, assumed_align=16) - b_ptr = tuple( - make_ptr(cutlass.Float4E2M1FN, - bi.data_ptr(), - cute.AddressSpace.gmem, - assumed_align=32) for bi in b_list) - b_sf_ptr = tuple( - make_ptr(cutlass.Float8E4M3FN, - bsfi.data_ptr(), - cute.AddressSpace.gmem, - assumed_align=16) for bsfi in b_sf_list) - alpha_ptr = tuple( - make_ptr(cutlass.Float32, ai.data_ptr(), cute.AddressSpace.gmem) - for ai in alpha_list) - torch_stream = torch.cuda.current_stream() stream = cuda.CUstream(torch_stream.cuda_stream) @@ -2169,7 +2130,7 @@ def forward(self, inputs: List[torch.Tensor], 0] == self.tile_size, f"Tactic ({tactic}) is incompatible with tile size ({self.tile_size})" cache_key = (self.scaling_vector_size, self.tile_size, mma_tiler_mn, - cluster_shape_mn, raster_along_m, b_tensor_l_sizes) + cluster_shape_mn, raster_along_m) if cache_key not in self.__class__.kernel_cache: gemm = self.__class__.kernel_class( sf_vec_size=self.scaling_vector_size, @@ -2177,7 +2138,6 @@ def forward(self, inputs: List[torch.Tensor], cluster_shape_mn=cluster_shape_mn, use_blkred=True, raster_along_m=raster_along_m, - b_tensor_l_sizes=b_tensor_l_sizes, ) # Compute max active clusters on current device hardware_info = cutlass.utils.HardwareInfo() @@ -2199,6 +2159,7 @@ def forward(self, inputs: List[torch.Tensor], m, n, k, + l, num_tokens, self.top_k, ] @@ -2230,6 +2191,7 @@ def forward(self, inputs: List[torch.Tensor], m, n, k, + l, num_tokens, self.top_k, ] @@ -2242,10 +2204,10 @@ def forward(self, inputs: List[torch.Tensor], device_types="cuda") def cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( input: torch.Tensor, - weight: List[torch.Tensor], + weight: torch.Tensor, input_scale: torch.Tensor, - weight_scale: List[torch.Tensor], - alpha: List[torch.Tensor], + weight_scale: torch.Tensor, + alpha: torch.Tensor, output: torch.Tensor, tile_idx_to_group_idx: torch.Tensor, tile_idx_to_mn_limit: torch.Tensor, @@ -2262,11 +2224,9 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( ) -> None: tuner = AutoTuner.get() - b_tensor_l_sizes = tuple(w.size(0) - for w in weight) if len(weight) > 1 else None runner = Sm100BlockScaledContiguousGroupedGemmFinalizeFusionRunner( num_experts, top_k, num_local_experts, local_expert_offset, - tile_size, output_dtype, scaling_vector_size, b_tensor_l_sizes) + tile_size, output_dtype, scaling_vector_size) inputs = [ input, weight, input_scale, weight_scale, alpha, output, @@ -2289,10 +2249,10 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( device_types="cuda") def cute_dsl_nvfp4_grouped_gemm_finalize_blackwell( input: torch.Tensor, - weight: List[torch.Tensor], + weight: torch.Tensor, input_scale: torch.Tensor, - weight_scale: List[torch.Tensor], - alpha: List[torch.Tensor], + weight_scale: torch.Tensor, + alpha: torch.Tensor, tile_idx_to_group_idx: torch.Tensor, tile_idx_to_mn_limit: torch.Tensor, permuted_idx_to_expanded_idx: torch.Tensor, @@ -2307,7 +2267,7 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_blackwell( scaling_vector_size: int = 16, ) -> torch.Tensor: num_tokens = token_final_scales.size(0) - n = weight[0].size(1) + n = weight.size(1) output = torch.zeros(num_tokens, n, dtype=output_dtype, @@ -2338,10 +2298,10 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_blackwell( "trtllm::cute_dsl_nvfp4_grouped_gemm_finalize_blackwell") def _( input: torch.Tensor, - weight: List[torch.Tensor], + weight: torch.Tensor, input_scale: torch.Tensor, - weight_scale: List[torch.Tensor], - alpha: List[torch.Tensor], + weight_scale: torch.Tensor, + alpha: torch.Tensor, tile_idx_to_group_idx: torch.Tensor, tile_idx_to_mn_limit: torch.Tensor, permuted_idx_to_expanded_idx: torch.Tensor, @@ -2356,7 +2316,7 @@ def _( scaling_vector_size: int = 16, ) -> torch.Tensor: num_tokens = token_final_scales.size(0) - n = weight[0].size(1) + n = weight.size(1) return torch.empty(num_tokens, n, dtype=output_dtype, @@ -2411,7 +2371,7 @@ def get_valid_tactics( ) -> List[Tuple[int, int]]: a, b, *_ = inputs m, k = a.size(0), a.size(1) * 2 - l, n = b.size(0), b.size(1) + l, n = b.size(0), b.size(1) # noqa: E741 mma_tiler_mn_candidates = [(self.tile_size, 128), (self.tile_size, 256)] @@ -2482,7 +2442,7 @@ def forward(self, inputs: List[torch.Tensor], assert alpha.dim() == 1 m, k = a.size(0), a.size(1) * 2 - l, n = b.size(0), b.size(1) + l, n = b.size(0), b.size(1) # noqa: E741 scale_k = k // self.scaling_vector_size interm_size = n // 2 assert m % self.tile_size == 0 @@ -2687,9 +2647,6 @@ class Sm100BlockScaledContiguousGatherGroupedGemmActFusionRunner( kernel_cache = dict() tuning_config_cache = dict() - # Maximum number of B tensors supported (must match kernel's MAX_B_TENSORS) - MAX_B_TENSORS = 4 - def __init__(self, num_experts: int, top_k: int, @@ -2697,13 +2654,10 @@ def __init__(self, local_expert_offset: int, tile_size: int, scaling_vector_size: int = 16, - b_tensor_l_sizes: Optional[Tuple[int, ...]] = None, activation_type: ActivationType = ActivationType.Swiglu): """Initialize the runner. Args: - b_tensor_l_sizes: Tuple of L sizes for each B tensor in multi-B mode. - None for single-B mode. Used for kernel cache key. activation_type: ``ActivationType`` for the fused epilogue. Only ``Swiglu`` (gated) and ``Relu2`` (non-gated) are supported. """ @@ -2720,7 +2674,6 @@ def __init__(self, ) self.tile_size = tile_size self.scaling_vector_size = scaling_vector_size - self.b_tensor_l_sizes = b_tensor_l_sizes if (sm_version := get_sm_version()) not in (100, 103): raise ValueError( @@ -2740,7 +2693,6 @@ def unique_id(self): self.local_expert_offset, self.tile_size, self.scaling_vector_size, - self.b_tensor_l_sizes, self.activation_type, ) @@ -2750,15 +2702,14 @@ def get_valid_tactics( profile: OptimizationProfile, **kwargs, ) -> List[Tuple[int, int]]: - # Tuning uses layout: a, b_list, a_sf, b_sf_list, alpha_list, ... + # Tuning uses layout: a, b, a_sf, b_sf, alpha, ... a = inputs[0] - b_list = inputs[1] # List of B tensors + b = inputs[1] permuted_idx_to_expanded_idx = inputs[7] # m is the permuted size from permuted_idx_to_expanded_idx, not from a m = permuted_idx_to_expanded_idx.size(0) k = a.size(1) * 2 - l = sum(bi.size(0) for bi in b_list) - n = b_list[0].size(1) + l, n = b.size(0), b.size(1) # noqa: E741 mma_tiler_mn_candidates = [(self.tile_size, 128), (self.tile_size, 256)] @@ -2799,8 +2750,7 @@ def get_tuning_config(self) -> TuningConfig: self.local_expert_offset, self.tile_size) # Tuning uses layout: - # a, b_list, a_sf, b_sf_list, alpha_list, tile_idx, tile_mn_limit, permuted_idx, ... - # Constraint indices adjusted for list inputs at positions 1, 3, 4 + # a, b, a_sf, b_sf, alpha, tile_idx, tile_mn_limit, permuted_idx, ... self.__class__.tuning_config_cache[key] = TuningConfig( # Use permuted_idx_to_expanded_idx (IDX_SHAPE_INFER) for tuning dynamic_tensor_specs=(DynamicTensorSpec( @@ -2824,46 +2774,41 @@ def get_tuning_config(self) -> TuningConfig: def forward(self, inputs: List, tactic: Optional[tuple]) -> torch.Tensor: - """Forward pass supporting both single tensor and list inputs. + """Forward pass. - Input layout (positions 1, 3, 4 are lists for multi-B support): + Input layout: 0: a - tensor - 1: b_list - list of tensors + 1: b - tensor 2: a_sf - tensor - 3: b_sf_list - list of tensors - 4: alpha_list - list of tensors + 3: b_sf - tensor + 4: alpha - tensor 5: tile_idx_to_group_idx - tensor 6: tile_idx_to_mn_limit - tensor 7: permuted_idx_to_expanded_idx - tensor 8: num_non_exiting_tiles - tensor 9: global_sf - tensor """ - a, b_list, a_sf, b_sf_list, alpha_list, tile_idx_to_group_idx, \ + a, b, a_sf, b_sf, alpha, tile_idx_to_group_idx, \ tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, \ num_non_exiting_tiles, global_sf = inputs - b_tensor_l_sizes = tuple(bi.size(0) for bi in b_list) - - b0 = b_list[0] # Use first B for shape inference - # Verify input dtypes and dimensions assert a.dtype == torch.float4_e2m1fn_x2 assert a.dim() == 2 - assert b0.dtype == torch.float4_e2m1fn_x2 - assert b0.dim() == 3 + assert b.dtype == torch.float4_e2m1fn_x2 + assert b.dim() == 3 assert a_sf.dtype == torch.uint8 assert a_sf.dim() == 2 - assert b_sf_list[0].dtype == torch.uint8 - assert b_sf_list[0].dim() == 3 - assert alpha_list[0].dtype == torch.float32 - assert alpha_list[0].dim() == 1 + assert b_sf.dtype == torch.uint8 + assert b_sf.dim() == 3 + assert alpha.dtype == torch.float32 + assert alpha.dim() == 1 # a.size(0) is orig_m (original input size before gather) # permuted_idx_to_expanded_idx.size(0) is m (permuted size after gather) orig_m, k = a.size(0), a.size(1) * 2 m = permuted_idx_to_expanded_idx.size(0) - n = b0.size(1) - sum(bi.size(0) for bi in b_list) + l, n = b.size(0), b.size(1) # noqa: E741 scale_k = k // self.scaling_vector_size interm_size = n // 2 if self.is_gated else n @@ -2873,7 +2818,7 @@ def forward(self, inputs: List, assert n % (self.scaling_vector_size * 4 * 2) == 0 else: assert n % (self.scaling_vector_size * 4) == 0 - assert b0.size(2) * 2 == k + assert b.size(2) * 2 == k assert a_sf.size(0) == orig_m assert a_sf.size(1) == scale_k @@ -2895,15 +2840,25 @@ def forward(self, inputs: List, dtype=a_sf.dtype, device=a_sf.device) - # Create common pointers + # Create pointers. a_ptr = make_ptr(cutlass.Float4E2M1FN, a.data_ptr(), cute.AddressSpace.gmem, assumed_align=32) + b_ptr = make_ptr(cutlass.Float4E2M1FN, + b.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=32) a_sf_ptr = make_ptr(cutlass.Float8E4M3FN, a_sf.data_ptr(), cute.AddressSpace.gmem, assumed_align=16) + b_sf_ptr = make_ptr(cutlass.Float8E4M3FN, + b_sf.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=16) + alpha_ptr = make_ptr(cutlass.Float32, alpha.data_ptr(), + cute.AddressSpace.gmem) c_ptr = make_ptr(cutlass.Float4E2M1FN, c.data_ptr(), cute.AddressSpace.gmem, @@ -2927,20 +2882,6 @@ def forward(self, inputs: List, global_sf_ptr = make_ptr(cutlass.Float32, global_sf.data_ptr(), cute.AddressSpace.gmem) - b_ptr = tuple( - make_ptr(cutlass.Float4E2M1FN, - bi.data_ptr(), - cute.AddressSpace.gmem, - assumed_align=32) for bi in b_list) - b_sf_ptr = tuple( - make_ptr(cutlass.Float8E4M3FN, - bsfi.data_ptr(), - cute.AddressSpace.gmem, - assumed_align=16) for bsfi in b_sf_list) - alpha_ptr = tuple( - make_ptr(cutlass.Float32, ai.data_ptr(), cute.AddressSpace.gmem) - for ai in alpha_list) - torch_stream = torch.cuda.current_stream() stream = cuda.CUstream(torch_stream.cuda_stream) @@ -2955,7 +2896,7 @@ def forward(self, inputs: List, cache_key = (self.scaling_vector_size, self.tile_size, self.top_k, mma_tiler_mn, cluster_shape_mn, raster_along_m, - b_tensor_l_sizes, self.activation_type) + self.activation_type) if cache_key not in self.__class__.kernel_cache: gemm = self.__class__.kernel_class( @@ -2965,7 +2906,6 @@ def forward(self, inputs: List, vectorized_f32=True, topk=self.top_k, raster_along_m=raster_along_m, - b_tensor_l_sizes=b_tensor_l_sizes, activation_type=self.activation_type, ) hardware_info = cutlass.utils.HardwareInfo() @@ -2989,6 +2929,7 @@ def forward(self, inputs: List, m, n, k, + l, ] compiled_gemm = cute.compile( @@ -3021,6 +2962,7 @@ def forward(self, inputs: List, m, n, k, + l, ] compiled_gemm(*exec_args, stream=stream) @@ -3028,15 +2970,15 @@ def forward(self, inputs: List, return c, c_sf @torch.library.custom_op( - "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b", + "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell", mutates_args=(), device_types="cuda") - def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( + def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( input: torch.Tensor, - weight: List[torch.Tensor], + weight: torch.Tensor, input_scale: torch.Tensor, - weight_scale: List[torch.Tensor], - alpha: List[torch.Tensor], + weight_scale: torch.Tensor, + alpha: torch.Tensor, tile_idx_to_group_idx: torch.Tensor, tile_idx_to_mn_limit: torch.Tensor, permuted_idx_to_expanded_idx: torch.Tensor, @@ -3050,24 +2992,22 @@ def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( scaling_vector_size: int = 16, activation_type: int = int(ActivationType.Swiglu), ) -> Tuple[torch.Tensor, torch.Tensor]: - """CuteDSL-based NVFP4 gather grouped GEMM with activation fusion (multi-B list interface). + """CuteDSL-based NVFP4 gather grouped GEMM with activation fusion. Supports ``ActivationType.Swiglu`` (gated) and ``ActivationType.Relu2`` - (non-gated) epilogues; other ``ActivationType`` values raise an assertion. - - Args: - weight: List of B tensors. Single-B mode: [b], multi-B mode: [b0, b1, ...]. - weight_scale: List of scale tensors, matching weight. - alpha: List of alpha tensors, matching weight. - activation_type: ``ActivationType`` value selecting the fused activation. + (non-gated) epilogues; other ``ActivationType`` values raise an + assertion in the runner. """ tuner = AutoTuner.get() - b_tensor_l_sizes = tuple(w.size(0) for w in weight) - runner = Sm100BlockScaledContiguousGatherGroupedGemmActFusionRunner( - num_experts, top_k, num_local_experts, local_expert_offset, - tile_size, scaling_vector_size, b_tensor_l_sizes, activation_type) + num_experts, + top_k, + num_local_experts, + local_expert_offset, + tile_size, + scaling_vector_size, + activation_type=activation_type) inputs = [ input, weight, input_scale, weight_scale, alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, @@ -3075,100 +3015,14 @@ def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( ] _, best_tactic = tuner.choose_one( - "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b", + "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell", [runner], runner.get_tuning_config(), inputs, ) - - # Call forward with inputs list output = runner.forward(inputs, tactic=best_tactic) return output - @torch.library.register_fake( - "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b" - ) - def _fake_multi_b( - input: torch.Tensor, - weight: List[torch.Tensor], - input_scale: torch.Tensor, - weight_scale: List[torch.Tensor], - alpha: List[torch.Tensor], - tile_idx_to_group_idx: torch.Tensor, - tile_idx_to_mn_limit: torch.Tensor, - permuted_idx_to_expanded_idx: torch.Tensor, - num_non_exiting_tiles: torch.Tensor, - global_sf: torch.Tensor, - num_experts: int, - top_k: int, - num_local_experts: int, - local_expert_offset: int, - tile_size: int, - scaling_vector_size: int = 16, - activation_type: int = int(ActivationType.Swiglu), - ) -> Tuple[torch.Tensor, torch.Tensor]: - m = permuted_idx_to_expanded_idx.size(0) - n = weight[0].size(1) - is_gated = is_gated_activation(ActivationType(activation_type)) - interm_size = n // 2 if is_gated else n - output = torch.empty(m, - interm_size // 2, - dtype=input.dtype, - device=input.device) - output_scale = torch.empty(m * interm_size // scaling_vector_size, - dtype=input_scale.dtype, - device=input_scale.device) - return output, output_scale - - @torch.library.custom_op( - "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell", - mutates_args=(), - device_types="cuda") - def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( - input: torch.Tensor, - weight: torch.Tensor, - input_scale: torch.Tensor, - weight_scale: torch.Tensor, - alpha: torch.Tensor, - tile_idx_to_group_idx: torch.Tensor, - tile_idx_to_mn_limit: torch.Tensor, - permuted_idx_to_expanded_idx: torch.Tensor, - num_non_exiting_tiles: torch.Tensor, - global_sf: torch.Tensor, - num_experts: int, - top_k: int, - num_local_experts: int, - local_expert_offset: int, - tile_size: int, - scaling_vector_size: int = 16, - activation_type: int = int(ActivationType.Swiglu), - ) -> Tuple[torch.Tensor, torch.Tensor]: - """CuteDSL-based NVFP4 gather grouped GEMM with activation fusion (single-B tensor interface). - - Thin wrapper: wraps single tensors into lists and calls - cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b. - ``activation_type`` must be ``ActivationType.Swiglu`` or ``ActivationType.Relu2``. - """ - return torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( - input, - [weight], - input_scale, - [weight_scale], - [alpha], - tile_idx_to_group_idx, - tile_idx_to_mn_limit, - permuted_idx_to_expanded_idx, - num_non_exiting_tiles, - global_sf, - num_experts, - top_k, - num_local_experts, - local_expert_offset, - tile_size, - scaling_vector_size, - activation_type, - ) - @torch.library.register_fake( "trtllm::cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell") def _fake_single_b( @@ -3905,7 +3759,7 @@ def get_valid_tactics( m = a.shape[0] k = a.shape[1] * 2 # fp4 packed in k dimension n = b.shape[0] * b.shape[1] # num_expert * weight_per_expert - l = 1 # dense GEMM + l = 1 # dense GEMM # noqa: E741 # Define candidates together mma_tiler_mn_candidates = [(128, 128), (128, 256), (256, 256)] @@ -3979,7 +3833,7 @@ def forward( m = a.shape[0] k = a.shape[1] * 2 # fp4 packed in k dimension n = b.shape[0] * b.shape[1] # num_expert * weight_per_expert - l = 1 # dense GEMM + l = 1 # dense GEMM # noqa: E741 n_out = n // 2 # SwiGLU output # Default tactic if not provided @@ -4200,7 +4054,7 @@ def _( m = input.shape[0] n = weight.shape[0] * weight.shape[1] # num_expert * weight_per_expert n_out = n // 2 # SwiGLU output - l = 1 # dense GEMM + l = 1 # dense GEMM # noqa: E741 if output_dtype == torch.float4_e2m1fn_x2: # FP4 packed: 2 elements per byte @@ -4290,7 +4144,7 @@ def get_valid_tactics( m = a.shape[0] k = a.shape[1] * 2 # fp4 packed in k dimension n = b.shape[0] - l = 1 # dense GEMM + l = 1 # dense GEMM # noqa: E741 # Define candidates mma_tiler_mn_candidates = [(128, 64), (128, 128), (128, 256), @@ -4378,7 +4232,7 @@ def forward( m = a.shape[0] k = a.shape[1] * 2 # fp4 packed in k dimension n = b.shape[0] - l = 1 # dense GEMM + l = 1 # dense GEMM # noqa: E741 # The kernel wrapper expects alpha_scale laid out token-major # (token has stride 1, expert has stride m), which gives diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py index 163903feb69a..7ac84efe9541 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py @@ -265,9 +265,6 @@ class BlockScaledContiguousGatherGroupedGemmKernel: ... ) """ - # Maximum number of B tensors supported - MAX_B_TENSORS = 4 - def __init__( self, sf_vec_size: int, @@ -276,7 +273,6 @@ def __init__( vectorized_f32: bool, topk: cutlass.Int64, raster_along_m: bool = False, - b_tensor_l_sizes: Optional[Tuple[int, ...]] = None, activation_type: ActivationType = ActivationType.Swiglu, ): """Initializes the configuration for a Blackwell blockscaled dense GEMM kernel with @@ -317,10 +313,6 @@ def __init__( :type vectorized_f32: bool :param topk: Number of experts selected per token (used for token ID mapping). :type topk: cutlass.Int64 - :param b_tensor_l_sizes: Optional tuple of L sizes for each B tensor. - E.g., (8, 8, 16) means 3 B tensors with L=8, 8, 16. Sum equals total L. - If None, single B tensor mode (backward compatible). - :type b_tensor_l_sizes: Optional[Tuple[int, ...]] :param activation_type: Fused activation. Must be ``ActivationType.Swiglu`` (gated, default) or ``ActivationType.Relu2`` (non-gated). :type activation_type: ActivationType @@ -408,26 +400,6 @@ def __init__( self.vectorized_f32 = vectorized_f32 - # Multi-B tensor configuration - if b_tensor_l_sizes is None: - self.num_b_tensors = 1 - self.b_tensor_l_sizes = None - # Offsets padded for safe indexing in kernel - self.b_tensor_l_offsets = (0,) + (2**30,) * self.MAX_B_TENSORS - else: - assert len(b_tensor_l_sizes) <= self.MAX_B_TENSORS, ( - f"Max {self.MAX_B_TENSORS} B tensors, got {len(b_tensor_l_sizes)}" - ) - self.num_b_tensors = len(b_tensor_l_sizes) - self.b_tensor_l_sizes = b_tensor_l_sizes - offsets = [0] - for l_size in b_tensor_l_sizes: - offsets.append(offsets[-1] + l_size) - # Pad to MAX_B_TENSORS + 1 for safe indexing - while len(offsets) < self.MAX_B_TENSORS + 1: - offsets.append(2**30) - self.b_tensor_l_offsets = tuple(offsets) - def _setup_attributes(self): """Set up configurations that are dependent on GEMM inputs @@ -721,25 +693,10 @@ def __call__( # Setup attributes that dependent on gemm inputs self._setup_attributes() - # Setup sfb tensors - create layout for each B tensor (use const_expr, not loop) + # Setup sfb tensor layout sfb_layout_0 = blockscaled_utils.tile_atom_to_shape_SF(b_tuple[0].shape, self.sf_vec_size) sfb_tensor_0 = cute.make_tensor(sfb_tuple[0].iterator, sfb_layout_0) sfb_tensors = [sfb_tensor_0] - if cutlass.const_expr(self.num_b_tensors >= 2): - sfb_layout_1 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[1].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[1].iterator, sfb_layout_1)) - if cutlass.const_expr(self.num_b_tensors >= 3): - sfb_layout_2 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[2].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[2].iterator, sfb_layout_2)) - if cutlass.const_expr(self.num_b_tensors >= 4): - sfb_layout_3 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[3].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[3].iterator, sfb_layout_3)) sfb_tuple = tuple(sfb_tensors) # Backward compat alias sfb = sfb_tuple[0] @@ -772,7 +729,7 @@ def __call__( ) atom_thr_size = cute.size(tiled_mma.thr_id.shape) - # Setup TMA ops (shared across all B tensors) + # Setup TMA ops b_op = sm100_utils.cluster_shape_to_tma_atom_B(self.cluster_shape_mn, tiled_mma.thr_id) sfb_op = sm100_utils.cluster_shape_to_tma_atom_SFB(self.cluster_shape_mn, tiled_mma.thr_id) b_smem_layout = cute.slice_(self.b_smem_layout_staged, (None, None, None, 0)) @@ -817,30 +774,12 @@ def _make_tma_b(b_tensor, sfb_tensor): ) return atom_b, tensor_b, atom_sfb, tensor_sfb - # Create TMA for all B tensors (use const_expr, not loop) + # Create TMA for the B tensor atom_b_0, tensor_b_0, atom_sfb_0, tensor_sfb_0 = _make_tma_b(b_tuple[0], sfb_tuple[0]) tma_atoms_b = [atom_b_0] tma_tensors_b = [tensor_b_0] tma_atoms_sfb = [atom_sfb_0] tma_tensors_sfb = [tensor_sfb_0] - if cutlass.const_expr(self.num_b_tensors >= 2): - atom_b_1, tensor_b_1, atom_sfb_1, tensor_sfb_1 = _make_tma_b(b_tuple[1], sfb_tuple[1]) - tma_atoms_b.append(atom_b_1) - tma_tensors_b.append(tensor_b_1) - tma_atoms_sfb.append(atom_sfb_1) - tma_tensors_sfb.append(tensor_sfb_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - atom_b_2, tensor_b_2, atom_sfb_2, tensor_sfb_2 = _make_tma_b(b_tuple[2], sfb_tuple[2]) - tma_atoms_b.append(atom_b_2) - tma_tensors_b.append(tensor_b_2) - tma_atoms_sfb.append(atom_sfb_2) - tma_tensors_sfb.append(tensor_sfb_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - atom_b_3, tensor_b_3, atom_sfb_3, tensor_sfb_3 = _make_tma_b(b_tuple[3], sfb_tuple[3]) - tma_atoms_b.append(atom_b_3) - tma_tensors_b.append(tensor_b_3) - tma_atoms_sfb.append(atom_sfb_3) - tma_tensors_sfb.append(tensor_sfb_3) tma_atoms_b = tuple(tma_atoms_b) tma_tensors_b = tuple(tma_tensors_b) tma_atoms_sfb = tuple(tma_atoms_sfb) @@ -1091,18 +1030,9 @@ def kernel( # Prefetch tma desc # if warp_idx == self.tma_b_warp_id: - # Prefetch TMA descriptors for all B tensors using const_expr conditions + # Prefetch TMA descriptors for the B tensor cpasync.prefetch_descriptor(tma_atoms_b[0]) cpasync.prefetch_descriptor(tma_atoms_sfb[0]) - if cutlass.const_expr(self.num_b_tensors >= 2): - cpasync.prefetch_descriptor(tma_atoms_b[1]) - cpasync.prefetch_descriptor(tma_atoms_sfb[1]) - if cutlass.const_expr(self.num_b_tensors >= 3): - cpasync.prefetch_descriptor(tma_atoms_b[2]) - cpasync.prefetch_descriptor(tma_atoms_sfb[2]) - if cutlass.const_expr(self.num_b_tensors >= 4): - cpasync.prefetch_descriptor(tma_atoms_b[3]) - cpasync.prefetch_descriptor(tma_atoms_sfb[3]) cpasync.prefetch_descriptor(tma_atom_c) use_2cta_instrs = cute.size(tiled_mma.thr_id.shape) == 2 @@ -1262,52 +1192,22 @@ def kernel( gA_mkl = cute.local_tile( mA_mkl, cute.slice_(self.cta_tile_shape_mnk, (None, 0, None)), (None, None, None) ) - # (bN, bK, loopN, loopK, loopL) - Use const_expr conditions for tuple indexing + # (bN, bK, loopN, loopK, loopL) gB_nkl_0 = cute.local_tile( mB_nkl_tuple[0], cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) ) - if cutlass.const_expr(self.num_b_tensors >= 2): - gB_nkl_1 = cute.local_tile( - mB_nkl_tuple[1], cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) - ) - if cutlass.const_expr(self.num_b_tensors >= 3): - gB_nkl_2 = cute.local_tile( - mB_nkl_tuple[2], cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) - ) - if cutlass.const_expr(self.num_b_tensors >= 4): - gB_nkl_3 = cute.local_tile( - mB_nkl_tuple[3], cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) - ) # (bM, bK, RestM, RestK, RestL) gSFA_mkl = cute.local_tile( mSFA_mkl, cute.slice_(self.cta_tile_shape_mnk_sfa, (None, 0, None)), (None, None, None) ) - # (bN, bK, RestN, RestK, RestL) - Use const_expr conditions for tuple indexing + # (bN, bK, RestN, RestK, RestL) gSFB_nkl_0 = cute.local_tile( mSFB_nkl_tuple[0], cute.slice_(self.mma_tiler_sfb, (0, None, None)), (None, None, None), ) - if cutlass.const_expr(self.num_b_tensors >= 2): - gSFB_nkl_1 = cute.local_tile( - mSFB_nkl_tuple[1], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 3): - gSFB_nkl_2 = cute.local_tile( - mSFB_nkl_tuple[2], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 4): - gSFB_nkl_3 = cute.local_tile( - mSFB_nkl_tuple[3], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) gToken_ml = cute.local_tile( token_id_mapping_tensor, cute.slice_(self.cta_tile_shape_mnk, (None, 0, 0)), (None,) @@ -1324,22 +1224,10 @@ def kernel( # thr_mma = tiled_mma.get_slice(mma_tile_coord_v) thr_mma_sfb = tiled_mma_sfb.get_slice(mma_tile_coord_v) - # (MMA, MMA_N, MMA_K, loopN, loopK, loopL) - const_expr conditions + # (MMA, MMA_N, MMA_K, loopN, loopK, loopL) tCgB_0 = thr_mma.partition_B(gB_nkl_0) - if cutlass.const_expr(self.num_b_tensors >= 2): - tCgB_1 = thr_mma.partition_B(gB_nkl_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - tCgB_2 = thr_mma.partition_B(gB_nkl_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - tCgB_3 = thr_mma.partition_B(gB_nkl_3) - # (MMA, MMA_N, MMA_K, RestN, RestK, RestL) - const_expr conditions + # (MMA, MMA_N, MMA_K, RestN, RestK, RestL) tCgSFB_0 = thr_mma_sfb.partition_B(gSFB_nkl_0) - if cutlass.const_expr(self.num_b_tensors >= 2): - tCgSFB_1 = thr_mma_sfb.partition_B(gSFB_nkl_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - tCgSFB_2 = thr_mma_sfb.partition_B(gSFB_nkl_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - tCgSFB_3 = thr_mma_sfb.partition_B(gSFB_nkl_3) # (MMA, MMA_M, MMA_N, loopM, loopN, loopL) tCgC = thr_mma.partition_C(gC_mnl) @@ -1353,7 +1241,7 @@ def kernel( sB_grouped = cute.group_modes(sB, 0, 3) sSFB_grouped = cute.group_modes(sSFB, 0, 3) - # TMA partition for B tensor 0 + # TMA partition for the B tensor tBsB_0, tBgB_0 = cpasync.tma_partition( tma_atoms_b[0], block_in_cluster_coord_vmnk[1], @@ -1371,60 +1259,6 @@ def kernel( tBsSFB_0 = cute.filter_zeros(tBsSFB_0) tBgSFB_0 = cute.filter_zeros(tBgSFB_0) - # TMA partition for B tensor 1 (tBsB shared memory partition is same for all, use _ to ignore) - if cutlass.const_expr(self.num_b_tensors >= 2): - _, tBgB_1 = cpasync.tma_partition( - tma_atoms_b[1], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - sB_grouped, - cute.group_modes(tCgB_1, 0, 3), - ) - _, tBgSFB_1 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[1], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - sSFB_grouped, - cute.group_modes(tCgSFB_1, 0, 3), - ) - tBgSFB_1 = cute.filter_zeros(tBgSFB_1) - - # TMA partition for B tensor 2 - if cutlass.const_expr(self.num_b_tensors >= 3): - _, tBgB_2 = cpasync.tma_partition( - tma_atoms_b[2], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - sB_grouped, - cute.group_modes(tCgB_2, 0, 3), - ) - _, tBgSFB_2 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[2], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - sSFB_grouped, - cute.group_modes(tCgSFB_2, 0, 3), - ) - tBgSFB_2 = cute.filter_zeros(tBgSFB_2) - - # TMA partition for B tensor 3 - if cutlass.const_expr(self.num_b_tensors >= 4): - _, tBgB_3 = cpasync.tma_partition( - tma_atoms_b[3], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - sB_grouped, - cute.group_modes(tCgB_3, 0, 3), - ) - _, tBgSFB_3 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[3], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - sSFB_grouped, - cute.group_modes(tCgSFB_3, 0, 3), - ) - tBgSFB_3 = cute.filter_zeros(tBgSFB_3) - # # Partition shared/tensor memory tensor for TiledMMA_A/B/C # @@ -1955,241 +1789,22 @@ def kernel( tBsSFB_pipe = tBsSFB_0[(None, b_producer_state.index)] tma_bar = b_pipeline.producer_get_barrier(b_producer_state) - # Select correct B tensor based on expert_idx - if cutlass.const_expr(self.num_b_tensors == 1): - # Single B tensor - original logic - tBgB_slice = tBgB_0[(None, mma_tile_coord_mnl[1], None, expert_idx)] - tBgSFB_slice = tBgSFB_0[(None, slice_n, None, expert_idx)] - cute.copy( - tma_atoms_b[0], - tBgB_slice[(None, b_producer_state.count)], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_slice[(None, b_producer_state.count)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - # Multi-B tensor - select based on expert_idx - # Use nested const_expr ifs to avoid index out of range at compile time - if cutlass.const_expr(self.num_b_tensors == 2): - # Exactly 2 B tensors - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, b_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, b_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif cutlass.const_expr(self.num_b_tensors == 3): - # Exactly 3 B tensors - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, b_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[2]: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, b_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_2 = expert_idx - self.b_tensor_l_offsets[2] - cute.copy( - tma_atoms_b[2], - tBgB_2[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_2, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[2], - tBgSFB_2[(None, slice_n, b_producer_state.count, local_l_2)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - # 4 B tensors - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, b_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[2]: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, b_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[3]: - local_l_2 = expert_idx - self.b_tensor_l_offsets[2] - cute.copy( - tma_atoms_b[2], - tBgB_2[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_2, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[2], - tBgSFB_2[(None, slice_n, b_producer_state.count, local_l_2)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_3 = expert_idx - self.b_tensor_l_offsets[3] - cute.copy( - tma_atoms_b[3], - tBgB_3[ - ( - None, - mma_tile_coord_mnl[1], - b_producer_state.count, - local_l_3, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[3], - tBgSFB_3[(None, slice_n, b_producer_state.count, local_l_3)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) + tBgB_slice = tBgB_0[(None, mma_tile_coord_mnl[1], None, expert_idx)] + tBgSFB_slice = tBgSFB_0[(None, slice_n, None, expert_idx)] + cute.copy( + tma_atoms_b[0], + tBgB_slice[(None, b_producer_state.count)], + tBsB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=b_full_mcast_mask, + ) + cute.copy( + tma_atoms_sfb[0], + tBgSFB_slice[(None, b_producer_state.count)], + tBsSFB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=sfb_full_mcast_mask, + ) b_producer_state.advance() peek_ab_empty_status = cutlass.Boolean(1) @@ -2634,37 +2249,7 @@ def kernel( # Get alpha for current group # expert_idx = mma_tile_coord_mnl[2] - - # Select alpha from correct tensor based on expert_idx - # Initialize alpha_val first to avoid DSL "None prior to if" error - alpha_val = alpha_tuple[0][expert_idx - self.b_tensor_l_offsets[0]] - if cutlass.const_expr(self.num_b_tensors == 1): - pass # Already initialized above - elif cutlass.const_expr(self.num_b_tensors == 2): - if expert_idx >= self.b_tensor_l_offsets[1]: - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif cutlass.const_expr(self.num_b_tensors == 3): - if ( - expert_idx >= self.b_tensor_l_offsets[1] - and expert_idx < self.b_tensor_l_offsets[2] - ): - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif expert_idx >= self.b_tensor_l_offsets[2]: - alpha_val = alpha_tuple[2][expert_idx - self.b_tensor_l_offsets[2]] - else: - # 4 B tensors - if ( - expert_idx >= self.b_tensor_l_offsets[1] - and expert_idx < self.b_tensor_l_offsets[2] - ): - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif ( - expert_idx >= self.b_tensor_l_offsets[2] - and expert_idx < self.b_tensor_l_offsets[3] - ): - alpha_val = alpha_tuple[2][expert_idx - self.b_tensor_l_offsets[2]] - elif expert_idx >= self.b_tensor_l_offsets[3]: - alpha_val = alpha_tuple[3][expert_idx - self.b_tensor_l_offsets[3]] + alpha_val = alpha_tuple[0][expert_idx] # # Slice to per mma tile index @@ -3699,12 +3284,12 @@ def can_implement( def wrapper( self, a_ptr: cute.Pointer, - b_ptr_tuple: Tuple[cute.Pointer, ...], + b_ptr: cute.Pointer, a_sf_ptr: cute.Pointer, - b_sf_ptr_tuple: Tuple[cute.Pointer, ...], + b_sf_ptr: cute.Pointer, c_ptr: cute.Pointer, c_sf_ptr: cute.Pointer, - alpha_ptr_tuple: Tuple[cute.Pointer, ...], + alpha_ptr: cute.Pointer, tile_idx_to_group_idx_ptr: cute.Pointer, tile_idx_to_mn_limit_ptr: cute.Pointer, token_id_mapping_ptr: cute.Pointer, @@ -3714,6 +3299,7 @@ def wrapper( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, + l: cutlass.Int64, # noqa: E741 tile_size: cutlass.Constexpr, scaling_vector_size: cutlass.Constexpr, max_active_clusters: cutlass.Constexpr, @@ -3721,18 +3307,16 @@ def wrapper( epilogue_op: cutlass.Constexpr = lambda x: x, activation_type: cutlass.Constexpr = ActivationType.Swiglu, ): - """Unified wrapper supporting both single-B and multi-B tensors. + """Single-B wrapper. - B tensors are always passed as tuples (length 1 for single-B). - L sizes are configured via b_tensor_l_sizes in __init__. - ``activation_type`` is an ``ActivationType`` value and must match the - one passed to ``__init__``; only ``Swiglu`` and ``Relu2`` are supported. + ``l`` is the number of experts in the (sole) B tensor. ``activation_type`` + must match the one passed to ``__init__``; only ``Swiglu`` and ``Relu2`` + are supported. """ is_gated = is_gated_activation(activation_type) scale_k = k // scaling_vector_size interm_size = n // 2 if is_gated else n num_tiles = m // tile_size - total_l = self.b_tensor_l_offsets[self.num_b_tensors] a = cute.make_tensor( a_ptr, layout=cute.make_ordered_layout((orig_m, k, 1), order=(1, 0, 2)) @@ -3746,74 +3330,18 @@ def wrapper( c_sf = cute.make_tensor( c_sf_ptr, layout=cute.make_ordered_layout( - (32, 4, m // 128, 4, interm_size // (scaling_vector_size * 4), total_l), + (32, 4, m // 128, 4, interm_size // (scaling_vector_size * 4), l), order=(2, 1, 4, 0, 3, 5), ), ) - - # Create B and alpha tensors using const_expr conditions - l_0 = self.b_tensor_l_sizes[0] - alpha_0 = cute.make_tensor(alpha_ptr_tuple[0], layout=cute.make_layout((l_0,))) - b_0 = cute.make_tensor( - b_ptr_tuple[0], layout=cute.make_ordered_layout((n, k, l_0), order=(1, 0, 2)) - ) - b_sf_0 = cute.make_tensor( - b_sf_ptr_tuple[0], + alpha = cute.make_tensor(alpha_ptr, layout=cute.make_layout((l,))) + b = cute.make_tensor(b_ptr, layout=cute.make_ordered_layout((n, k, l), order=(1, 0, 2))) + b_sf = cute.make_tensor( + b_sf_ptr, layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_0), order=(2, 1, 4, 0, 3, 5) + (32, 4, n // 128, 4, scale_k // 4, l), order=(2, 1, 4, 0, 3, 5) ), ) - b_tuple = [b_0] - b_sf_tuple = [b_sf_0] - alpha_tuple = [alpha_0] - - if cutlass.const_expr(self.num_b_tensors >= 2): - l_1 = self.b_tensor_l_sizes[1] - alpha_1 = cute.make_tensor(alpha_ptr_tuple[1], layout=cute.make_layout((l_1,))) - b_1 = cute.make_tensor( - b_ptr_tuple[1], layout=cute.make_ordered_layout((n, k, l_1), order=(1, 0, 2)) - ) - b_sf_1 = cute.make_tensor( - b_sf_ptr_tuple[1], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_1), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_1) - b_sf_tuple.append(b_sf_1) - alpha_tuple.append(alpha_1) - - if cutlass.const_expr(self.num_b_tensors >= 3): - l_2 = self.b_tensor_l_sizes[2] - alpha_2 = cute.make_tensor(alpha_ptr_tuple[2], layout=cute.make_layout((l_2,))) - b_2 = cute.make_tensor( - b_ptr_tuple[2], layout=cute.make_ordered_layout((n, k, l_2), order=(1, 0, 2)) - ) - b_sf_2 = cute.make_tensor( - b_sf_ptr_tuple[2], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_2), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_2) - b_sf_tuple.append(b_sf_2) - alpha_tuple.append(alpha_2) - - if cutlass.const_expr(self.num_b_tensors >= 4): - l_3 = self.b_tensor_l_sizes[3] - alpha_3 = cute.make_tensor(alpha_ptr_tuple[3], layout=cute.make_layout((l_3,))) - b_3 = cute.make_tensor( - b_ptr_tuple[3], layout=cute.make_ordered_layout((n, k, l_3), order=(1, 0, 2)) - ) - b_sf_3 = cute.make_tensor( - b_sf_ptr_tuple[3], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_3), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_3) - b_sf_tuple.append(b_sf_3) - alpha_tuple.append(alpha_3) tile_idx_to_group_idx = cute.make_tensor( tile_idx_to_group_idx_ptr, layout=cute.make_layout((num_tiles,)) @@ -3829,17 +3357,17 @@ def wrapper( return self( a, - tuple(b_tuple), + b, c, a_sf, - tuple(b_sf_tuple), + b_sf, c_sf, global_sf, tile_idx_to_group_idx, tile_idx_to_mn_limit, token_id_mapping, num_non_exiting_tiles, - tuple(alpha_tuple), + alpha, max_active_clusters=max_active_clusters, stream=stream, epilogue_op=epilogue_op, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py index 63fc0a94cbba..30d3864cf1d6 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py @@ -26,7 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import Optional, Tuple, Type, Union +from typing import Tuple, Type, Union import cuda.bindings.driver as cuda import cutlass @@ -201,9 +201,6 @@ class Sm100BlockScaledContiguousGroupedGemmFinalizeFusionKernel: ... ) """ - # Maximum number of B tensors supported - MAX_B_TENSORS = 4 - def __init__( self, sf_vec_size: int, @@ -211,7 +208,6 @@ def __init__( cluster_shape_mn: Tuple[int, int], use_blkred: bool = False, raster_along_m: bool = False, - b_tensor_l_sizes: Optional[Tuple[int, ...]] = None, ): """Initializes the configuration for a Blackwell blockscaled dense GEMM kernel. @@ -229,10 +225,6 @@ def __init__( :type cluster_shape_mn: Tuple[int, int] :param raster_along_m: Boolean, True to use raster along M. :type raster_along_m: bool - :param b_tensor_l_sizes: Optional tuple of L sizes for each B tensor. - E.g., (8, 8, 16) means 3 B tensors with L=8, 8, 16. Sum equals total L. - If None, single B tensor mode (backward compatible). - :type b_tensor_l_sizes: Optional[Tuple[int, ...]] """ self.sf_vec_size = sf_vec_size @@ -294,26 +286,6 @@ def __init__( # TMEM offset for final accumulator self.tmem_final_offset = 384 - # Multi-B tensor configuration - if b_tensor_l_sizes is None: - self.num_b_tensors = 1 - self.b_tensor_l_sizes = None - # Offsets padded for safe indexing in kernel - self.b_tensor_l_offsets = (0,) + (2**30,) * self.MAX_B_TENSORS - else: - assert len(b_tensor_l_sizes) <= self.MAX_B_TENSORS, ( - f"Max {self.MAX_B_TENSORS} B tensors, got {len(b_tensor_l_sizes)}" - ) - self.num_b_tensors = len(b_tensor_l_sizes) - self.b_tensor_l_sizes = b_tensor_l_sizes - offsets = [0] - for l_size in b_tensor_l_sizes: - offsets.append(offsets[-1] + l_size) - # Pad to MAX_B_TENSORS + 1 for safe indexing - while len(offsets) < self.MAX_B_TENSORS + 1: - offsets.append(2**30) - self.b_tensor_l_offsets = tuple(offsets) - def _setup_attributes(self): """Set up configurations that are dependent on GEMM inputs @@ -572,21 +544,6 @@ def __call__( sfb_layout_0 = blockscaled_utils.tile_atom_to_shape_SF(b_tuple[0].shape, self.sf_vec_size) sfb_tensor_0 = cute.make_tensor(sfb_tuple[0].iterator, sfb_layout_0) sfb_tensors = [sfb_tensor_0] - if cutlass.const_expr(self.num_b_tensors >= 2): - sfb_layout_1 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[1].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[1].iterator, sfb_layout_1)) - if cutlass.const_expr(self.num_b_tensors >= 3): - sfb_layout_2 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[2].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[2].iterator, sfb_layout_2)) - if cutlass.const_expr(self.num_b_tensors >= 4): - sfb_layout_3 = blockscaled_utils.tile_atom_to_shape_SF( - b_tuple[3].shape, self.sf_vec_size - ) - sfb_tensors.append(cute.make_tensor(sfb_tuple[3].iterator, sfb_layout_3)) sfb_tuple = tuple(sfb_tensors) # Backward compat alias sfb = sfb_tuple[0] @@ -684,30 +641,12 @@ def _make_tma_b(b_tensor, sfb_tensor): tensor_sfb = cute.make_tensor(tensor_sfb.iterator, tensor_sfb_new_layout) return atom_b, tensor_b, atom_sfb, tensor_sfb - # Create TMA for all B tensors (use const_expr, not loop) + # Create TMA for the B tensor atom_b_0, tensor_b_0, atom_sfb_0, tensor_sfb_0 = _make_tma_b(b_tuple[0], sfb_tuple[0]) tma_atoms_b = [atom_b_0] tma_tensors_b = [tensor_b_0] tma_atoms_sfb = [atom_sfb_0] tma_tensors_sfb = [tensor_sfb_0] - if cutlass.const_expr(self.num_b_tensors >= 2): - atom_b_1, tensor_b_1, atom_sfb_1, tensor_sfb_1 = _make_tma_b(b_tuple[1], sfb_tuple[1]) - tma_atoms_b.append(atom_b_1) - tma_tensors_b.append(tensor_b_1) - tma_atoms_sfb.append(atom_sfb_1) - tma_tensors_sfb.append(tensor_sfb_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - atom_b_2, tensor_b_2, atom_sfb_2, tensor_sfb_2 = _make_tma_b(b_tuple[2], sfb_tuple[2]) - tma_atoms_b.append(atom_b_2) - tma_tensors_b.append(tensor_b_2) - tma_atoms_sfb.append(atom_sfb_2) - tma_tensors_sfb.append(tensor_sfb_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - atom_b_3, tensor_b_3, atom_sfb_3, tensor_sfb_3 = _make_tma_b(b_tuple[3], sfb_tuple[3]) - tma_atoms_b.append(atom_b_3) - tma_tensors_b.append(tensor_b_3) - tma_atoms_sfb.append(atom_sfb_3) - tma_tensors_sfb.append(tensor_sfb_3) tma_atoms_b = tuple(tma_atoms_b) tma_tensors_b = tuple(tma_tensors_b) tma_atoms_sfb = tuple(tma_atoms_sfb) @@ -932,15 +871,6 @@ def kernel( cpasync.prefetch_descriptor(tma_atom_sfa) cpasync.prefetch_descriptor(tma_atoms_b[0]) cpasync.prefetch_descriptor(tma_atoms_sfb[0]) - if cutlass.const_expr(self.num_b_tensors >= 2): - cpasync.prefetch_descriptor(tma_atoms_b[1]) - cpasync.prefetch_descriptor(tma_atoms_sfb[1]) - if cutlass.const_expr(self.num_b_tensors >= 3): - cpasync.prefetch_descriptor(tma_atoms_b[2]) - cpasync.prefetch_descriptor(tma_atoms_sfb[2]) - if cutlass.const_expr(self.num_b_tensors >= 4): - cpasync.prefetch_descriptor(tma_atoms_b[3]) - cpasync.prefetch_descriptor(tma_atoms_sfb[3]) use_2cta_instrs = cute.size(tiled_mma.thr_id.shape) == 2 @@ -1078,24 +1008,6 @@ def kernel( cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None), ) - if cutlass.const_expr(self.num_b_tensors >= 2): - gB_nkl_1 = cute.local_tile( - mB_nkl_tuple[1], - cute.slice_(self.mma_tiler, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 3): - gB_nkl_2 = cute.local_tile( - mB_nkl_tuple[2], - cute.slice_(self.mma_tiler, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 4): - gB_nkl_3 = cute.local_tile( - mB_nkl_tuple[3], - cute.slice_(self.mma_tiler, (0, None, None)), - (None, None, None), - ) # (bM, bK, RestM, RestK, RestL) gSFA_mkl = cute.local_tile( @@ -1108,24 +1020,6 @@ def kernel( cute.slice_(self.mma_tiler_sfb, (0, None, None)), (None, None, None), ) - if cutlass.const_expr(self.num_b_tensors >= 2): - gSFB_nkl_1 = cute.local_tile( - mSFB_nkl_tuple[1], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 3): - gSFB_nkl_2 = cute.local_tile( - mSFB_nkl_tuple[2], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) - if cutlass.const_expr(self.num_b_tensors >= 4): - gSFB_nkl_3 = cute.local_tile( - mSFB_nkl_tuple[3], - cute.slice_(self.mma_tiler_sfb, (0, None, None)), - (None, None, None), - ) k_tile_cnt = cutlass.Int32(cute.size(gA_mkl, mode=[3])) @@ -1138,22 +1032,10 @@ def kernel( tCgA = thr_mma.partition_A(gA_mkl) # (MMA, MMA_N, MMA_K, loopN, loopK, loopL) tCgB_0 = thr_mma.partition_B(gB_nkl_0) - if cutlass.const_expr(self.num_b_tensors >= 2): - tCgB_1 = thr_mma.partition_B(gB_nkl_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - tCgB_2 = thr_mma.partition_B(gB_nkl_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - tCgB_3 = thr_mma.partition_B(gB_nkl_3) # (MMA, MMA_M, MMA_K, RestM, RestK, RestL) tCgSFA = thr_mma.partition_A(gSFA_mkl) # (MMA, MMA_N, MMA_K, RestN, RestK, RestL) tCgSFB_0 = thr_mma_sfb.partition_B(gSFB_nkl_0) - if cutlass.const_expr(self.num_b_tensors >= 2): - tCgSFB_1 = thr_mma_sfb.partition_B(gSFB_nkl_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - tCgSFB_2 = thr_mma_sfb.partition_B(gSFB_nkl_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - tCgSFB_3 = thr_mma_sfb.partition_B(gSFB_nkl_3) # # Partition global/shared tensor for TMA load A/B @@ -1180,30 +1062,6 @@ def kernel( cute.group_modes(sB, 0, 3), cute.group_modes(tCgB_0, 0, 3), ) - if cutlass.const_expr(self.num_b_tensors >= 2): - _, tBgB_1 = cpasync.tma_partition( - tma_atoms_b[1], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - cute.group_modes(sB, 0, 3), - cute.group_modes(tCgB_1, 0, 3), - ) - if cutlass.const_expr(self.num_b_tensors >= 3): - _, tBgB_2 = cpasync.tma_partition( - tma_atoms_b[2], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - cute.group_modes(sB, 0, 3), - cute.group_modes(tCgB_2, 0, 3), - ) - if cutlass.const_expr(self.num_b_tensors >= 4): - _, tBgB_3 = cpasync.tma_partition( - tma_atoms_b[3], - block_in_cluster_coord_vmnk[1], - b_cta_layout, - cute.group_modes(sB, 0, 3), - cute.group_modes(tCgB_3, 0, 3), - ) # TMA load SFA partition_S/D sfa_cta_layout = a_cta_layout @@ -1236,33 +1094,6 @@ def kernel( ) tBsSFB_0 = cute.filter_zeros(tBsSFB_0) tBgSFB_0 = cute.filter_zeros(tBgSFB_0) - if cutlass.const_expr(self.num_b_tensors >= 2): - _, tBgSFB_1 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[1], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - cute.group_modes(sSFB, 0, 3), - cute.group_modes(tCgSFB_1, 0, 3), - ) - tBgSFB_1 = cute.filter_zeros(tBgSFB_1) - if cutlass.const_expr(self.num_b_tensors >= 3): - _, tBgSFB_2 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[2], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - cute.group_modes(sSFB, 0, 3), - cute.group_modes(tCgSFB_2, 0, 3), - ) - tBgSFB_2 = cute.filter_zeros(tBgSFB_2) - if cutlass.const_expr(self.num_b_tensors >= 4): - _, tBgSFB_3 = cute.nvgpu.cpasync.tma_partition( - tma_atoms_sfb[3], - block_in_cluster_coord_sfb_vmnk[1], - sfb_cta_layout, - cute.group_modes(sSFB, 0, 3), - cute.group_modes(tCgSFB_3, 0, 3), - ) - tBgSFB_3 = cute.filter_zeros(tBgSFB_3) # # Partition shared/tensor memory tensor for TiledMMA_A/B/C @@ -1495,235 +1326,22 @@ def kernel( tma_bar_ptr=tma_bar, mcast_mask=sfa_full_mcast_mask, ) - # Select correct B tensor based on expert_idx - if cutlass.const_expr(self.num_b_tensors == 1): - tBgB_slice = tBgB_0[(None, mma_tile_coord_mnl[1], None, expert_idx)] - tBgSFB_slice = tBgSFB_0[(None, slice_n, None, expert_idx)] - cute.copy( - tma_atoms_b[0], - tBgB_slice[(None, ab_producer_state.count)], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_slice[(None, ab_producer_state.count)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - if cutlass.const_expr(self.num_b_tensors == 2): - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, ab_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, ab_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif cutlass.const_expr(self.num_b_tensors == 3): - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, ab_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[2]: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, ab_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_2 = expert_idx - self.b_tensor_l_offsets[2] - cute.copy( - tma_atoms_b[2], - tBgB_2[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_2, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[2], - tBgSFB_2[(None, slice_n, ab_producer_state.count, local_l_2)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - if expert_idx < self.b_tensor_l_offsets[1]: - local_l_0 = expert_idx - self.b_tensor_l_offsets[0] - cute.copy( - tma_atoms_b[0], - tBgB_0[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_0, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[0], - tBgSFB_0[(None, slice_n, ab_producer_state.count, local_l_0)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[2]: - local_l_1 = expert_idx - self.b_tensor_l_offsets[1] - cute.copy( - tma_atoms_b[1], - tBgB_1[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_1, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[1], - tBgSFB_1[(None, slice_n, ab_producer_state.count, local_l_1)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - elif expert_idx < self.b_tensor_l_offsets[3]: - local_l_2 = expert_idx - self.b_tensor_l_offsets[2] - cute.copy( - tma_atoms_b[2], - tBgB_2[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_2, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[2], - tBgSFB_2[(None, slice_n, ab_producer_state.count, local_l_2)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) - else: - local_l_3 = expert_idx - self.b_tensor_l_offsets[3] - cute.copy( - tma_atoms_b[3], - tBgB_3[ - ( - None, - mma_tile_coord_mnl[1], - ab_producer_state.count, - local_l_3, - ) - ], - tBsB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=b_full_mcast_mask, - ) - cute.copy( - tma_atoms_sfb[3], - tBgSFB_3[(None, slice_n, ab_producer_state.count, local_l_3)], - tBsSFB_pipe, - tma_bar_ptr=tma_bar, - mcast_mask=sfb_full_mcast_mask, - ) + tBgB_slice = tBgB_0[(None, mma_tile_coord_mnl[1], None, expert_idx)] + tBgSFB_slice = tBgSFB_0[(None, slice_n, None, expert_idx)] + cute.copy( + tma_atoms_b[0], + tBgB_slice[(None, ab_producer_state.count)], + tBsB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=b_full_mcast_mask, + ) + cute.copy( + tma_atoms_sfb[0], + tBgSFB_slice[(None, ab_producer_state.count)], + tBsSFB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=sfb_full_mcast_mask, + ) # Peek (try_wait) AB buffer empty for k_tile = prefetch_k_tile_cnt + k_tile + 1 ab_producer_state.advance() @@ -2061,33 +1679,7 @@ def kernel( # expert_idx = mma_tile_coord_mnl[2] - alpha_val = alpha_tuple[0][expert_idx - self.b_tensor_l_offsets[0]] - if cutlass.const_expr(self.num_b_tensors == 1): - pass - elif cutlass.const_expr(self.num_b_tensors == 2): - if expert_idx >= self.b_tensor_l_offsets[1]: - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif cutlass.const_expr(self.num_b_tensors == 3): - if ( - expert_idx >= self.b_tensor_l_offsets[1] - and expert_idx < self.b_tensor_l_offsets[2] - ): - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif expert_idx >= self.b_tensor_l_offsets[2]: - alpha_val = alpha_tuple[2][expert_idx - self.b_tensor_l_offsets[2]] - else: - if ( - expert_idx >= self.b_tensor_l_offsets[1] - and expert_idx < self.b_tensor_l_offsets[2] - ): - alpha_val = alpha_tuple[1][expert_idx - self.b_tensor_l_offsets[1]] - elif ( - expert_idx >= self.b_tensor_l_offsets[2] - and expert_idx < self.b_tensor_l_offsets[3] - ): - alpha_val = alpha_tuple[2][expert_idx - self.b_tensor_l_offsets[2]] - elif expert_idx >= self.b_tensor_l_offsets[3]: - alpha_val = alpha_tuple[3][expert_idx - self.b_tensor_l_offsets[3]] + alpha_val = alpha_tuple[0][expert_idx] tile_m_start = tile_info[0] * self.cta_tile_shape_mnk[0] permuted_row = tile_m_start + epi_tidx @@ -2785,11 +2377,11 @@ def can_implement( def wrapper( self, a_ptr: cute.Pointer, - b_ptr_tuple: Tuple[cute.Pointer, ...], + b_ptr: cute.Pointer, a_sf_ptr: cute.Pointer, - b_sf_ptr_tuple: Tuple[cute.Pointer, ...], + b_sf_ptr: cute.Pointer, c_ptr: cute.Pointer, - alpha_ptr_tuple: Tuple[cute.Pointer, ...], + alpha_ptr: cute.Pointer, tile_idx_to_group_idx_ptr: cute.Pointer, tile_idx_to_mn_limit_ptr: cute.Pointer, permuted_idx_to_expanded_idx_ptr: cute.Pointer, @@ -2798,6 +2390,7 @@ def wrapper( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, + l: cutlass.Int64, # noqa: E741 num_tokens: cutlass.Int64, top_k: cutlass.Int64, tile_size: cutlass.Constexpr, @@ -2806,10 +2399,9 @@ def wrapper( stream: cuda.CUstream, epilogue_op: cutlass.Constexpr = lambda x: x, ): - """Unified wrapper supporting both single-B and multi-B tensors. + """Single-B wrapper. - B tensors are always passed as tuples (length 1 for single-B). - L sizes are configured via b_tensor_l_sizes in __init__. + ``l`` is the number of experts in the (sole) B tensor. """ scale_k = k // scaling_vector_size num_tiles = m // tile_size @@ -2825,68 +2417,14 @@ def wrapper( c_ptr, layout=cute.make_ordered_layout((num_tokens, n, 1), order=(1, 0, 2)) ) - l_0 = self.b_tensor_l_sizes[0] - alpha_0 = cute.make_tensor(alpha_ptr_tuple[0], layout=cute.make_layout((l_0,))) - b_0 = cute.make_tensor( - b_ptr_tuple[0], layout=cute.make_ordered_layout((n, k, l_0), order=(1, 0, 2)) - ) - b_sf_0 = cute.make_tensor( - b_sf_ptr_tuple[0], + alpha = cute.make_tensor(alpha_ptr, layout=cute.make_layout((l,))) + b = cute.make_tensor(b_ptr, layout=cute.make_ordered_layout((n, k, l), order=(1, 0, 2))) + b_sf = cute.make_tensor( + b_sf_ptr, layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_0), order=(2, 1, 4, 0, 3, 5) + (32, 4, n // 128, 4, scale_k // 4, l), order=(2, 1, 4, 0, 3, 5) ), ) - b_tuple = [b_0] - b_sf_tuple = [b_sf_0] - alpha_tuple = [alpha_0] - - if cutlass.const_expr(self.num_b_tensors >= 2): - l_1 = self.b_tensor_l_sizes[1] - alpha_1 = cute.make_tensor(alpha_ptr_tuple[1], layout=cute.make_layout((l_1,))) - b_1 = cute.make_tensor( - b_ptr_tuple[1], layout=cute.make_ordered_layout((n, k, l_1), order=(1, 0, 2)) - ) - b_sf_1 = cute.make_tensor( - b_sf_ptr_tuple[1], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_1), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_1) - b_sf_tuple.append(b_sf_1) - alpha_tuple.append(alpha_1) - - if cutlass.const_expr(self.num_b_tensors >= 3): - l_2 = self.b_tensor_l_sizes[2] - alpha_2 = cute.make_tensor(alpha_ptr_tuple[2], layout=cute.make_layout((l_2,))) - b_2 = cute.make_tensor( - b_ptr_tuple[2], layout=cute.make_ordered_layout((n, k, l_2), order=(1, 0, 2)) - ) - b_sf_2 = cute.make_tensor( - b_sf_ptr_tuple[2], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_2), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_2) - b_sf_tuple.append(b_sf_2) - alpha_tuple.append(alpha_2) - - if cutlass.const_expr(self.num_b_tensors >= 4): - l_3 = self.b_tensor_l_sizes[3] - alpha_3 = cute.make_tensor(alpha_ptr_tuple[3], layout=cute.make_layout((l_3,))) - b_3 = cute.make_tensor( - b_ptr_tuple[3], layout=cute.make_ordered_layout((n, k, l_3), order=(1, 0, 2)) - ) - b_sf_3 = cute.make_tensor( - b_sf_ptr_tuple[3], - layout=cute.make_ordered_layout( - (32, 4, n // 128, 4, scale_k // 4, l_3), order=(2, 1, 4, 0, 3, 5) - ), - ) - b_tuple.append(b_3) - b_sf_tuple.append(b_sf_3) - alpha_tuple.append(alpha_3) tile_idx_to_group_idx = cute.make_tensor( tile_idx_to_group_idx_ptr, layout=cute.make_layout((num_tiles,)) @@ -2907,14 +2445,14 @@ def wrapper( return self( a, - tuple(b_tuple), + b, c, a_sf, - tuple(b_sf_tuple), + b_sf, tile_idx_to_group_idx, num_non_exiting_tiles, tile_idx_to_mn_limit, - tuple(alpha_tuple), + alpha, max_active_clusters=max_active_clusters, stream=stream, permuted_idx_to_expanded_idx=permuted_idx_to_expanded_idx, diff --git a/tensorrt_llm/_torch/modules/dwdp/__init__.py b/tensorrt_llm/_torch/modules/dwdp/__init__.py new file mode 100644 index 000000000000..44207e828757 --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/__init__.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DWDP (Distributed Weight Data Parallelism) VA-based infrastructure. + +MNNVL-based physical memory allocation, composite VA layout, page-aligned +mapping, and double buffer lifecycle management for DWDP expert weights. + +Key design principles: +- No D2D copy: local experts are always zero-copy via cuMemMap +- Per-layer weight specs: different MoE layers may have different weight + shapes/dtypes +- Page pool: remote buffer physical memory is a pool of pages assigned + layer-by-layer +- Separation of concerns: WeightBuffer owns VA layout; Transport owns handle + exchange; WeightManager owns scheduling +- MPI everywhere: single communication backend (replaces tekit's TCPDWDPStore) +""" + +from .page_pool import PagePool, compute_slot_sizes +from .setup import setup_dwdp +from .specs import ( + EdgeInfo, + LayerWeightSpecs, + MnnvlHandleSet, + PageAlignedLayout, + PeerRange, + PeerRanges, + WeightSpec, + compute_peer_ranges, + lookup_owner, +) +from .transport import DWDPTransport +from .vmm import ( + VARegion, + VMMHandle, + align_down, + align_up, + get_allocation_granularity, + get_allocation_prop, + tensor_from_ptr, +) +from .weight_buffer import WeightBuffer +from .weight_manager import DWDPWeightManager + +__all__ = [ + # Data classes + "WeightSpec", + "LayerWeightSpecs", + "MnnvlHandleSet", + "EdgeInfo", + "PageAlignedLayout", + "PeerRange", + "PeerRanges", + "compute_peer_ranges", + "lookup_owner", + # VMM utilities + "align_up", + "align_down", + "get_allocation_prop", + "get_allocation_granularity", + "tensor_from_ptr", + "VMMHandle", + "VARegion", + # Page pool + "PagePool", + "compute_slot_sizes", + # Main classes + "DWDPTransport", + "WeightBuffer", + "DWDPWeightManager", + # Setup orchestration + "setup_dwdp", +] diff --git a/tensorrt_llm/_torch/modules/dwdp/page_pool.py b/tensorrt_llm/_torch/modules/dwdp/page_pool.py new file mode 100644 index 000000000000..a89badeba87c --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/page_pool.py @@ -0,0 +1,400 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Page pool for DWDP remote double buffer. + +The page pool manages per-page fabric handles for the remote regions of the +composite buffer. Two pools (slot 0, slot 1) support double buffering. + +Each pool is a list of page-sized fabric handles that can be mapped at different +VA positions for different layers. Pages are reused across layers in the same slot: + - Layer 3 (buf_idx=0): pool[0] pages 0..K for pre/post regions + - Layer 5 (buf_idx=0): SAME pool[0] pages (double buffer reuse) + - Layer 4 (buf_idx=1): pool[1] pages 0..M + +This naturally handles heterogeneous layers: each layer maps exactly the pages +it needs, even if remote sizes differ across layers. + +Verified on GB200: + - Per-page fabric handles composited into one VA: PASS + - Same page handles reused across layers: PASS + - Non-fabric handles cannot be mapped: FAIL (not supported on GB200) +""" + +from __future__ import annotations + +from typing import List, Tuple + +from tensorrt_llm.logger import logger + +from .vmm import ( + align_up, + create_local_handle, + get_allocation_granularity, + map_handle, + release_handle, + unmap_va, +) + + +class PagePool: + """Pool of per-page fabric handles for remote double buffer. + + Two pools (slot 0, slot 1) for double buffering. Each pool is a list + of page-sized fabric handles. Different layers map these pages at + different VA positions within their composite VA. + + Pages are reused across layers in the same slot: + - Layer 3 (buf_idx=0): pool[0] pages 0..K for pre/post regions + - Layer 5 (buf_idx=0): SAME pool[0] pages (double buffer reuse) + - Layer 4 (buf_idx=1): pool[1] pages 0..M + + Pool page size vs VMM granularity: + Each pool handle is ``page_size`` bytes (default 8 * granularity = + 16MB on GB200). Larger pool pages reduce the number of cuMemMap + calls by 8x, avoiding the CUDA driver internal tracking limit + (~130K mappings per process) that causes OOM when 4 processes share + a GB200 node with 2MB pages. + + Attributes: + device_id: CUDA device ordinal. + granularity: VMM page granularity in bytes (typically 2MB). + page_size: Pool page handle size in bytes (typically 16MB). + slot_sizes: Size of each slot in bytes. + slot_pages: Number of pool pages in each slot. + """ + + __slots__ = ( + "_device_id", + "_granularity", + "_page_size", + "_slot_sizes", + "_slot_pages", + "_page_handles", + "_released", + ) + + # Default pool page multiplier: each pool handle = 8 * granularity. + # On GB200 with 2MB granularity this gives 16MB pool pages, reducing + # cuMemMap calls by 8x compared to 1 * granularity. + DEFAULT_PAGE_SIZE_MULTIPLIER = 8 + + def __init__( + self, + slot_sizes: List[int], + device_id: int, + granularity: int | None = None, + page_size: int | None = None, + ): + """Create a page pool with the specified slot sizes. + + Args: + slot_sizes: [max_remote_bytes_slot0, max_remote_bytes_slot1] + Computed as max over all layers assigned to each slot. + device_id: CUDA device ordinal. + granularity: VMM page granularity in bytes. If None, queries the + device. + page_size: Pool page handle size in bytes. Must be a positive + multiple of ``granularity``. If None, defaults to + ``DEFAULT_PAGE_SIZE_MULTIPLIER * granularity`` (16MB on GB200). + """ + self._device_id = device_id + + if granularity is None: + self._granularity = get_allocation_granularity(device_id) + else: + self._granularity = granularity + + if page_size is None: + self._page_size = self.DEFAULT_PAGE_SIZE_MULTIPLIER * self._granularity + else: + if page_size <= 0 or page_size % self._granularity != 0: + raise ValueError( + f"page_size must be a positive multiple of granularity " + f"({self._granularity}), got {page_size}" + ) + # page_size must also be a power of 2 because vmm.align_up + # requires it (used in map_pages for safety alignment). + if (page_size & (page_size - 1)) != 0: + raise ValueError(f"page_size must be a power of 2, got {page_size}") + self._page_size = page_size + + self._slot_sizes = list(slot_sizes) + # Number of pool pages per slot, using page_size (not granularity). + self._slot_pages = [ + align_up(size, self._page_size) // self._page_size for size in slot_sizes + ] + + # Allocate per-page LOCAL handles for each slot. + # Uses HANDLE_TYPE_NONE (not fabric) — these pages are only accessed + # locally (P2P writes + local reads). Non-fabric handles do NOT + # consume NVLink fabric routing table entries, which are limited to + # ~928 per GPU on GB200 and must be reserved for MNNVL expert data. + # + # Each handle is page_size bytes (e.g. 16MB), NOT granularity bytes. + # This is the key change that reduces cuMemMap calls by 8x. + self._page_handles: List[List[int]] = [] + self._released = False + + for slot_idx, num_pages in enumerate(self._slot_pages): + handles = [] + for page_idx in range(num_pages): + try: + handle = create_local_handle(self._page_size, device_id) + handles.append(handle) + except Exception as e: + # Cleanup on failure + for h in handles: + release_handle(h) + for prev_handles in self._page_handles: + for h in prev_handles: + release_handle(h) + raise RuntimeError( + f"Failed to allocate page {page_idx} in slot {slot_idx}: {e}" + ) + self._page_handles.append(handles) + logger.debug( + f"[PagePool] Allocated {num_pages} pages ({self._page_size} B each) " + f"for slot {slot_idx} ({self._slot_sizes[slot_idx]} bytes)" + ) + + @classmethod + def create( + cls, + slot_sizes: List[int], + device_id: int, + page_size: int | None = None, + ) -> PagePool: + """Factory method to create a page pool. + + Args: + slot_sizes: [max_remote_bytes_slot0, max_remote_bytes_slot1] + Computed as max over all layers assigned to each slot. + device_id: CUDA device ordinal. + page_size: Pool page handle size in bytes. If None, uses default + (8 * granularity = 16MB on GB200). + + Returns: + Initialized PagePool instance. + """ + return cls(slot_sizes, device_id, page_size=page_size) + + @property + def device_id(self) -> int: + """Get CUDA device ordinal.""" + return self._device_id + + @property + def granularity(self) -> int: + """Get VMM page granularity in bytes (typically 2MB).""" + return self._granularity + + @property + def page_size(self) -> int: + """Get pool page handle size in bytes (typically 16MB).""" + return self._page_size + + @property + def num_slots(self) -> int: + """Get number of double buffer slots.""" + return len(self._page_handles) + + def num_pages(self, slot: int) -> int: + """Get number of pages in a slot. + + Args: + slot: Double buffer slot (0 or 1). + + Returns: + Number of pages in the slot. + """ + return self._slot_pages[slot] + + def slot_size(self, slot: int) -> int: + """Get the size of a slot in bytes. + + Args: + slot: Double buffer slot (0 or 1). + + Returns: + Size of the slot in bytes. + """ + return self._slot_sizes[slot] + + def get_page_handle(self, slot: int, page_idx: int) -> int: + """Get handle for a specific page. + + Args: + slot: Double buffer slot (0 or 1). + page_idx: Page index within the slot. + + Returns: + Page handle as integer. + + Raises: + IndexError: If slot or page_idx is out of range. + RuntimeError: If pool has been released. + """ + if self._released: + raise RuntimeError("PagePool has been released") + if slot < 0 or slot >= len(self._page_handles): + raise IndexError(f"Invalid slot {slot}, must be in [0, {len(self._page_handles)})") + if page_idx < 0 or page_idx >= len(self._page_handles[slot]): + raise IndexError( + f"Invalid page_idx {page_idx} for slot {slot}, " + f"must be in [0, {len(self._page_handles[slot])})" + ) + return self._page_handles[slot][page_idx] + + def map_pages( + self, + slot: int, + va_start: int, + size: int, + page_offset: int = 0, + ) -> List[Tuple[int, int]]: + """Map pages from pool[slot] starting at page_offset into VA. + + Maps ceil(size / page_size) pool page handles sequentially: + cuMemMap(va_start + i*page_size, page_size, 0, + pages[slot][page_offset + i], 0) + + Each cuMemMap call maps exactly ``page_size`` bytes (the full pool + handle). Using 16MB pool pages instead of 2MB VMM pages reduces the + number of cuMemMap calls by 8x. + + Args: + slot: Double buffer slot (0 or 1). + va_start: Page-aligned VA position to start mapping. + size: Total bytes to map (must be a multiple of page_size; + callers should use pool_granularity-aligned pre/post sizes). + page_offset: First page index within the pool slot. + + Returns: + List of (va, size) tuples for each mapping made. + + Raises: + ValueError: If mapping would exceed available pages. + RuntimeError: If pool has been released. + """ + if self._released: + raise RuntimeError("PagePool has been released") + + aligned_size = align_up(size, self._page_size) + num_pages_needed = aligned_size // self._page_size + + if page_offset + num_pages_needed > len(self._page_handles[slot]): + raise ValueError( + f"Mapping {num_pages_needed} pages from offset {page_offset} " + f"exceeds slot {slot} capacity of {len(self._page_handles[slot])} pages" + ) + + mappings = [] + for i in range(num_pages_needed): + va = va_start + i * self._page_size + handle = self._page_handles[slot][page_offset + i] + map_handle(va, self._page_size, handle, offset=0) + mappings.append((va, self._page_size)) + + # NOTE: set_access is intentionally NOT called here. + # The caller (WeightBuffer._setup_layer) issues a single + # cuMemSetAccess for the entire composite VA (pre + mnnvl + post) + # after all sub-regions are mapped. This reduces the total number + # of cuMemSetAccess calls from 3 per weight per layer to 1, which + # is necessary to stay within the CUDA VMM NVLink fabric routing + # table limit on GB200 MNNVL nodes. + + return mappings + + def unmap_pages(self, mappings: List[Tuple[int, int]]) -> None: + """Unmap previously mapped pages. + + Args: + mappings: List of (va, size) tuples from map_pages(). + """ + for va, size in mappings: + try: + unmap_va(va, size) + except Exception as e: + logger.warning(f"[PagePool] Failed to unmap VA {va:#x}: {e}") + + def release(self) -> None: + """Release all page handles. Idempotent. Safe from __del__.""" + if self._released: + return + + self._released = True + + for slot_idx, handles in enumerate(self._page_handles): + for page_idx, handle in enumerate(handles): + try: + release_handle(handle) + except Exception as e: + logger.warning( + f"[PagePool] Failed to release page {page_idx} in slot {slot_idx}: {e}" + ) + + self._page_handles = [[], []] + logger.debug("[PagePool] Released all page handles") + + def __del__(self): + """Clean up on destruction (best-effort; errors logged to debug).""" + try: + self.release() + except Exception as e: + logger.debug(f"[PagePool] __del__ release failed (ignored): {e!r}") + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.release() + return False + + +def compute_slot_sizes( + layouts: dict[int, dict[str, "PageAlignedLayout"]], # noqa: F821 + buffer_slot_assignments: dict[int, int], +) -> List[int]: + """Compute the required slot sizes for a set of layouts. + + For each slot, computes the maximum remote buffer size across all + layers assigned to that slot. + + Args: + layouts: Mapping from layer_idx to {weight_name: PageAlignedLayout}. + buffer_slot_assignments: Mapping from layer_idx to slot (0 or 1). + + Returns: + [slot0_size, slot1_size] in bytes. + """ + + slot_sizes = [0, 0] + + for layer_idx, weight_layouts in layouts.items(): + slot = buffer_slot_assignments.get(layer_idx, layer_idx % 2) + + # Sum remote sizes across all weight names within a layer. + # Each weight name needs its own pool pages (they hold different data + # and are prefetched independently by the WeightManager). + layer_remote_total = 0 + for layout in weight_layouts.values(): + layer_remote_total += layout.pre_size + layout.post_size + + slot_sizes[slot] = max(slot_sizes[slot], layer_remote_total) + + return slot_sizes diff --git a/tensorrt_llm/_torch/modules/dwdp/setup.py b/tensorrt_llm/_torch/modules/dwdp/setup.py new file mode 100644 index 000000000000..db857bbee818 --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/setup.py @@ -0,0 +1,1187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DWDP setup orchestration: Transport -> WeightBuffer -> WeightManager -> backend patching. + +This module provides ``setup_dwdp()``, the single entry point that runs ONCE +after model weight loading and BEFORE any inference forward pass. It: + +1. Collects MoE expert weight parameters from the model layers specified by + ``layer_indices`` (the Single Source of Truth coming from + ``DwdpManager._registered_layers``). +2. Builds per-layer WeightSpecs from the collected parameter shapes. +3. Computes each rank's local expert range. +4. Allocates MNNVL fabric handles via ``DWDPTransport.create()``, copying + local weights into fabric memory and freeing the originals. Handles + are exchanged via the DWDP MPI communicator. +5. Constructs a WeightBuffer with composite VA layout (zero-copy local + + page-pool-backed remote double buffer). +6. Fills page-alignment edge bytes at MNNVL region boundaries. +7. Creates a DWDPWeightManager for runtime P2P prefetch scheduling. +8. Patches each MoE backend so it sees all ``num_experts`` (ep_size=1) and + allgathers small EP-sharded parameters (e.g. e_score_correction_bias, + fc31_alpha, fc2_alpha) via MPI. Large scale params + (w3_w1_weight_scale, w2_weight_scale) go through Transport (MNNVL + P2P). + +The function is idempotent: calling it a second time on the same model is a +no-op (returns the existing manager stored on the model). +""" + +from __future__ import annotations + +from typing import Dict, List, Optional, Tuple + +import torch +from torch import nn + +from tensorrt_llm.logger import logger + +from .specs import LayerWeightSpecs, PeerRanges, WeightSpec, lookup_owner +from .transport import DWDPTransport +from .weight_buffer import WeightBuffer +from .weight_manager import DWDPWeightManager + +# Sentinel attribute to mark idempotent setup completion. +# Must match the attribute name read in DeepseekV3Model.forward(). +_DWDP_SETUP_DONE = "dwdp_weight_manager" + +# Weight parameter names handled by the full DWDP pipeline: +# Transport (MNNVL handles) → WeightBuffer (composite VA) → WeightManager (P2P prefetch) +# Includes the main expert weights AND their large EP-sharded scale params. +_EXPERT_WEIGHT_NAMES = ( + "w3_w1_weight", + "w2_weight", + "w3_w1_weight_scale", + "w2_weight_scale", +) + + +def setup_dwdp( + model: nn.Module, + mapping, + device_id: int, + comm, + layer_indices: List[int], + num_experts_per_worker: int, + num_prefetch_experts: int, + contention_opt: bool = False, +) -> Optional[DWDPWeightManager]: + """Set up DWDP for a model after weight loading. + + This is the single orchestration entry point. It must be called exactly + once, after all model weights are loaded and before any forward pass. + + Args: + model: The top-level causal-LM model (e.g. DeepseekV3ForCausalLM). + mapping: ``tensorrt_llm.mapping.Mapping`` instance with DWDP fields. + device_id: CUDA device ordinal for this rank. + comm: mpi4py communicator scoped to the DWDP group (owned by + DwdpManager; lifetime exceeds setup_dwdp). + layer_indices: MoE layer indices collected by + ``DwdpManager._registered_layers`` (SSOT). Each index must + correspond to an MoE layer in the model. + num_experts_per_worker: Range size — number of experts each rank + owns (``DwdpConfig.num_experts_per_worker``). Used as the local + expert range length: ``[start, start + num_experts_per_worker)``. + This is also the *uniform storage chunk size* installed on every + fused MoE backend by ``_init_dwdp_expert_layout``; the tail + rank may have a few padding slots past ``num_experts``. + num_prefetch_experts: Stride between consecutive ranks + (``DwdpConfig.num_prefetch_experts``). Used as the rank-to-rank + offset: rank ``r`` owns ``[r * num_prefetch_experts, ...)``. + For uniform integer-division partitioning these two values are + equal (e.g. dwdp=4: 64/64, dwdp=8: 32/32). Setting + ``num_prefetch_experts < num_experts_per_worker`` enables + redundancy: adjacent ranks' ranges overlap by + ``num_experts_per_worker - num_prefetch_experts`` experts and + ``lookup_owner`` resolves the lowest-rank owner. + + Returns: + A ready-to-use DWDPWeightManager if DWDP is enabled, ``None`` otherwise. + The manager is also stored on ``model._dwdp_weight_manager`` so that + a second call returns the cached instance (idempotent). + """ + # --- Guard: DWDP not enabled --- + if not mapping.dwdp_enabled: + return None + + # --- Guard: already set up? --- + # Check the inner decoder model (where DeepseekV3Model.forward() reads + # self.dwdp_weight_manager). + decoder_model = _get_decoder_model(model) + if hasattr(decoder_model, _DWDP_SETUP_DONE) and getattr(decoder_model, _DWDP_SETUP_DONE) is not None: + logger.info("[DWDP Setup] Already initialized; returning cached manager.") + return getattr(decoder_model, _DWDP_SETUP_DONE) + + # Log GPU memory at the start of DWDP setup + free_mem, total_mem = torch.cuda.mem_get_info(device_id) + logger.info( + f"[DWDP Setup] GPU {device_id} memory before setup: " + f"free={free_mem/(1024**3):.2f}GB / total={total_mem/(1024**3):.2f}GB " + f"(used={((total_mem-free_mem)/(1024**3)):.2f}GB)" + ) + logger.info( + f"[DWDP Setup] Starting DWDP setup: dwdp_rank={mapping.dwdp_rank}, " + f"dwdp_size={mapping.dwdp_size}, device_id={device_id}, " + f"layer_indices={layer_indices}" + ) + + if not layer_indices: + raise ValueError( + "[DWDP Setup] layer_indices is empty. DwdpManager must register " + "MoE layers via add_layer() before setup_dwdp() is called." + ) + + # 1. Collect MoE params (SSOT: iterate passed layer_indices only) + local_params, weight_names = collect_moe_params(model, layer_indices) + logger.info( + f"[DWDP Setup] Collected params from {len(layer_indices)} MoE layers, " + f"weight_names={weight_names}" + ) + + # Query the model's *actual* gate-side num_experts from the first MoE + # layer. This is the routing-side expert count (e.g., 72 for DSv3-Lite, + # 256 for DeepSeek-V3 / R1). It is distinct from the per-rank storage + # extent (``num_experts_per_worker * dwdp_size``) which may be larger + # under non-uniform / redundancy Mode B partitioning. All Phase 1+2 + # plumbing — composite VA size, peer_ranges cap, validation — uses + # this gate-side count, not storage extent. + num_experts_total = _get_model_num_experts(model, layer_indices) + + # 2. Build weight specs (full_shape uses the gate-side num_experts) + layer_weight_specs = build_weight_specs( + local_params, weight_names, mapping, num_experts_total + ) + logger.info( + f"[DWDP Setup] Built weight specs for {len(layer_weight_specs)} layers, " + f"num_experts={num_experts_total}" + ) + + # 3. Validate partition config against the gate-side expert count. + first_spec = _get_first_spec(layer_weight_specs) + _validate_partition_config( + num_experts_per_worker=num_experts_per_worker, + num_prefetch_experts=num_prefetch_experts, + num_experts_total=num_experts_total, + dwdp_size=mapping.dwdp_size, + loaded_local_experts=first_spec.local_experts, + ) + local_start = mapping.dwdp_rank * num_prefetch_experts + local_end = local_start + num_experts_per_worker + logger.info( + f"[DWDP Setup] Expert range: local=[{local_start}, {local_end}), " + f"total={num_experts_total}, " + f"size={num_experts_per_worker}, stride={num_prefetch_experts}" + ) + + # 4. Transport: MNNVL alloc -> copy -> free originals -> exchange handles + logger.info("[DWDP Setup] Creating transport (MNNVL handle exchange)...") + transport = DWDPTransport.create( + layer_weight_specs=layer_weight_specs, + local_params=local_params, + comm=comm, + dwdp_rank=mapping.dwdp_rank, + dwdp_size=mapping.dwdp_size, + device_id=device_id, + local_start=local_start, + local_end=local_end, + num_experts_per_worker=num_experts_per_worker, + num_prefetch_experts=num_prefetch_experts, + ) + logger.info("[DWDP Setup] Transport created successfully.") + free_mem, total_mem = torch.cuda.mem_get_info(device_id) + logger.info( + f"[DWDP Setup] GPU {device_id} memory after transport: " + f"free={free_mem/(1024**3):.2f}GB / total={total_mem/(1024**3):.2f}GB " + f"(used={((total_mem-free_mem)/(1024**3)):.2f}GB)" + ) + + # 5. WeightBuffer: composite VA layout + logger.info("[DWDP Setup] Creating weight buffer (composite VA layout)...") + weight_buffer = WeightBuffer.create( + layer_weight_specs=layer_weight_specs, + handles=transport.get_handle_set(), + local_start=local_start, + local_end=local_end, + dwdp_size=mapping.dwdp_size, + device_id=device_id, + ) + logger.info("[DWDP Setup] Weight buffer created successfully.") + + # 6. Fill edge bytes + logger.info("[DWDP Setup] Filling page-alignment edge bytes...") + peer_ranges = transport.get_peer_ranges() + fill_edge_bytes( + weight_buffer=weight_buffer, + peer_views=transport.get_peer_views(), + local_start=local_start, + local_end=local_end, + peer_ranges=peer_ranges, + ) + logger.info("[DWDP Setup] Edge bytes filled.") + + # 7. WeightManager + logger.info("[DWDP Setup] Creating weight manager...") + weight_manager = DWDPWeightManager( + weight_buffer=weight_buffer, + peer_views=transport.get_peer_views(), + peer_ranges=peer_ranges, + moe_layer_indices=layer_indices, + weight_names=list(weight_names), + dwdp_rank=mapping.dwdp_rank, + dwdp_size=mapping.dwdp_size, + contention_opt=contention_opt, + ) + logger.info("[DWDP Setup] Weight manager created.") + + # 8. Patch MoE backends + logger.info("[DWDP Setup] Patching MoE backends...") + fixup_moe_backends( + model, + layer_indices, + num_experts_total, + comm, + mapping.dwdp_rank, + mapping.dwdp_size, + num_experts_per_worker=num_experts_per_worker, + peer_ranges=peer_ranges, + ) + logger.info("[DWDP Setup] MoE backends patched.") + + # 9. Keep transport alive (its handles underpin the VA mappings) + weight_manager._transport = transport + + # Store on both the top-level model (for idempotent retrieval) and the + # inner decoder model (where DeepseekV3Model.forward() reads it). + setattr(model, _DWDP_SETUP_DONE, weight_manager) + decoder_model.dwdp_weight_manager = weight_manager + + logger.info("[DWDP Setup] Setup complete.") + return weight_manager + + +# --------------------------------------------------------------------------- +# Helper: collect MoE params (SSOT: layer_indices is input, not discovered) +# --------------------------------------------------------------------------- + + +def collect_moe_params( + model: nn.Module, + layer_indices: List[int], +) -> Tuple[Dict[Tuple[int, str], torch.Tensor], List[str]]: + """Extract MoE expert weight/scale tensors from the specified layers. + + Iterates the passed ``layer_indices`` (the authoritative list from + ``DwdpManager._registered_layers``) and pulls expert parameters off + each layer's MoE module. Layer discovery is NOT done here — that is + the DwdpManager's job via ``add_layer()``. + + Args: + model: Top-level CausalLM model (e.g. DeepseekV3ForCausalLM). + layer_indices: Sorted list of MoE decoder layer indices, registered + by ConfigurableMoE.__init__() → DwdpManager.add_layer(). + + Returns: + Tuple of: + - local_params: Dict mapping ``(layer_idx, weight_name)`` to the + parameter tensor (still on device, will be consumed by Transport). + - weight_names: List of weight parameter names collected (subset of + ``_EXPERT_WEIGHT_NAMES`` that actually exist on the backend). + + Raises: + RuntimeError: If any layer in ``layer_indices`` does not have an + MoE experts module with the expected weight parameters. + """ + local_params: Dict[Tuple[int, str], torch.Tensor] = {} + weight_names_set: List[str] = [] + + decoder_model = _get_decoder_model(model) + layers = decoder_model.layers + + for layer_idx in layer_indices: + layer = layers[layer_idx] + _moe_module, experts_module = _get_moe_and_experts(layer) + if experts_module is None: + raise RuntimeError( + f"[DWDP Setup] Layer {layer_idx} is registered as an MoE layer " + f"but no experts module was found. Check DwdpManager.add_layer() " + f"call sites." + ) + + found_names = [] + for wname in _EXPERT_WEIGHT_NAMES: + param = getattr(experts_module, wname, None) + if param is not None and isinstance(param, (torch.Tensor, nn.Parameter)): + found_names.append(wname) + local_params[(layer_idx, wname)] = ( + param.data if isinstance(param, nn.Parameter) else param + ) + + if not found_names: + raise RuntimeError( + f"[DWDP Setup] Layer {layer_idx} MoE experts module has none of " + f"the expected weight parameters {_EXPERT_WEIGHT_NAMES}." + ) + + if not weight_names_set: + weight_names_set = found_names + logger.debug( + f"[DWDP Setup] Layer {layer_idx}: collected {found_names}, " + f"shapes={[local_params[(layer_idx, n)].shape for n in found_names]}" + ) + + return local_params, weight_names_set + + +# --------------------------------------------------------------------------- +# Helper: build weight specs +# --------------------------------------------------------------------------- + + +def build_weight_specs( + local_params: Dict[Tuple[int, str], torch.Tensor], + weight_names: List[str], + mapping, + num_experts_total: int, +) -> LayerWeightSpecs: + """Convert collected parameters to ``LayerWeightSpecs``. + + Each local parameter has shape ``(experts_per_rank, ...)`` where + ``experts_per_rank`` is the per-rank storage size + (``num_experts_per_worker`` from ``DwdpConfig`` after + ``_init_dwdp_expert_layout`` runs). The composite VA's ``full_shape`` + uses ``num_experts_total`` (gate-side, e.g. 72 for DSv3-Lite) so + every expert id in ``[0, num_experts_total)`` is addressable — and + *only* those. Under Mode B redundancy ``experts_per_rank * + dwdp_size`` may exceed ``num_experts_total`` (overlap duplicates), + but those duplicates live within each peer's MNNVL handle, not in + the composite VA. + + Args: + local_params: Dict from ``collect_moe_params``. + weight_names: Weight names list. + mapping: Mapping with dwdp_size. + num_experts_total: Gate-side expert count read from the model + (``MoE.num_experts``), used as ``WeightSpec.num_experts`` / + ``full_shape[0]``. + + Returns: + LayerWeightSpecs: ``Dict[layer_idx, Dict[weight_name, WeightSpec]]``. + """ + layer_weight_specs: LayerWeightSpecs = {} + + # Group params by layer + layers_seen: Dict[int, List[str]] = {} + for (layer_idx, wname) in local_params: + if layer_idx not in layers_seen: + layers_seen[layer_idx] = [] + layers_seen[layer_idx].append(wname) + + for layer_idx in sorted(layers_seen.keys()): + specs: Dict[str, WeightSpec] = {} + for wname in weight_names: + key = (layer_idx, wname) + if key not in local_params: + continue + param = local_params[key] + chunk_shape = tuple(param.shape) + full_shape = (num_experts_total,) + chunk_shape[1:] + specs[wname] = WeightSpec( + num_experts=num_experts_total, + chunk_shape=chunk_shape, + full_shape=full_shape, + dtype=param.dtype, + ) + layer_weight_specs[layer_idx] = specs + + return layer_weight_specs + + +def _get_model_num_experts(model: nn.Module, layer_indices: List[int]) -> int: + """Read the gate-side ``num_experts`` from the first MoE layer. + + The fused MoE backend stores the routing-side expert count + (``MoE.num_experts`` from the model config, e.g. + ``n_routed_experts``) on each ``experts_module``. All MoE layers in + a given model share the same value, so reading the first one is + sufficient. + + Args: + model: Top-level CausalLM model. + layer_indices: Non-empty list of MoE layer indices. + + Returns: + Gate-side expert count (positive integer). + + Raises: + RuntimeError: If the first MoE layer's experts module cannot be + found or does not expose ``num_experts``. + """ + decoder_model = _get_decoder_model(model) + layers = decoder_model.layers + first_layer_idx = sorted(layer_indices)[0] + _, experts_module = _get_moe_and_experts(layers[first_layer_idx]) + if experts_module is None: + raise RuntimeError( + f"[DWDP Setup] Cannot read num_experts: layer {first_layer_idx} " + f"has no experts module." + ) + n = getattr(experts_module, "num_experts", None) + if not isinstance(n, int) or n <= 0: + raise RuntimeError( + f"[DWDP Setup] Layer {first_layer_idx} experts module has " + f"invalid num_experts={n!r}." + ) + return n + + +# --------------------------------------------------------------------------- +# Helper: fill edge bytes +# --------------------------------------------------------------------------- + + +def fill_edge_bytes( + weight_buffer: WeightBuffer, + peer_views: Dict[Tuple[int, int, str], torch.Tensor], + local_start: int, + local_end: int, + peer_ranges: PeerRanges, +) -> None: + """Fill page-alignment edge bytes at MNNVL region boundaries. + + When local expert data does not align to page boundaries, the MNNVL pages + contain edge regions that must be filled with the correct peer data. This + happens at most twice per (layer, weight): a *leading edge* before + ``local_start`` and a *trailing edge* after ``local_end``. + + The data is read from peer MNNVL tensor views and written into the local + MNNVL handle's mapped VA. This is a one-time setup operation. + + Args: + weight_buffer: WeightBuffer with composite VA layout. + peer_views: ``{(peer_rank, layer_idx, name): tensor}`` from Transport. + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + peer_ranges: Per-peer ``(local_start, local_end_capped)`` tuples. + ``lookup_owner`` resolves the owner of ``prev_expert`` / + ``next_expert`` here, so non-uniform / redundancy partitions + pick the right peer. + """ + if local_start >= local_end: + return + + for layer_idx in weight_buffer.layer_indices: + for name in weight_buffer.weight_names(layer_idx): + edge_info = weight_buffer.get_edge_info(layer_idx, name) + + if edge_info.leading_edge == 0 and edge_info.trailing_edge == 0: + logger.debug( + f"[DWDP Setup] Layer {layer_idx}/{name}: no edge bytes to fill" + ) + continue + + # Get the full tensor view (composite VA) + full_tensor = weight_buffer.get_full_tensor(layer_idx, name) + + # Leading edge: bytes in [page_start, local_start_bytes). + # These belong to experts just before local_start. + if edge_info.leading_edge > 0 and local_start > 0: + # The leading edge is within the expert just before + # local_start. Use lookup_owner so non-uniform / redundant + # partitions pick the lowest-rank peer that owns + # ``prev_expert``. + prev_expert = local_start - 1 + peer_rank = lookup_owner(prev_expert, peer_ranges) + peer_chunk_start, _ = peer_ranges[peer_rank] + peer_local_offset = prev_expert - peer_chunk_start + + peer_key = (peer_rank, layer_idx, name) + if peer_key in peer_views: + src = peer_views[peer_key] + # The edge bytes correspond to the tail portion of + # prev_expert's data that spills into our MNNVL page. + # Copy the full expert slice (the composite VA's + # expert dimension is page-aligned). + full_tensor[prev_expert].copy_(src[peer_local_offset]) + logger.debug( + f"[DWDP Setup] Layer {layer_idx}/{name}: filled leading edge " + f"from peer {peer_rank}, expert {prev_expert}" + ) + + # Trailing edge: bytes in [local_end_bytes, page_end). + # These belong to experts just after local_end. + if edge_info.trailing_edge > 0 and local_end < full_tensor.shape[0]: + next_expert = local_end + peer_rank = lookup_owner(next_expert, peer_ranges) + peer_chunk_start, _ = peer_ranges[peer_rank] + peer_local_offset = next_expert - peer_chunk_start + + peer_key = (peer_rank, layer_idx, name) + if peer_key in peer_views: + src = peer_views[peer_key] + full_tensor[next_expert].copy_(src[peer_local_offset]) + logger.debug( + f"[DWDP Setup] Layer {layer_idx}/{name}: filled trailing edge " + f"from peer {peer_rank}, expert {next_expert}" + ) + + torch.cuda.synchronize(weight_buffer.device_id) + + +# --------------------------------------------------------------------------- +# Helper: fixup MoE backends +# --------------------------------------------------------------------------- + + +def fixup_moe_backends( + model: nn.Module, + moe_layer_indices: List[int], + num_experts_total: int, + comm, + dwdp_rank: int, + dwdp_size: int, + *, + num_experts_per_worker: int, + peer_ranges: PeerRanges, +) -> None: + """Patch each MoE backend to see all experts (ep_size=1) after DWDP setup. + + After DWDP setup, each rank conceptually owns *all* experts (the weight + buffer has a full [num_experts, ...] tensor via composite VA). The MoE + backend must therefore be reconfigured: + + - ``ep_size = 1``, ``ep_rank = 0``: no more EP sharding from the backend's + perspective; DWDP handles the distribution. + - ``num_experts``: unchanged (already the global count). + - ``expert_size_per_partition = num_experts``: all experts are "local". + - ``slot_start = 0``, ``slot_end = num_experts``. + - ``initial_local_expert_ids = list(range(num_experts))``. + - ``initial_global_assignments = list(range(num_experts))``. + - ``num_slots = num_experts``. + + Additionally, small EP-sharded scale parameters + (``fc31_alpha`` / ``fc2_alpha``) are allgathered to full size. The + gate's ``e_score_correction_bias`` is checked too, but on the + Phase-1 ``moe_ep_size = 1`` setup it is already loaded full and the + helper short-circuits. + + Args: + model: Top-level CausalLM model. + moe_layer_indices: MoE layer indices to patch. + num_experts_total: Total number of experts across all ranks. + comm: DWDP MPI communicator used for small tensor allgathers. + dwdp_rank: This rank's DWDP rank. + dwdp_size: Total DWDP ranks. + num_experts_per_worker: Storage chunk size — uniform across ranks. + Used as the EP shard size for scale-param detection (the scale + params loaded by the fused MoE backend match + ``self.expert_size_per_partition`` set by + ``_init_dwdp_expert_layout``). + peer_ranges: Per-peer ``(local_start, local_end_capped)`` tuples; + used to scatter each peer's allgathered shard back into the + correct slice of the reconstructed full-size tensor (handles + tail-rank padding and redundant overlap). + """ + decoder_model = _get_decoder_model(model) + layers = decoder_model.layers + + for layer_idx in moe_layer_indices: + layer = layers[layer_idx] + moe_module, experts_module = _get_moe_and_experts(layer) + if experts_module is None: + logger.warning( + f"[DWDP Setup] Layer {layer_idx}: could not find experts module " + f"for backend patching (skipped)" + ) + continue + + # --- Patch expert-parallel attributes on both the ConfigurableMoE --- + # --- wrapper (if present) AND the backend. --- + # ConfigurableMoE has its own ep_size, slot_start, etc. that are used + # in its forward path. The backend is the inner module that holds + # weight parameters. + configurable_moe = getattr(layer.mlp, "experts", None) + targets = [experts_module] + if configurable_moe is not None and configurable_moe is not experts_module: + targets.insert(0, configurable_moe) + + old_ep_size = getattr(experts_module, "ep_size", None) + old_ep_rank = getattr(experts_module, "ep_rank", None) + old_slot_start = getattr(experts_module, "slot_start", None) + old_slot_end = getattr(experts_module, "slot_end", None) + + for target in targets: + target.ep_size = 1 + target.ep_rank = 0 + target.expert_size_per_partition = num_experts_total + target.slot_start = 0 + target.slot_end = num_experts_total + target.num_slots = num_experts_total + target.initial_local_expert_ids = list(range(num_experts_total)) + target.initial_global_assignments = list(range(num_experts_total)) + + logger.debug( + f"[DWDP Setup] Layer {layer_idx}: patched " + f"{'ConfigurableMoE + ' if len(targets) > 1 else ''}backend " + f"ep_size={old_ep_size}->{1}, ep_rank={old_ep_rank}->{0}, " + f"slots=[{old_slot_start},{old_slot_end})->[0,{num_experts_total})" + ) + + # --- Allgather e_score_correction_bias --- + gate_module = _get_gate_module(moe_module) + if gate_module is not None: + _allgather_e_score_correction_bias( + gate_module, layer_idx, dwdp_rank, dwdp_size, + num_experts_total, comm, + num_experts_per_worker=num_experts_per_worker, + peer_ranges=peer_ranges, + ) + + # --- Allgather small expert scale/dequant parameters via MPI --- + # Large scale params (w3_w1_weight_scale, w2_weight_scale) are already + # handled by Transport (MNNVL + P2P) — they are in _EXPERT_WEIGHT_NAMES. + # Small params (fc31_alpha, fc2_alpha) are allgathered here. + # ``num_experts_per_worker`` (uniform storage chunk size across ranks) + # is the right shape match for these EP-sharded scale params under the + # Phase-1 layout, even when the partition is non-uniform or has + # redundancy. + _allgather_expert_scales( + experts_module, layer_idx, dwdp_rank, dwdp_size, comm, + experts_per_rank=num_experts_per_worker, + num_experts_total=num_experts_total, + peer_ranges=peer_ranges, + ) + _rebuild_quant_scales(experts_module, layer_idx) + + +def _allgather_e_score_correction_bias( + gate_module: nn.Module, + layer_idx: int, + dwdp_rank: int, + dwdp_size: int, + num_experts_total: int, + comm, + *, + num_experts_per_worker: Optional[int] = None, + peer_ranges: Optional[PeerRanges] = None, +) -> None: + """Allgather the EP-sharded e_score_correction_bias to get the full vector. + + Each rank has a shard of size ``num_experts_per_worker`` (the uniform + storage chunk set by ``_init_dwdp_expert_layout``). After allgather + each rank holds the full ``(num_experts,)`` bias vector reconstructed + via ``peer_ranges`` (so non-uniform tail padding and redundant + overlapping ranges are placed at the correct global expert indices). + + If the bias is already full-sized (not EP-sharded), this is a no-op. + The gate's e_score_correction_bias covers ALL experts for routing and + is typically loaded full when ``moe_ep_size = 1``. + + Args: + gate_module: The DeepseekV3Gate (or compatible) module with + ``e_score_correction_bias`` parameter. + layer_idx: Layer index (for log messages only). + dwdp_rank: This rank's DWDP rank. + dwdp_size: Total DWDP ranks. + num_experts_total: Total number of experts across all ranks. + comm: DWDP MPI communicator. + num_experts_per_worker: Uniform storage shard size used to detect + EP-sharded bias. Defaults to ``num_experts_total // dwdp_size`` + (the uniform-partition shard size). + peer_ranges: ``(local_start, local_end_capped)`` per peer; used + to scatter shards back to the right global expert indices. + When ``None`` the function falls back to ``torch.cat`` and + truncates to ``num_experts_total`` (uniform / non-divisible + tail-padding case). Required for redundancy. + """ + bias_param = getattr(gate_module, "e_score_correction_bias", None) + if bias_param is None: + logger.debug( + f"[DWDP Setup] Layer {layer_idx}: no e_score_correction_bias found" + ) + return + + bias_data = bias_param.data if isinstance(bias_param, nn.Parameter) else bias_param + bias_size = bias_data.shape[0] + expected_shard_size = ( + num_experts_per_worker if num_experts_per_worker is not None + else num_experts_total // dwdp_size + ) + + logger.info( + f"[DWDP Setup] Layer {layer_idx}: e_score_correction_bias " + f"type={type(bias_param).__name__}, shape={bias_data.shape}, " + f"device={bias_data.device}, " + f"num_experts_total={num_experts_total}, " + f"expected_shard={expected_shard_size}" + ) + + # The gate's e_score_correction_bias covers ALL experts for routing. + # It may or may not be EP-sharded depending on the model. + # Only allgather if the bias is actually sharded (size matches shard). + if bias_size == num_experts_total: + logger.info( + f"[DWDP Setup] Layer {layer_idx}: e_score_correction_bias " + f"already full-sized ({bias_size}), skipping allgather" + ) + return + if bias_size != expected_shard_size: + logger.warning( + f"[DWDP Setup] Layer {layer_idx}: e_score_correction_bias " + f"size={bias_size} doesn't match expected shard size " + f"({expected_shard_size}) or full size ({num_experts_total}), " + f"skipping allgather" + ) + return + + # Allgather local shards via MPI. Move to CPU first so the allgather + # doesn't require CUDA-aware MPI. + local_shard = bias_data.cpu().contiguous() + all_shards = comm.allgather(local_shard) + if peer_ranges is not None: + full_bias = _scatter_shards_to_full( + shards=all_shards, + peer_ranges=peer_ranges, + num_experts_total=num_experts_total, + ref=bias_data, + ) + else: + # Legacy uniform path: cat then truncate to handle non-divisible + # tail padding. Sufficient for the no-redundancy uniform case. + full_bias = torch.cat(all_shards, dim=0).to(bias_data.device) + if full_bias.shape[0] > num_experts_total: + full_bias = full_bias[:num_experts_total].contiguous() + + if isinstance(bias_param, nn.Parameter): + # Re-create parameter with the full size + gate_module.e_score_correction_bias = nn.Parameter( + full_bias, requires_grad=False + ) + else: + bias_param.data = full_bias + + logger.info( + f"[DWDP Setup] Layer {layer_idx}: allgathered e_score_correction_bias " + f"({expected_shard_size} -> {full_bias.shape[0]})" + ) + + +def _allgather_expert_scales( + experts_module: nn.Module, + layer_idx: int, + dwdp_rank: int, + dwdp_size: int, + comm, + experts_per_rank: Optional[int] = None, + *, + num_experts_total: Optional[int] = None, + peer_ranges: Optional[PeerRanges] = None, +) -> None: + """Allgather small EP-sharded quantization scale/dequant parameters via MPI. + + Only handles SMALL scale params (e.g. ``fc31_alpha``, ``fc2_alpha``) + that are NOT in ``_EXPERT_WEIGHT_NAMES``. Large scale params + (``w3_w1_weight_scale``, ``w2_weight_scale``) are handled by DWDP + Transport (MNNVL + P2P). + + Heuristic: any ``nn.Parameter`` on ``experts_module`` whose name contains + ``"scale"``, ``"scaling_factor"``, ``"dequant"``, or ``"alpha"``, whose + ``shape[0] == experts_per_rank``, and which is NOT in + ``_EXPERT_WEIGHT_NAMES``, is treated as a small EP-sharded tensor. + + When ``peer_ranges`` is provided each shard is scattered into the + correct global expert slice — this is required for non-uniform / + redundant partitions, where naive ``torch.cat`` would mis-place + overlapping ranges or include tail-rank padding. When ``peer_ranges`` + is ``None`` the function falls back to the simple uniform-partition + concat (legacy behavior). + + Args: + experts_module: The MoE backend module (e.g. CutlassFusedMoE). + layer_idx: Layer index (for log messages only). + dwdp_rank: This rank's DWDP rank. + dwdp_size: Total DWDP ranks. + comm: DWDP MPI communicator. + experts_per_rank: Number of experts per DWDP rank — i.e., the + uniform storage chunk size. If None, inferred from + ``w3_w1_weight.shape[0]`` (only valid before DWDP weight + buffer replaces the original tensor). + num_experts_total: Global expert count. Required when + ``peer_ranges`` is provided (for the destination tensor size). + peer_ranges: Per-peer ``(local_start, local_end_capped)``; when + provided enables scatter-based reconstruction that handles + non-uniform / redundant partitions. + """ + _SCALE_KEYWORDS = ("scale", "scaling_factor", "dequant", "alpha") + + # Discover EP-sharded scale parameters. + if experts_per_rank is None: + # Fallback: infer from main weight (only valid before DWDP buffer swap). + ref_param = getattr(experts_module, "w3_w1_weight", None) + if ref_param is None: + return + ref_data = ref_param.data if isinstance(ref_param, nn.Parameter) else ref_param + experts_per_rank = ref_data.shape[0] + if experts_per_rank == 0: + return + + for pname, param in list(experts_module.named_parameters()): + # Only look at direct parameters (no '.' in name — skip sub-modules). + if "." in pname: + continue + if not any(kw in pname for kw in _SCALE_KEYWORDS): + continue + # Large scale params in _EXPERT_WEIGHT_NAMES are handled by Transport + # (MNNVL + P2P), not MPI allgather. + if pname in _EXPERT_WEIGHT_NAMES: + continue + + pdata = param.data + if pdata.ndim == 0 or pdata.shape[0] != experts_per_rank: + # Scalar or non-EP-sharded — skip. + continue + + local_shard = pdata.cpu().contiguous() + logger.info( + f"[DWDP Setup] Layer {layer_idx}: allgathering {pname} " + f"shape={local_shard.shape} dtype={local_shard.dtype} via MPI" + ) + + all_shards = comm.allgather(local_shard) + if peer_ranges is not None: + assert num_experts_total is not None, ( + "num_experts_total must be provided alongside peer_ranges" + ) + full_tensor = _scatter_shards_to_full( + shards=all_shards, + peer_ranges=peer_ranges, + num_experts_total=num_experts_total, + ref=pdata, + ) + else: + full_tensor = torch.cat(all_shards, dim=0).to(pdata.device) + + # Replace parameter with full-sized version. + setattr( + experts_module, + pname, + nn.Parameter(full_tensor, requires_grad=False), + ) + + logger.info( + f"[DWDP Setup] Layer {layer_idx}: allgathered {pname} " + f"({experts_per_rank} -> {full_tensor.shape[0]})" + ) + + +def _scatter_shards_to_full( + *, + shards: List[torch.Tensor], + peer_ranges: PeerRanges, + num_experts_total: int, + ref: torch.Tensor, +) -> torch.Tensor: + """Reconstruct a full ``(num_experts_total, *trailing)`` tensor from + per-peer shards using ``peer_ranges`` as the index map. + + Each peer's shard has shape ``(num_experts_per_worker, *trailing)`` + (uniform across ranks). The valid prefix + ``shard[:end_capped - start]`` is placed at ``full[start:end_capped]``. + For redundancy (overlapping ranges) writes overwrite — values agree + across peers so the result is deterministic. + + Args: + shards: Per-peer tensors as returned by ``comm.allgather``. + peer_ranges: Per-peer ``(local_start, local_end_capped)``. + num_experts_total: Global expert count (= full tensor's first dim). + ref: Reference tensor for dtype + device. + """ + if not shards: + raise ValueError("shards must be non-empty") + trailing_shape = tuple(shards[0].shape[1:]) + full = torch.zeros( + (num_experts_total, *trailing_shape), + dtype=ref.dtype, + device=ref.device, + ) + for peer_rank, (start, end) in enumerate(peer_ranges): + valid = end - start + if valid <= 0: + continue + full[start:end] = shards[peer_rank][:valid].to(ref.device) + return full + + +# --------------------------------------------------------------------------- +# Internal utilities +# --------------------------------------------------------------------------- + + +def _rebuild_quant_scales(experts_module: nn.Module, layer_idx: int) -> None: + """Rebuild the ``quant_scales`` NamedTuple to reference the current Parameters. + + After setup, the quant_scales NamedTuple may hold stale references: + - w3_w1_weight_scale / w2_weight_scale: handled by DWDP Transport. Their + original storage is freed (resized to 0), but the Parameter objects + still exist. At runtime, wait_and_bind() does param.data = composite_VA, + which swaps in the full tensor. The NamedTuple references the same + Parameter, so it sees the updated data. + - fc31_alpha / fc2_alpha: allgathered via MPI. _allgather_expert_scales + replaces these with new full-sized Parameter objects via setattr(), so the + old NamedTuple references are stale and must be rebuilt. + + This function reconstructs the NamedTuple from the module's current + Parameter state so all fields point to the live objects. + + Args: + experts_module: The MoE backend module whose ``quant_scales`` to rebuild. + layer_idx: Layer index (for log messages only). + """ + qs = getattr(experts_module, "quant_scales", None) + if qs is None: + return + + # Detect FusedMoEQuantScalesNVFP4 by its field names. + qs_type = type(qs) + field_names = getattr(qs_type, "_fields", ()) + if "fc1_weight_block" in field_names and "fc2_weight_block" in field_names: + # FusedMoEQuantScalesNVFP4: + # fc1_act_global = module.fc31_input_scale (scalar, not EP-sharded) + # fc1_weight_block = module.w3_w1_weight_scale (Transport: empty now, filled at runtime) + # fc1_global = module.fc31_alpha (MPI allgathered: full-sized) + # fc2_act_global = module.fc2_input_scale (scalar, not EP-sharded) + # fc2_weight_block = module.w2_weight_scale (Transport: empty now, filled at runtime) + # fc2_global = module.fc2_alpha (MPI allgathered: full-sized) + new_qs = qs_type( + fc1_act_global=experts_module.fc31_input_scale, + fc1_weight_block=experts_module.w3_w1_weight_scale, + fc1_global=experts_module.fc31_alpha, + fc2_act_global=experts_module.fc2_input_scale, + fc2_weight_block=experts_module.w2_weight_scale, + fc2_global=experts_module.fc2_alpha, + ) + experts_module.quant_scales = new_qs + logger.info( + f"[DWDP Setup] Layer {layer_idx}: rebuilt FusedMoEQuantScalesNVFP4 " + f"(weight_scale params via Transport, alpha params via MPI allgather)" + ) + else: + logger.debug( + f"[DWDP Setup] Layer {layer_idx}: quant_scales type {qs_type.__name__} " + f"not handled by _rebuild_quant_scales (fields={field_names}); skipping." + ) + + +def _get_decoder_model(model: nn.Module) -> nn.Module: + """Navigate from top-level CausalLM to the decoder model with .layers. + + Handles DeepseekV3ForCausalLM -> .model (DeepseekV3Model). + Falls back to searching common attribute names. + + Args: + model: Top-level model. + + Returns: + The decoder model that has a ``.layers`` ModuleList. + + Raises: + RuntimeError: If the decoder model cannot be found. + """ + # Direct attribute: model.model + if hasattr(model, "model") and hasattr(model.model, "layers"): + return model.model + + # Maybe the model itself has layers + if hasattr(model, "layers"): + return model + + # Search for common patterns + for attr_name in ("transformer", "decoder", "backbone"): + child = getattr(model, attr_name, None) + if child is not None and hasattr(child, "layers"): + return child + + raise RuntimeError( + "[DWDP Setup] Cannot find decoder model with .layers attribute. " + f"Model type: {type(model).__name__}" + ) + + +def _get_moe_and_experts( + layer: nn.Module, +) -> Tuple[Optional[nn.Module], Optional[nn.Module]]: + """From a decoder layer, find the MoE wrapper and its experts backend. + + The standard path for DeepSeek is: + layer.mlp (Deepseekv3MoE) -> .experts (MoE backend) + + Returns: + Tuple of (moe_module, experts_module) where moe_module is the wrapper + (e.g. Deepseekv3MoE) and experts_module is the backend (e.g. + CutlassFusedMoE, ConfigurableMoE, etc.). Both may be None if the + layer is not an MoE layer. + """ + mlp = getattr(layer, "mlp", None) + if mlp is None: + return None, None + + # Check if mlp itself is an MoE backend (has w3_w1_weight) + if hasattr(mlp, "w3_w1_weight"): + return mlp, mlp + + # Standard path: mlp.experts + experts = getattr(mlp, "experts", None) + if experts is not None: + # Prefer the inner backend (ConfigurableMoE wraps it) + backend = getattr(experts, "backend", None) + if backend is not None and hasattr(backend, "w3_w1_weight"): + return mlp, backend + # Fallback: direct backend without ConfigurableMoE wrapper + if hasattr(experts, "w3_w1_weight"): + return mlp, experts + + return None, None + + +def _get_gate_module(moe_module: nn.Module) -> Optional[nn.Module]: + """Get the gate module from an MoE wrapper. + + Args: + moe_module: The MoE wrapper (e.g. Deepseekv3MoE). + + Returns: + Gate module with e_score_correction_bias, or None. + """ + gate = getattr(moe_module, "gate", None) + if gate is not None and hasattr(gate, "e_score_correction_bias"): + return gate + return None + + +def _get_first_spec(layer_weight_specs: LayerWeightSpecs) -> WeightSpec: + """Get the first WeightSpec from layer_weight_specs. + + Args: + layer_weight_specs: The weight specs dict. + + Returns: + First WeightSpec found. + + Raises: + ValueError: If specs are empty. + """ + for layer_idx, weight_specs in layer_weight_specs.items(): + for name, spec in weight_specs.items(): + return spec + raise ValueError("layer_weight_specs is empty") + + +def _validate_partition_config( + *, + num_experts_per_worker: int, + num_prefetch_experts: int, + num_experts_total: int, + dwdp_size: int, + loaded_local_experts: int, +) -> None: + """Validate that the DwdpConfig partition is internally consistent. + + Checks (in order, each independent): + + 1. Both ``num_experts_per_worker`` and ``num_prefetch_experts`` are + positive integers. + 2. ``num_prefetch_experts <= num_experts_per_worker`` — stride must + not exceed range size, otherwise consecutive ranks leave a gap + (some experts owned by no one). Equality is the no-redundancy + case; ``stride < size`` is intentional redundancy where adjacent + ranks' ranges overlap by ``size - stride`` experts. + 3. Strict equality ``(dwdp_size - 1) * stride + size == num_experts``. + The last rank's storage must end exactly at ``num_experts``: + + * ``< num_experts``: insufficient coverage — some experts owned + by no rank. + * ``> num_experts``: tail padding — last rank's storage extends + past the final expert, which the GB200 cuMemMap-with-fabric- + handle ABI cannot map into a ``num_experts``-sized composite + VA (partial mapping returns ``CUDA_ERROR_NOT_SUPPORTED``). + Pick a (size, stride) pair satisfying equality instead, e.g. + dwdp=3 + 256 experts → ``size=86, stride=85``; + dwdp=5 + 256 → ``52, 51``; + dwdp=7 + 256 → ``40, 36`` (or any other ``a*stride + b*size`` + summing to ``num_experts``). + 4. ``loaded_local_experts == num_experts_per_worker`` — the chunk + shape produced by the fused MoE weight loader must match what + DwdpConfig declares, so DWDP does not redirect prefetch to a + tensor of the wrong size. ``_init_dwdp_expert_layout`` arranges + this by overriding ``expert_size_per_partition`` to + ``num_experts_per_worker`` before ``create_weights`` runs. + + Together these guarantee that every expert in + ``[0, num_experts_total)`` is owned by at least one rank, that the + runtime range computation matches the stored weights, and that + every storage slot maps to a valid (in-range) expert id. + """ + if num_experts_per_worker <= 0: + raise ValueError( + f"[DWDP Setup] num_experts_per_worker must be positive, " + f"got {num_experts_per_worker}" + ) + if num_prefetch_experts <= 0: + raise ValueError( + f"[DWDP Setup] num_prefetch_experts must be positive, " + f"got {num_prefetch_experts}" + ) + if num_prefetch_experts > num_experts_per_worker: + raise ValueError( + f"[DWDP Setup] num_prefetch_experts ({num_prefetch_experts}) " + f"must be <= num_experts_per_worker ({num_experts_per_worker}); " + f"a stride larger than the range size leaves expert gaps " + f"between consecutive ranks." + ) + coverage = (dwdp_size - 1) * num_prefetch_experts + num_experts_per_worker + if coverage < num_experts_total: + raise ValueError( + f"[DWDP Setup] DWDP partition does not cover all experts: " + f"(dwdp_size - 1) * num_prefetch_experts + num_experts_per_worker " + f"= ({dwdp_size} - 1) * {num_prefetch_experts} + " + f"{num_experts_per_worker} = {coverage} < num_experts_total " + f"= {num_experts_total}. Increase num_prefetch_experts or " + f"num_experts_per_worker so coverage >= num_experts." + ) + if coverage > num_experts_total: + raise ValueError( + f"[DWDP Setup] DWDP partition exceeds num_experts: " + f"(dwdp_size - 1) * num_prefetch_experts + num_experts_per_worker " + f"= ({dwdp_size} - 1) * {num_prefetch_experts} + " + f"{num_experts_per_worker} = {coverage} > num_experts_total " + f"= {num_experts_total}. The last rank's storage extends " + f"past the final expert; GB200 fabric handles cannot be " + f"partially mapped into the composite VA, so tail padding " + f"is unsupported. Pick a (size, stride) pair with " + f"(dwdp_size - 1) * stride + size == num_experts (Mode B " + f"overlap), e.g. dwdp=3 + 256 -> size=86 stride=85; " + f"dwdp=5 + 256 -> 52 / 51; dwdp=7 + 256 -> 40 / 36." + ) + if loaded_local_experts != num_experts_per_worker: + raise ValueError( + f"[DWDP Setup] DwdpConfig.num_experts_per_worker=" + f"{num_experts_per_worker} does not match the fused MoE chunk " + f"shape ({loaded_local_experts}). _init_dwdp_expert_layout is " + f"expected to set expert_size_per_partition = " + f"num_experts_per_worker before create_weights() runs; check " + f"that get_global_dwdp_manager() is set during model " + f"construction (it is set by py_executor_creator before " + f"model loading)." + ) diff --git a/tensorrt_llm/_torch/modules/dwdp/specs.py b/tensorrt_llm/_torch/modules/dwdp/specs.py new file mode 100644 index 000000000000..dd56862dc64c --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/specs.py @@ -0,0 +1,439 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data classes for DWDP Weight Buffer specifications and layout computation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import cached_property +from typing import Dict, List, Tuple + +import torch + +# (local_start, local_end_capped) for one peer DWDP rank. ``local_start`` +# is ``peer_rank * num_prefetch_experts`` (the rank-to-rank stride). +# ``local_end_capped`` is the *valid* upper bound — i.e., +# ``min(local_start + num_experts_per_worker, num_experts_total)``; the +# tail rank's storage may extend past ``num_experts_total`` but the cap +# truncates the valid range. Adjacent ranks' valid ranges may overlap +# when ``num_prefetch_experts < num_experts_per_worker`` (redundancy mode). +PeerRange = Tuple[int, int] +# Indexed by peer DWDP rank; ``len(PeerRanges) == dwdp_size``. +PeerRanges = List[PeerRange] + + +def compute_peer_ranges( + *, + dwdp_size: int, + num_experts_per_worker: int, + num_prefetch_experts: int, + num_experts_total: int, +) -> PeerRanges: + """Compute every peer rank's valid expert range. + + Each peer's storage holds ``num_experts_per_worker`` slots starting + at ``peer_rank * num_prefetch_experts``. The valid range is the + storage range capped at ``num_experts_total`` (the model's total + expert count), which truncates any padding on the tail rank. + + All ranks deterministically compute the same ``PeerRanges`` from the + shared ``DwdpConfig``, so no allgather is required. + """ + ranges: PeerRanges = [] + for peer_rank in range(dwdp_size): + start = peer_rank * num_prefetch_experts + end_capped = min(start + num_experts_per_worker, num_experts_total) + ranges.append((start, end_capped)) + return ranges + + +def lookup_owner(expert_id: int, peer_ranges: PeerRanges) -> int: + """Return the lowest peer rank whose valid range contains ``expert_id``. + + For uniform partition this is equivalent to + ``expert_id // (num_experts_total // dwdp_size)``. For redundancy + (overlapping ranges) the first-match policy picks the lowest-rank + owner, matching the IPC implementation's behavior so reads of the + same expert id are deterministic across peers. + + Raises: + ValueError: If no peer's valid range contains ``expert_id``. + This is impossible when the partition passes the coverage + check ``(dwdp_size - 1) * stride + size >= num_experts``. + """ + for peer_rank, (start, end) in enumerate(peer_ranges): + if start <= expert_id < end: + return peer_rank + raise ValueError(f"expert_id={expert_id} not owned by any peer in peer_ranges={peer_ranges}") + + +@dataclass(frozen=True) +class WeightSpec: + """Shape and dtype of one expert weight parameter. + + Attributes: + num_experts: Total experts for this layer (e.g., 256). + chunk_shape: Per-rank chunk shape (e.g., (64, 4096, 448)). + full_shape: Full expert shape (e.g., (256, 4096, 448)). + dtype: Data type of the weight tensor. + """ + + num_experts: int + chunk_shape: Tuple[int, ...] + full_shape: Tuple[int, ...] + dtype: torch.dtype + + def __post_init__(self) -> None: + if self.num_experts <= 0: + raise ValueError(f"num_experts must be positive, got {self.num_experts}") + if len(self.chunk_shape) == 0: + raise ValueError("chunk_shape cannot be empty") + if len(self.full_shape) == 0: + raise ValueError("full_shape cannot be empty") + if self.full_shape[0] != self.num_experts: + raise ValueError( + f"full_shape[0]={self.full_shape[0]} must equal num_experts={self.num_experts}" + ) + + @cached_property + def expert_bytes(self) -> int: + """Bytes per single expert (computed from full_shape and dtype).""" + n = 1 + for d in self.full_shape[1:]: + n *= d + element_size = torch.tensor([], dtype=self.dtype).element_size() + return n * element_size + + @cached_property + def chunk_bytes(self) -> int: + """Total bytes for the chunk (all local experts).""" + n = 1 + for d in self.chunk_shape: + n *= d + element_size = torch.tensor([], dtype=self.dtype).element_size() + return n * element_size + + @property + def local_experts(self) -> int: + """Number of local experts in the chunk.""" + return self.chunk_shape[0] + + +# Type alias for per-layer weight specs +LayerWeightSpecs = Dict[int, Dict[str, WeightSpec]] + + +@dataclass +class MnnvlHandleSet: + """MNNVL handles produced by Transport, consumed by WeightBuffer. + + One handle per (layer_idx, weight_name). Each handle is a + CUmemGenericAllocationHandle pointing to local physical memory + containing this rank's expert chunk. + + Attributes: + handles: Mapping from (layer_idx, weight_name) to handle integer. + sizes: Mapping from (layer_idx, weight_name) to physical size in bytes. + """ + + handles: Dict[Tuple[int, str], int] + sizes: Dict[Tuple[int, str], int] + + def __post_init__(self) -> None: + if set(self.handles.keys()) != set(self.sizes.keys()): + raise ValueError("handles and sizes must have the same keys") + + def get_handle(self, layer_idx: int, name: str) -> int: + """Get handle for a specific (layer, weight) pair.""" + key = (layer_idx, name) + if key not in self.handles: + raise KeyError(f"No handle for {key}") + return self.handles[key] + + def get_size(self, layer_idx: int, name: str) -> int: + """Get physical size for a specific (layer, weight) pair.""" + key = (layer_idx, name) + if key not in self.sizes: + raise KeyError(f"No size for {key}") + return self.sizes[key] + + @property + def layer_indices(self) -> list[int]: + """Get sorted list of unique layer indices.""" + return sorted(set(layer_idx for layer_idx, _ in self.handles.keys())) + + def weight_names(self, layer_idx: int) -> list[str]: + """Get weight names for a specific layer.""" + return [name for (lidx, name) in self.handles.keys() if lidx == layer_idx] + + +@dataclass(frozen=True) +class EdgeInfo: + """Page-alignment edge information for setup-time peer data fill. + + When the local expert data doesn't align to page boundaries, there are + edge regions within the MNNVL pages that must be filled with peer data. + + Attributes: + data_offset: Bytes from handle VA start to local data start. + leading_edge: Bytes [page_start, local_start) - peer data region. + trailing_edge: Bytes [local_end, page_end) - peer data region. + page_start: Byte offset of first MNNVL page. + page_end: Byte offset past last MNNVL page. + expert_bytes: Bytes per single expert for this weight. + """ + + data_offset: int + leading_edge: int + trailing_edge: int + page_start: int + page_end: int + expert_bytes: int + + def __post_init__(self) -> None: + if self.data_offset < 0: + raise ValueError(f"data_offset must be non-negative, got {self.data_offset}") + if self.leading_edge < 0: + raise ValueError(f"leading_edge must be non-negative, got {self.leading_edge}") + if self.trailing_edge < 0: + raise ValueError(f"trailing_edge must be non-negative, got {self.trailing_edge}") + if self.page_end < self.page_start: + raise ValueError(f"page_end={self.page_end} must be >= page_start={self.page_start}") + + +@dataclass(frozen=True) +class PageAlignedLayout: + """Page-aligned layout for a specific (layer, weight) pair. + + This computes the VA layout for the composite buffer including: + - Pre-remote region: experts before local_start + - MNNVL region: local expert pages (may include edge bytes) + - Post-remote region: experts after local_end + + Two granularities control alignment: + + - ``granularity`` (typically 2MB): the CUDA VMM page size used for MNNVL + handle mapping. ``page_start`` and ``page_end`` are aligned to this. + - ``pool_granularity`` (typically 16MB = 8 * granularity): the pool page + handle size. ``pre_size`` and ``post_size`` are rounded UP to this so + that each pool-backed region is an exact multiple of pool page handles, + avoiding cuMemMap overlap between pool pages and the MNNVL handle. + + When ``pool_granularity > granularity``, a padding region appears between + the pool-backed pre-region and the MNNVL region (``pre_padding`` bytes) and + between the MNNVL region and the pool-backed post-region (``post_padding`` + bytes). The tensor view must start at ``va_base + pre_padding`` so that + expert indices line up with the correct physical memory. + + Attributes: + expert_bytes: Bytes per single expert. + num_experts: Total number of experts for this layer. + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + granularity: CUDA VMM page granularity in bytes. + pool_granularity: Pool page handle size in bytes (>= granularity, + must be a multiple of granularity). + page_start: align_down(local_start_bytes, granularity). + page_end: align_up(local_end_bytes, granularity). + pre_size: VA bytes for pool-backed pre-region (multiple of + pool_granularity). + mnnvl_size: VA bytes for MNNVL region (multiple of granularity). + post_size: VA bytes for pool-backed post-region (multiple of + pool_granularity). + pre_padding: Extra bytes at end of pre-region due to pool_granularity + rounding (pre_size - page_start). + post_padding: Extra bytes at end of post-region due to pool_granularity + rounding. + data_offset: local_start_bytes - page_start (same as leading_edge). + leading_edge: Bytes before local data within first MNNVL page. + trailing_edge: Bytes after local data within last MNNVL page. + total_size: Total VA size for the composite buffer. + handle_phys_size: Physical size of the MNNVL handle. + """ + + expert_bytes: int + num_experts: int + local_start: int + local_end: int + granularity: int + pool_granularity: int + page_start: int + page_end: int + pre_size: int + mnnvl_size: int + post_size: int + pre_padding: int + post_padding: int + data_offset: int + leading_edge: int + trailing_edge: int + total_size: int + handle_phys_size: int + + @classmethod + def compute( + cls, + expert_bytes: int, + num_experts: int, + local_start: int, + local_end: int, + granularity: int, + handle_phys_size: int, + pool_granularity: int | None = None, + ) -> PageAlignedLayout: + """Compute page-aligned layout for given parameters. + + Args: + expert_bytes: Bytes per single expert. + num_experts: Total number of experts. + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + granularity: CUDA VMM page granularity in bytes. + handle_phys_size: Physical size of the MNNVL handle. + pool_granularity: Pool page handle size in bytes. Must be a + positive multiple of ``granularity``. Defaults to + ``granularity`` (i.e. each pool page = one VMM page). + + Returns: + PageAlignedLayout with all computed fields. + + Raises: + ValueError: If parameters are invalid. + """ + if local_start < 0 or local_end > num_experts or local_start >= local_end: + raise ValueError( + f"Invalid expert range: local_start={local_start}, " + f"local_end={local_end}, num_experts={num_experts}" + ) + if granularity <= 0 or (granularity & (granularity - 1)) != 0: + raise ValueError(f"granularity must be a positive power of 2, got {granularity}") + + if pool_granularity is None: + pool_granularity = granularity + if pool_granularity <= 0 or pool_granularity % granularity != 0: + raise ValueError( + f"pool_granularity must be a positive multiple of granularity " + f"({granularity}), got {pool_granularity}" + ) + if (pool_granularity & (pool_granularity - 1)) != 0: + raise ValueError(f"pool_granularity must be a power of 2, got {pool_granularity}") + + # Compute byte offsets + local_start_bytes = local_start * expert_bytes + local_end_bytes = local_end * expert_bytes + total_expert_bytes = num_experts * expert_bytes + + # Page-align the local region to VMM granularity (for MNNVL handle mapping) + page_start = _align_down(local_start_bytes, granularity) + page_end = _align_up(local_end_bytes, granularity) + + # Edge bytes (within the MNNVL region) + data_offset = local_start_bytes - page_start + leading_edge = data_offset + trailing_edge = page_end - local_end_bytes + + # MNNVL region size is the page-aligned span (unchanged by pool_granularity) + mnnvl_size = page_end - page_start + + # Validate: mnnvl_size must not exceed the handle's physical size. + # On GB200, cuMemMap with fabric handles requires size == phys_size + # (partial mapping returns CUDA_ERROR_NOT_SUPPORTED). + if mnnvl_size > handle_phys_size: + raise ValueError( + f"mnnvl_size ({mnnvl_size}) exceeds handle_phys_size " + f"({handle_phys_size}). The Transport must allocate handles " + f"with phys_size >= page_end - page_start. " + f"page_start={page_start}, page_end={page_end}, " + f"local_start={local_start}, local_end={local_end}, " + f"expert_bytes={expert_bytes}, granularity={granularity}" + ) + + # Pre region: everything before the MNNVL region, rounded UP to + # pool_granularity so each pool page maps cleanly without overlapping + # the MNNVL handle. + pre_size = _align_up(page_start, pool_granularity) + pre_padding = pre_size - page_start + + # Post region: everything after the MNNVL region, rounded UP to + # pool_granularity. + post_size_raw = _align_up(total_expert_bytes, granularity) - page_end + post_size = _align_up(post_size_raw, pool_granularity) + post_padding = post_size - post_size_raw + + # Total VA size + total_size = pre_size + mnnvl_size + post_size + + return cls( + expert_bytes=expert_bytes, + num_experts=num_experts, + local_start=local_start, + local_end=local_end, + granularity=granularity, + pool_granularity=pool_granularity, + page_start=page_start, + page_end=page_end, + pre_size=pre_size, + mnnvl_size=mnnvl_size, + post_size=post_size, + pre_padding=pre_padding, + post_padding=post_padding, + data_offset=data_offset, + leading_edge=leading_edge, + trailing_edge=trailing_edge, + total_size=total_size, + handle_phys_size=handle_phys_size, + ) + + def get_edge_info(self) -> EdgeInfo: + """Get EdgeInfo for this layout.""" + return EdgeInfo( + data_offset=self.data_offset, + leading_edge=self.leading_edge, + trailing_edge=self.trailing_edge, + page_start=self.page_start, + page_end=self.page_end, + expert_bytes=self.expert_bytes, + ) + + @property + def pre_pages(self) -> int: + """Number of pool pages in the pre-remote region.""" + return self.pre_size // self.pool_granularity + + @property + def mnnvl_pages(self) -> int: + """Number of VMM pages in the MNNVL region.""" + return self.mnnvl_size // self.granularity + + @property + def post_pages(self) -> int: + """Number of pool pages in the post-remote region.""" + return self.post_size // self.pool_granularity + + @property + def remote_pages(self) -> int: + """Total number of pool pages needed for remote regions.""" + return self.pre_pages + self.post_pages + + +def _align_up(value: int, alignment: int) -> int: + """Align value up to the nearest multiple of alignment.""" + return ((value + alignment - 1) // alignment) * alignment + + +def _align_down(value: int, alignment: int) -> int: + """Align value down to the nearest multiple of alignment.""" + return (value // alignment) * alignment diff --git a/tensorrt_llm/_torch/modules/dwdp/transport.py b/tensorrt_llm/_torch/modules/dwdp/transport.py new file mode 100644 index 000000000000..f018276538c4 --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/transport.py @@ -0,0 +1,646 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DWDP Transport layer: MNNVL handle allocation and cross-process exchange. + +DWDPTransport handles the setup-time work of: +1. Allocating MNNVL fabric handles for each local expert weight chunk. +2. Copying local parameter data into the fabric handle memory. +3. Exchanging handle bytes via an MPI communicator so all peers can import them. +4. Importing peer handles and creating read-only tensor views for P2P reads. + +After create() completes, the caller obtains: +- MnnvlHandleSet: local handles for WeightBuffer to map into its composite VA. +- peer_views: immutable tensor views into every peer's MNNVL memory, used by + WeightManager to read remote expert data via P2P. + +Design: +- Interleaved allocation: each (layer, weight) is processed one at a time. + After copying data into the MNNVL handle the original parameter is freed + immediately to avoid OOM on memory-constrained systems. +- GB200 constraint: all handles use CU_MEM_HANDLE_TYPE_FABRIC. +- Peer views are immutable: after setup, MNNVL handles are never written again. + P2P reads are safe across processes without synchronization. +""" + +from __future__ import annotations + +from typing import Dict, List, Tuple + +import torch + +from tensorrt_llm.logger import logger + +try: + from cuda.bindings import driver as cuda + + logger.debug("[DWDP transport] Using cuda.bindings.driver") +except ImportError: + from cuda import cuda + + logger.debug("[DWDP transport] Falling back to legacy `cuda` bindings") + +from .specs import LayerWeightSpecs, MnnvlHandleSet, PeerRanges, compute_peer_ranges +from .vmm import ( + align_down, + align_up, + check_cu_result, + create_fabric_handle, + free_va, + get_allocation_granularity, + map_handle, + release_handle, + reserve_va, + set_access, + tensor_from_ptr, + unmap_va, +) + + +def _export_fabric_handle(handle: int, device_id: int) -> bytes: + """Export an MNNVL fabric handle to shareable bytes. + + Uses cuMemExportToShareableHandle with CU_MEM_HANDLE_TYPE_FABRIC to + produce a byte representation that can be sent to peer processes via + the DWDP MPI communicator. + + Args: + handle: CUDA memory handle (from create_fabric_handle). + device_id: CUDA device ordinal (used to build allocation properties). + + Returns: + Byte representation of the fabric handle. + + Raises: + RuntimeError: If the CUDA export call fails. + """ + fabric_handle = check_cu_result( + cuda.cuMemExportToShareableHandle( + handle, + cuda.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_FABRIC, + 0, + ) + ) + # The returned fabric_handle is a CUmemFabricHandle struct. + # Convert to bytes for transmission via MPI. + if hasattr(fabric_handle, "data"): + return bytes(fabric_handle.data) + return bytes(fabric_handle) + + +def _import_fabric_handle(handle_bytes: bytes) -> int: + """Import a fabric handle from shareable bytes. + + Uses cuMemImportFromShareableHandle with CU_MEM_HANDLE_TYPE_FABRIC to + reconstruct a CUDA memory handle from bytes received via MPI allgather. + + Args: + handle_bytes: Byte representation produced by _export_fabric_handle. + + Returns: + Imported CUDA memory handle as integer. + + Raises: + RuntimeError: If the CUDA import call fails. + """ + imported_handle = check_cu_result( + cuda.cuMemImportFromShareableHandle( + handle_bytes, + cuda.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_FABRIC, + ) + ) + return int(imported_handle) + + +class DWDPTransport: + """MNNVL handle allocation and cross-process exchange for DWDP. + + This class manages the full lifecycle of MNNVL fabric handles: + allocation, data population, export/import via an MPI communicator, + and creation of peer tensor views for P2P reads. + + Use the ``create()`` class method to construct an instance. After + construction, ``get_handle_set()`` provides handles for WeightBuffer + and ``get_peer_views()`` provides read-only tensor views for + WeightManager. + + Attributes: + dwdp_rank: This process's DWDP rank. + dwdp_size: Total number of DWDP ranks. + device_id: CUDA device ordinal. + granularity: VMM page granularity in bytes. + """ + + __slots__ = ( + "_dwdp_rank", + "_dwdp_size", + "_device_id", + "_granularity", + "_handle_set", + "_peer_views", + "_peer_ranges", + "_local_handles", + "_imported_handles", + "_peer_va_mappings", + "_released", + ) + + def __init__( + self, + dwdp_rank: int, + dwdp_size: int, + device_id: int, + granularity: int, + handle_set: MnnvlHandleSet, + peer_views: Dict[Tuple[int, int, str], torch.Tensor], + peer_ranges: PeerRanges, + local_handles: List[int], + imported_handles: List[int], + peer_va_mappings: List[Tuple[int, int]], + ) -> None: + """Internal constructor. Use DWDPTransport.create() instead. + + Args: + dwdp_rank: This process's DWDP rank. + dwdp_size: Total number of DWDP ranks. + device_id: CUDA device ordinal. + granularity: VMM page granularity in bytes. + handle_set: Local MNNVL handles for WeightBuffer. + peer_views: Immutable tensor views into peer MNNVL memory. + peer_ranges: Per-peer ``(local_start, local_end_capped)`` tuples + indexed by DWDP rank. The capped end truncates the tail + rank's padding; consumers (lookup_owner, fill_edge_bytes, + weight_manager) read these to find the actual owner of a + given expert id. + local_handles: Raw local handle integers (for cleanup). + imported_handles: Raw imported handle integers (for cleanup). + peer_va_mappings: List of (va, size) for peer VA regions (for cleanup). + """ + self._dwdp_rank = dwdp_rank + self._dwdp_size = dwdp_size + self._device_id = device_id + self._granularity = granularity + self._handle_set = handle_set + self._peer_views = peer_views + self._peer_ranges = peer_ranges + self._local_handles = local_handles + self._imported_handles = imported_handles + self._peer_va_mappings = peer_va_mappings + self._released = False + + @classmethod + def create( + cls, + layer_weight_specs: LayerWeightSpecs, + local_params: Dict[Tuple[int, str], torch.Tensor], + comm, + dwdp_rank: int, + dwdp_size: int, + device_id: int, + local_start: int, + local_end: int, + num_experts_per_worker: int, + num_prefetch_experts: int, + ) -> DWDPTransport: + """Allocate MNNVL handles, populate them, and exchange with all peers. + + This is the main entry point. Each of the ``dwdp_size`` processes calls + this method concurrently. The method: + + 1. For each (layer_idx, weight_name) — one at a time to limit peak + memory usage — every rank in the DWDP communicator: + a. Allocates a fabric handle large enough for the local chunk. + b. Maps the handle to a temporary VA and copies the local parameter + tensor into it at the correct ``data_offset``. + c. Frees the original parameter tensor immediately. + d. Unmaps the temporary VA (the physical handle persists). + e. Exports the handle to bytes and shares it with every peer via + a per-pair ``comm.allgather(handle_bytes)``. The allgather is + itself a synchronization point so no explicit barrier is needed + between Phase 1 iterations. + 2. For each peer rank (skipping self), for each (layer_idx, weight_name): + a. Retrieves the peer's exported handle bytes from the Phase 1 + allgather cache. + b. Imports the fabric handle. + c. Maps it to a VA and creates a tensor view for P2P reads. + 3. ``comm.Barrier()`` after all imports — ensures no rank starts + consuming peer views before every rank has finished importing. + + Args: + layer_weight_specs: Per-layer weight specifications. Keys are layer + indices, values are dicts mapping weight names to WeightSpec. + local_params: Local parameter tensors keyed by (layer_idx, name). + Each tensor has shape == spec.chunk_shape and is on the correct + CUDA device. These tensors are consumed (freed) during the call. + comm: mpi4py communicator scoped to the DWDP group (created by + DwdpManager). Must have exactly ``dwdp_size`` ranks. + dwdp_rank: This process's DWDP rank (0..dwdp_size-1). + dwdp_size: Total number of DWDP ranks in ``comm``. + device_id: CUDA device ordinal. + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + + Returns: + Ready-to-use DWDPTransport with handles and peer views. + + Raises: + ValueError: If local_params keys do not match layer_weight_specs. + RuntimeError: If any CUDA operation fails. + """ + granularity = get_allocation_granularity(device_id) + + # Pick any spec to read the model's total expert count. All layers + # share the same ``num_experts`` (the gate-side global expert table). + first_spec = next(iter(next(iter(layer_weight_specs.values())).values())) + num_experts_total = first_spec.num_experts + + # Compute every peer's valid expert range deterministically from the + # shared DwdpConfig. Used by lookup_owner downstream + # (fill_edge_bytes, weight_manager.prefetch_layer, fixup-side + # allgathers) so the same first-match owner policy is applied + # everywhere — including non-uniform tail padding and redundancy + # where adjacent ranges overlap. + peer_ranges = compute_peer_ranges( + dwdp_size=dwdp_size, + num_experts_per_worker=num_experts_per_worker, + num_prefetch_experts=num_prefetch_experts, + num_experts_total=num_experts_total, + ) + + # Validate that every (layer_idx, name) in specs has a matching param + expected_keys = set() + for layer_idx, weight_specs in layer_weight_specs.items(): + for name in weight_specs: + expected_keys.add((layer_idx, name)) + provided_keys = set(local_params.keys()) + if expected_keys != provided_keys: + missing = expected_keys - provided_keys + extra = provided_keys - expected_keys + raise ValueError(f"local_params keys mismatch. Missing: {missing}, Extra: {extra}") + + handles: Dict[Tuple[int, str], int] = {} + sizes: Dict[Tuple[int, str], int] = {} + local_handles: List[int] = [] + imported_handles: List[int] = [] + peer_va_mappings: List[Tuple[int, int]] = [] + peer_views: Dict[Tuple[int, int, str], torch.Tensor] = {} + + # Per-pair allgather result cache: (layer_idx, name) -> list[bytes] + # indexed by peer rank. Populated in Phase 1, consumed in Phase 2. + all_exports: Dict[Tuple[int, str], List[bytes]] = {} + + try: + # ---------------------------------------------------------- + # Phase 1: Allocate local handles and allgather bytes + # ---------------------------------------------------------- + # Every rank iterates pairs in the same sorted order so that + # per-pair comm.allgather calls match across ranks. + for layer_idx in sorted(layer_weight_specs.keys()): + weight_specs = layer_weight_specs[layer_idx] + for name in sorted(weight_specs.keys()): + spec = weight_specs[name] + key = (layer_idx, name) + local_param = local_params[key] + + # Compute physical handle size using the SAME formula + # as PageAlignedLayout.compute() to guarantee that + # phys_size == layout.mnnvl_size. On GB200, fabric + # handles require cuMemMap with size == phys_size + # (partial mapping returns CUDA_ERROR_NOT_SUPPORTED). + local_start_bytes = local_start * spec.expert_bytes + local_end_bytes = local_end * spec.expert_bytes + page_start = align_down(local_start_bytes, granularity) + page_end = align_up(local_end_bytes, granularity) + phys_size = page_end - page_start + data_offset = local_start_bytes - page_start + + # (a) Create the fabric handle + handle = create_fabric_handle(phys_size, device_id) + local_handles.append(handle) + handles[key] = handle + sizes[key] = phys_size + + # (b) Reserve temp VA, map, set access, create tensor view + temp_va = reserve_va(phys_size, granularity) + mapped = False + try: + map_handle(temp_va, phys_size, handle, 0) + mapped = True + set_access(temp_va, phys_size, device_id) + + # Create a flat tensor view at data_offset within the + # mapped VA, sized exactly to the chunk. + data_ptr = temp_va + data_offset + handle_tensor = tensor_from_ptr( + ptr=data_ptr, + shape=spec.chunk_shape, + dtype=spec.dtype, + device_id=device_id, + ) + + # (c) Copy local param into the handle memory + handle_tensor.copy_(local_param) + torch.cuda.synchronize(device_id) + + # (d) Free the original param to reclaim memory + local_param.untyped_storage().resize_(0) + torch.cuda.empty_cache() + + # (e) Unmap temp VA (handle stays alive) + unmap_va(temp_va, phys_size) + mapped = False + finally: + if mapped: + try: + unmap_va(temp_va, phys_size) + except Exception as e: + logger.warning( + f"[DWDPTransport] temp VA unmap on error " + f"path failed (ignored): {e!r}" + ) + free_va(temp_va, phys_size) + + # (f) Export and allgather to all peers + handle_bytes = _export_fabric_handle(handle, device_id) + # comm.allgather implicitly synchronizes all ranks on this + # (layer_idx, name) — no explicit barrier required. + all_exports[key] = comm.allgather(handle_bytes) + + logger.debug( + f"[DWDPTransport] Rank {dwdp_rank}: exported + allgathered " + f"handle for layer={layer_idx} name={name} " + f"phys_size={phys_size} data_offset={data_offset}" + ) + + logger.info( + f"[DWDPTransport] Rank {dwdp_rank}: all handles exported, importing peer handles..." + ) + + # ---------------------------------------------------------- + # Phase 2: Import peer handles and create tensor views + # ---------------------------------------------------------- + for peer_rank in range(dwdp_size): + if peer_rank == dwdp_rank: + continue + + # Compute peer's expert range using the same config-driven + # formula every rank uses for itself (size = num_experts_per_worker, + # stride = num_prefetch_experts). All ranks share the same + # DwdpConfig, so peer ranges are deterministic — no allgather needed. + for layer_idx in sorted(layer_weight_specs.keys()): + weight_specs = layer_weight_specs[layer_idx] + for name in sorted(weight_specs.keys()): + spec = weight_specs[name] + + peer_local_start = peer_rank * num_prefetch_experts + peer_local_end = peer_local_start + num_experts_per_worker + + peer_start_bytes = peer_local_start * spec.expert_bytes + peer_end_bytes = peer_local_end * spec.expert_bytes + peer_page_start = align_down(peer_start_bytes, granularity) + peer_page_end = align_up(peer_end_bytes, granularity) + phys_size = peer_page_end - peer_page_start + peer_data_offset = peer_start_bytes - peer_page_start + + # (a) Retrieve peer handle bytes from Phase 1 cache + peer_handle_bytes = all_exports[(layer_idx, name)][peer_rank] + + # (b) Import the fabric handle + imported_handle = _import_fabric_handle(peer_handle_bytes) + imported_handles.append(imported_handle) + + # (c) Reserve VA, map, set access + peer_va = reserve_va(phys_size, granularity) + map_handle(peer_va, phys_size, imported_handle, 0) + set_access(peer_va, phys_size, device_id) + peer_va_mappings.append((peer_va, phys_size)) + + peer_data_ptr = peer_va + peer_data_offset + peer_tensor = tensor_from_ptr( + ptr=peer_data_ptr, + shape=spec.chunk_shape, + dtype=spec.dtype, + device_id=device_id, + ) + + view_key = (peer_rank, layer_idx, name) + peer_views[view_key] = peer_tensor + + logger.debug( + f"[DWDPTransport] Rank {dwdp_rank}: imported " + f"peer={peer_rank} layer={layer_idx} name={name} " + f"peer_data_offset={peer_data_offset}" + ) + + # ---------------------------------------------------------- + # Phase 3: Barrier — all handles imported + # ---------------------------------------------------------- + comm.Barrier() + + logger.info( + f"[DWDPTransport] Rank {dwdp_rank}: setup complete. " + f"{len(handles)} local handles, " + f"{len(peer_views)} peer views." + ) + + handle_set = MnnvlHandleSet(handles=handles, sizes=sizes) + + return cls( + dwdp_rank=dwdp_rank, + dwdp_size=dwdp_size, + device_id=device_id, + granularity=granularity, + handle_set=handle_set, + peer_views=peer_views, + peer_ranges=peer_ranges, + local_handles=local_handles, + imported_handles=imported_handles, + peer_va_mappings=peer_va_mappings, + ) + + except Exception as exc: + # On failure, log the original exception then clean up everything + # we allocated so far. + logger.error( + f"[DWDPTransport] Rank {dwdp_rank}: create() failed with {exc!r}; " + f"cleaning up partial allocations" + ) + _cleanup_resources( + local_handles=local_handles, + imported_handles=imported_handles, + peer_va_mappings=peer_va_mappings, + ) + raise + + # ------------------------------------------------------------------ + # Public accessors + # ------------------------------------------------------------------ + + def get_handle_set(self) -> MnnvlHandleSet: + """Return the local MNNVL handle set for WeightBuffer. + + Each entry maps (layer_idx, weight_name) to a CUDA fabric handle + containing this rank's local expert chunk. + + Returns: + MnnvlHandleSet with handles and physical sizes. + + Raises: + RuntimeError: If transport has been released. + """ + if self._released: + raise RuntimeError("DWDPTransport has been released") + return self._handle_set + + def get_peer_views(self) -> Dict[Tuple[int, int, str], torch.Tensor]: + """Return immutable P2P tensor views into every peer's MNNVL memory. + + Keys are (peer_rank, layer_idx, weight_name). Each tensor has + shape == spec.chunk_shape and is read-only (writes would corrupt + the peer's data). + + Returns: + Dict mapping (peer_rank, layer_idx, name) to tensor views. + + Raises: + RuntimeError: If transport has been released. + """ + if self._released: + raise RuntimeError("DWDPTransport has been released") + return self._peer_views + + def get_peer_ranges(self) -> PeerRanges: + """Return per-peer ``(local_start, local_end_capped)`` tuples. + + Used by ``fill_edge_bytes`` and the runtime ``WeightManager`` to + resolve the owner of any given expert id under non-uniform + partition or redundancy. The end is capped at ``num_experts`` + (i.e., reflects the *valid* expert range — not the storage range + which may include tail-padding slots). + + Raises: + RuntimeError: If transport has been released. + """ + if self._released: + raise RuntimeError("DWDPTransport has been released") + return self._peer_ranges + + @property + def dwdp_rank(self) -> int: + """This process's DWDP rank.""" + return self._dwdp_rank + + @property + def dwdp_size(self) -> int: + """Total number of DWDP ranks.""" + return self._dwdp_size + + @property + def device_id(self) -> int: + """CUDA device ordinal.""" + return self._device_id + + @property + def granularity(self) -> int: + """VMM page granularity in bytes.""" + return self._granularity + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def release(self) -> None: + """Release all CUDA VMM resources. Idempotent. + + This unmaps all peer VA regions, releases imported handles, and + releases local handles. After this call, ``get_handle_set()`` and + ``get_peer_views()`` will raise RuntimeError. + """ + if self._released: + return + self._released = True + + # Invalidate tensor views (they point to VA we are about to free) + self._peer_views.clear() + + _cleanup_resources( + local_handles=self._local_handles, + imported_handles=self._imported_handles, + peer_va_mappings=self._peer_va_mappings, + ) + + self._local_handles.clear() + self._imported_handles.clear() + self._peer_va_mappings.clear() + + logger.debug(f"[DWDPTransport] Rank {self._dwdp_rank}: released all resources") + + def __del__(self) -> None: + """Clean up on garbage collection (best-effort; errors logged to debug).""" + try: + self.release() + except Exception as e: + logger.debug(f"[DWDPTransport] __del__ release failed (ignored): {e!r}") + + def __enter__(self) -> DWDPTransport: + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + """Context manager exit — release resources.""" + self.release() + return False + + +def _cleanup_resources( + local_handles: List[int], + imported_handles: List[int], + peer_va_mappings: List[Tuple[int, int]], +) -> None: + """Best-effort cleanup of CUDA resources. + + Unmap and free all peer VA regions, release imported handles, then + release local handles. Errors are logged but not raised so that + cleanup proceeds as far as possible. + + Args: + local_handles: Local MNNVL handle integers. + imported_handles: Imported MNNVL handle integers. + peer_va_mappings: List of (va, size) for peer VA regions. + """ + # Step 1: Unmap and free peer VA regions + for va, size in peer_va_mappings: + try: + unmap_va(va, size) + except Exception as e: + logger.warning(f"[DWDPTransport] Failed to unmap peer VA 0x{va:x}: {e}") + try: + free_va(va, size) + except Exception as e: + logger.warning(f"[DWDPTransport] Failed to free peer VA 0x{va:x}: {e}") + + # Step 2: Release imported handles + for h in imported_handles: + try: + release_handle(h) + except Exception as e: + logger.warning(f"[DWDPTransport] Failed to release imported handle {h}: {e}") + + # Step 3: Release local handles + for h in local_handles: + try: + release_handle(h) + except Exception as e: + logger.warning(f"[DWDPTransport] Failed to release local handle {h}: {e}") diff --git a/tensorrt_llm/_torch/modules/dwdp/vmm.py b/tensorrt_llm/_torch/modules/dwdp/vmm.py new file mode 100644 index 000000000000..578b52e25ac9 --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/vmm.py @@ -0,0 +1,596 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CUDA Virtual Memory Management (VMM) utilities for DWDP. + +This module provides low-level CUDA VMM operations including: +- Page alignment utilities +- Allocation property configuration +- Granularity queries +- Handle creation and mapping +- Tensor creation from VA pointers +""" + +from __future__ import annotations + +import functools +import platform +from typing import Tuple + +import torch + +from tensorrt_llm.logger import logger + +try: + from cuda.bindings import driver as cuda + + logger.debug("[DWDP vmm] Using cuda.bindings.driver") +except ImportError: + from cuda import cuda + + logger.debug("[DWDP vmm] Falling back to legacy `cuda` bindings (cuda.bindings not available)") + + +def check_cu_result(cu_func_ret): + """Check CUDA driver API result and raise on error. + + Args: + cu_func_ret: Return value from CUDA driver API call. + + Returns: + Extracted result(s) from the tuple, or None if only status. + + Raises: + RuntimeError: If CUDA call failed. + """ + if isinstance(cu_func_ret, tuple): + cu_result, *others = cu_func_ret + if cu_result != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"CUDA error: {cu_result}") + if len(others) == 1: + return others[0] + elif len(others) > 1: + return tuple(others) + else: + return None + else: + if cu_func_ret != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"CUDA error: {cu_func_ret}") + return None + + +def align_up(value: int, alignment: int) -> int: + """Align value up to the nearest multiple of alignment. + + Args: + value: Value to align. + alignment: Alignment boundary (must be power of 2). + + Returns: + Value aligned up to alignment boundary. + + Raises: + ValueError: If alignment is not a positive power of 2. + """ + if alignment <= 0 or (alignment & (alignment - 1)) != 0: + raise ValueError(f"alignment must be a positive power of 2, got {alignment}") + return ((value + alignment - 1) // alignment) * alignment + + +def align_down(value: int, alignment: int) -> int: + """Align value down to the nearest multiple of alignment. + + Args: + value: Value to align. + alignment: Alignment boundary (must be power of 2). + + Returns: + Value aligned down to alignment boundary. + + Raises: + ValueError: If alignment is not a positive power of 2. + """ + if alignment <= 0 or (alignment & (alignment - 1)) != 0: + raise ValueError(f"alignment must be a positive power of 2, got {alignment}") + return (value // alignment) * alignment + + +def get_allocation_prop(device_id: int, fabric_only: bool = True) -> cuda.CUmemAllocationProp: + """Get allocation property for MNNVL memory. + + Args: + device_id: CUDA device ordinal. + fabric_only: If True, only return fabric handle type (required for GB200). + If False, fall back to POSIX_FILE_DESCRIPTOR on x86_64. + + Returns: + CUmemAllocationProp configured for the device. + """ + location = cuda.CUmemLocation() + location.type = cuda.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + location.id = device_id + + allocation_prop = cuda.CUmemAllocationProp() + allocation_prop.type = cuda.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + allocation_prop.location = location + + if fabric_only: + allocation_prop.requestedHandleTypes = ( + cuda.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_FABRIC + ) + else: + # Differentiate based on architecture + arch = platform.machine().lower() + is_on_aarch64 = "aarch64" in arch + if is_on_aarch64: + allocation_prop.requestedHandleTypes = ( + cuda.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_FABRIC + ) + else: + allocation_prop.requestedHandleTypes = ( + cuda.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + + return allocation_prop + + +@functools.lru_cache(maxsize=None) +def _get_allocation_granularity_cached(device_id: int) -> int: + """Cached implementation of get_allocation_granularity (thread-safe via lru_cache).""" + allocation_prop = get_allocation_prop(device_id) + option = cuda.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED + return check_cu_result(cuda.cuMemGetAllocationGranularity(prop=allocation_prop, option=option)) + + +def get_allocation_granularity(device_id: int, use_cache: bool = True) -> int: + """Get allocation granularity for VMM on the specified device. + + Args: + device_id: CUDA device ordinal. + use_cache: If True, cache and reuse granularity per device (thread-safe). + + Returns: + Allocation granularity in bytes (typically 2MB on GB200). + """ + if use_cache: + return _get_allocation_granularity_cached(device_id) + + allocation_prop = get_allocation_prop(device_id) + option = cuda.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED + return check_cu_result(cuda.cuMemGetAllocationGranularity(prop=allocation_prop, option=option)) + + +def get_access_desc(device_id: int) -> cuda.CUmemAccessDesc: + """Get memory access descriptor for read/write access. + + Args: + device_id: CUDA device ordinal. + + Returns: + CUmemAccessDesc for read/write access. + """ + location = cuda.CUmemLocation() + location.type = cuda.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + location.id = device_id + + madesc = cuda.CUmemAccessDesc() + madesc.location = location + madesc.flags = cuda.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + + return madesc + + +def create_fabric_handle(size: int, device_id: int) -> int: + """Create a fabric memory handle. + + Args: + size: Size in bytes (must be aligned to granularity). + device_id: CUDA device ordinal. + + Returns: + Handle as integer. + + Raises: + RuntimeError: If creation fails. + """ + allocation_prop = get_allocation_prop(device_id, fabric_only=True) + handle = check_cu_result(cuda.cuMemCreate(size, allocation_prop, flags=0)) + return int(handle) + + +def create_local_handle(size: int, device_id: int) -> int: + """Create a local (non-shareable) memory handle. + + Uses CU_MEM_HANDLE_TYPE_NONE — the handle cannot be exported to other + processes but avoids consuming NVLink fabric routing table entries. + Use this for buffers that only need local GPU access (e.g., page pool + double buffers that are written via P2P copy and read locally). + + Args: + size: Size in bytes (must be aligned to granularity). + device_id: CUDA device ordinal. + + Returns: + Handle as integer. + + Raises: + RuntimeError: If creation fails. + """ + allocation_prop = get_allocation_prop(device_id, fabric_only=False) + # Override to HANDLE_TYPE_NONE (no shareable handle) + allocation_prop.requestedHandleTypes = cuda.CUmemAllocationHandleType(0) + handle = check_cu_result(cuda.cuMemCreate(size, allocation_prop, flags=0)) + return int(handle) + + +def release_handle(handle: int) -> None: + """Release a memory handle. + + Args: + handle: Handle to release. + """ + if handle != 0: + check_cu_result(cuda.cuMemRelease(handle)) + + +def reserve_va(size: int, granularity: int) -> int: + """Reserve virtual address space. + + Args: + size: Size in bytes. + granularity: Alignment granularity. + + Returns: + Virtual address as integer. + """ + va = check_cu_result(cuda.cuMemAddressReserve(size, granularity, 0, 0)) + return int(va) + + +def free_va(va: int, size: int) -> None: + """Free reserved virtual address space. + + Args: + va: Virtual address to free. + size: Size of the reserved region. + """ + if va != 0: + device_ptr = cuda.CUdeviceptr(va) + check_cu_result(cuda.cuMemAddressFree(device_ptr, size)) + + +def map_handle(va: int, size: int, handle: int, offset: int = 0) -> None: + """Map a memory handle to virtual address. + + Args: + va: Virtual address to map to. + size: Size in bytes. + handle: Memory handle. + offset: Offset within the handle (typically 0 for fabric handles). + """ + check_cu_result(cuda.cuMemMap(va, size, offset, handle, 0)) + + +def unmap_va(va: int, size: int) -> None: + """Unmap virtual address region. + + Args: + va: Virtual address to unmap. + size: Size in bytes. + """ + check_cu_result(cuda.cuMemUnmap(va, size)) + + +def set_access(va: int, size: int, device_id: int) -> None: + """Set memory access permissions. + + Args: + va: Virtual address. + size: Size in bytes. + device_id: CUDA device ordinal. + """ + madesc = get_access_desc(device_id) + check_cu_result(cuda.cuMemSetAccess(va, size, [madesc], 1)) + + +def tensor_from_ptr( + ptr: int, + shape: Tuple[int, ...], + dtype: torch.dtype, + device_id: int, +) -> torch.Tensor: + """Create a torch tensor from a CUDA virtual address pointer. + + This creates a tensor view over existing CUDA memory. The caller is + responsible for ensuring the memory remains valid for the tensor's lifetime. + + Args: + ptr: CUDA virtual address pointer. + shape: Shape of the tensor. + dtype: Data type of the tensor. + device_id: CUDA device ordinal. + + Returns: + Torch tensor viewing the memory at ptr. + + Raises: + ValueError: If ptr is 0 or shape is invalid. + """ + if ptr == 0: + raise ValueError("Cannot create tensor from null pointer") + + numel = 1 + for dim in shape: + if dim <= 0: + raise ValueError(f"All dimensions must be positive, got shape={shape}") + numel *= dim + + element_size = torch.tensor([], dtype=dtype).element_size() + total_bytes = numel * element_size + + # Use DLPack approach for robust pointer wrapping + # This mimics the pack_strided_memory pattern from _dlpack_utils + try: + from tensorrt_llm._dlpack_utils import create_dlpack_capsule + + # For contiguous memory, we create a single segment + capsule_wrapper = create_dlpack_capsule( + ptr=ptr, + segment_size=total_bytes, + segment_stride=0, # Not used for single segment + num_segments=1, + torch_dtype=dtype, + dev_id=device_id, + ) + + # Convert via DLPack + tensor = torch.utils.dlpack.from_dlpack(capsule_wrapper.capsule) + # Keep reference to capsule to prevent GC + tensor._capsule_wrapper = capsule_wrapper + + # Reshape to desired shape + return tensor.reshape(shape) + + except ImportError: + logger.info( + "[DWDP vmm] tensorrt_llm._dlpack_utils not available; " + "falling back to _tensor_from_ptr_internal (D2D copy, not zero-copy)" + ) + return _tensor_from_ptr_internal(ptr, shape, dtype, device_id) + + +def _tensor_from_ptr_internal( + ptr: int, + shape: Tuple[int, ...], + dtype: torch.dtype, + device_id: int, +) -> torch.Tensor: + """Internal tensor creation using PyTorch storage. + + Used as fallback when tensorrt_llm._dlpack_utils is not available. + This uses PyTorch's internal _UntypedStorage to wrap a CUDA pointer. + """ + import ctypes + + numel = 1 + for dim in shape: + numel *= dim + + element_size = torch.tensor([], dtype=dtype).element_size() + total_bytes = numel * element_size + + # Create an untyped storage from the CUDA pointer + # Using torch.cuda.cudart() to access the storage creation + device = torch.device(f"cuda:{device_id}") + + # Create storage using ctypes to call internal PyTorch function + # This approach uses torch.Storage.from_buffer equivalent for CUDA + try: + # Get the data pointer of the new storage and copy memory reference + # Note: This creates a view, we need to use set_ instead + tensor = torch.empty(shape, dtype=dtype, device=device) + + # Use ctypes to set the data pointer directly + # This is a workaround - in production, use tensorrt_llm._dlpack_utils + data_ptr_func = ctypes.pythonapi.PyLong_AsVoidPtr + data_ptr_func.argtypes = [ctypes.py_object] + data_ptr_func.restype = ctypes.c_void_p + + # For now, create the tensor and copy the data using cudaMemcpy + # This is not zero-copy but safe for testing + # In production with full tensorrt_llm, the DLPack path will be used + cuda.cuMemcpyDtoD(tensor.data_ptr(), ptr, total_bytes) + + return tensor + + except Exception as e: + logger.warning( + f"[DWDP vmm] _tensor_from_ptr_internal UntypedStorage path failed ({e!r}); " + "falling back to empty-tensor + cuMemcpyDtoD (not zero-copy)" + ) + tensor = torch.empty(shape, dtype=dtype, device=device) + cuda.cuMemcpyDtoD(tensor.data_ptr(), ptr, total_bytes) + return tensor + + +class VMMHandle: + """RAII wrapper for a CUDA VMM physical memory handle. + + This class manages the lifecycle of a CUDA memory handle, + ensuring proper cleanup on destruction. + + Attributes: + handle: The raw CUDA handle as integer. + size: Size of the allocated memory in bytes. + device_id: CUDA device ordinal. + """ + + __slots__ = ("_handle", "_size", "_device_id", "_released") + + def __init__(self, size: int, device_id: int): + """Create a new VMM handle. + + Args: + size: Size in bytes (will be aligned to granularity). + device_id: CUDA device ordinal. + """ + granularity = get_allocation_granularity(device_id) + aligned_size = align_up(size, granularity) + + self._handle = create_fabric_handle(aligned_size, device_id) + self._size = aligned_size + self._device_id = device_id + self._released = False + + @property + def handle(self) -> int: + """Get the raw handle value.""" + if self._released: + raise RuntimeError("Handle has been released") + return self._handle + + @property + def size(self) -> int: + """Get the allocated size.""" + return self._size + + @property + def device_id(self) -> int: + """Get the device ID.""" + return self._device_id + + def release(self) -> None: + """Release the handle. Idempotent.""" + if not self._released: + release_handle(self._handle) + self._released = True + + def __del__(self): + """Clean up on destruction.""" + try: + self.release() + except Exception: + pass # Ignore errors during destruction + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - release handle.""" + self.release() + return False + + +class VARegion: + """RAII wrapper for a CUDA virtual address region. + + This class manages the lifecycle of a reserved VA region, + including mapping/unmapping of handles. + + Attributes: + va: The virtual address as integer. + size: Total size of the region in bytes. + device_id: CUDA device ordinal. + """ + + __slots__ = ("_va", "_size", "_device_id", "_granularity", "_mappings", "_released") + + def __init__(self, size: int, device_id: int): + """Reserve a VA region. + + Args: + size: Size in bytes. + device_id: CUDA device ordinal. + """ + self._device_id = device_id + self._granularity = get_allocation_granularity(device_id) + aligned_size = align_up(size, self._granularity) + + self._va = reserve_va(aligned_size, self._granularity) + self._size = aligned_size + self._mappings: list[Tuple[int, int]] = [] # List of (offset, size) mappings + self._released = False + + @property + def va(self) -> int: + """Get the base virtual address.""" + if self._released: + raise RuntimeError("VA region has been released") + return self._va + + @property + def size(self) -> int: + """Get the total size.""" + return self._size + + def map(self, offset: int, size: int, handle: int, handle_offset: int = 0) -> int: + """Map a handle at an offset within this VA region. + + Args: + offset: Offset within this VA region. + size: Size to map. + handle: Memory handle to map. + handle_offset: Offset within the handle. + + Returns: + Virtual address of the mapping. + + Raises: + ValueError: If the mapping would exceed the region bounds. + """ + if offset + size > self._size: + raise ValueError( + f"Mapping at offset={offset} size={size} exceeds region size={self._size}" + ) + + va = self._va + offset + map_handle(va, size, handle, handle_offset) + set_access(va, size, self._device_id) + self._mappings.append((offset, size)) + + return va + + def unmap_all(self) -> None: + """Unmap all mappings in this region.""" + for offset, size in self._mappings: + try: + unmap_va(self._va + offset, size) + except Exception: + pass # Best effort cleanup + self._mappings.clear() + + def release(self) -> None: + """Release the VA region. Idempotent.""" + if not self._released: + self.unmap_all() + free_va(self._va, self._size) + self._released = True + + def __del__(self): + """Clean up on destruction.""" + try: + self.release() + except Exception: + pass # Ignore errors during destruction + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - release region.""" + self.release() + return False diff --git a/tensorrt_llm/_torch/modules/dwdp/weight_buffer.py b/tensorrt_llm/_torch/modules/dwdp/weight_buffer.py new file mode 100644 index 000000000000..bad47c0d8425 --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/weight_buffer.py @@ -0,0 +1,742 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WeightBuffer: Composite VA layout with zero-copy local + page-pool-backed remote double buffer. + +The WeightBuffer is the central component of DWDP weight management. It creates +a composite virtual address space that seamlessly combines: + - Local region: Zero-copy mapping of the MNNVL handle (no D2D copy) + - Remote regions (pre/post): Page pool backed buffers for P2P copy targets + +Key invariant: The local expert region is NEVER the target of a memcpy. It is +always a direct cuMemMap to the MNNVL physical handle. + +NOT a singleton - one instance per DWDP group. However, within a DWDP group +there is exactly one WeightBuffer managing all MoE layers for that group. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch + +# Try to use tensorrt_llm logger if available, otherwise use standard logging +try: + from tensorrt_llm.logger import logger +except ImportError: + logger = logging.getLogger(__name__) + +from .page_pool import PagePool, compute_slot_sizes +from .specs import EdgeInfo, LayerWeightSpecs, MnnvlHandleSet, PageAlignedLayout +from .vmm import ( + free_va, + get_allocation_granularity, + map_handle, + reserve_va, + set_access, + tensor_from_ptr, + unmap_va, +) + + +@dataclass +class LayerBufferState: + """Internal state for a single layer's composite buffer. + + Attributes: + layer_idx: Layer index. + va_base: Base virtual address for the first weight's composite buffer + (kept for legacy compatibility; may be 0 if unused). + va_size: Total size of the first weight's reserved VA region. + layouts: Per-weight PageAlignedLayout. + tensors: Per-weight full tensor views. + remote_slices: Per-weight remote slice info. + mappings: List of (va, size) for all sub-region mappings (for unmap cleanup). + va_regions: List of (va_base, va_size) for all per-weight VA regions + (for free_va cleanup, one entry per weight name). + """ + + layer_idx: int + va_base: int + va_size: int + layouts: Dict[str, PageAlignedLayout] + tensors: Dict[str, torch.Tensor] + remote_slices: Dict[str, List[Tuple[torch.Tensor, int, int]]] + mappings: List[Tuple[int, int]] + va_regions: List[Tuple[int, int]] + + +class WeightBuffer: + """Composite VA layout with zero-copy local + page-pool-backed remote double buffer. + + The WeightBuffer does NOT copy any data. It only sets up VA mappings: + - Local region: cuMemMap to MNNVL handle (zero-copy, no D2D) + - Remote region: cuMemMap to page pool (P2P copy target, written by WeightManager) + + Attributes: + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + dwdp_size: Number of DWDP ranks. + device_id: CUDA device ordinal. + granularity: VMM page granularity in bytes. + """ + + __slots__ = ( + "_layer_weight_specs", + "_handles", + "_local_start", + "_local_end", + "_dwdp_size", + "_device_id", + "_granularity", + "_pool_page_size", + "_page_pool", + "_layer_states", + "_moe_layer_indices", + "_released", + ) + + def __init__( + self, + layer_weight_specs: LayerWeightSpecs, + handles: MnnvlHandleSet, + local_start: int, + local_end: int, + dwdp_size: int, + device_id: int, + granularity: Optional[int] = None, + pool_page_size: Optional[int] = None, + ): + """Internal constructor. Use WeightBuffer.create() factory method. + + Args: + layer_weight_specs: Per-layer weight specifications. + handles: MNNVL handles from Transport. + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + dwdp_size: Number of DWDP ranks. + device_id: CUDA device ordinal. + granularity: VMM page granularity. If None, queries device. + pool_page_size: Pool page handle size in bytes. If None, + defaults to ``PagePool.DEFAULT_PAGE_SIZE_MULTIPLIER * + granularity`` (16MB on GB200). Must be a multiple of + granularity. + """ + self._layer_weight_specs = layer_weight_specs + self._handles = handles + self._local_start = local_start + self._local_end = local_end + self._dwdp_size = dwdp_size + self._device_id = device_id + + if granularity is None: + self._granularity = get_allocation_granularity(device_id) + else: + self._granularity = granularity + + if pool_page_size is None: + self._pool_page_size = PagePool.DEFAULT_PAGE_SIZE_MULTIPLIER * self._granularity + else: + self._pool_page_size = pool_page_size + + self._page_pool: Optional[PagePool] = None + self._layer_states: Dict[int, LayerBufferState] = {} + self._moe_layer_indices: List[int] = sorted(layer_weight_specs.keys()) + self._released = False + + @classmethod + def create( + cls, + layer_weight_specs: LayerWeightSpecs, + handles: MnnvlHandleSet, + local_start: int, + local_end: int, + dwdp_size: int, + device_id: int, + ) -> WeightBuffer: + """Create all VA mappings and remote page pool. + + Steps: + 1. Compute page-aligned layout for each (layer, weight) pair + (num_experts comes from WeightSpec.num_experts per layer) + 2. Allocate page pool for remote double buffer + 3. Create per-layer composite VA mappings + 4. Create tensor views + + Args: + layer_weight_specs: Per-layer weight specifications + (includes num_experts per layer). + handles: MNNVL handles from Transport (local expert data lives here). + local_start: First local expert index (inclusive). + local_end: Last local expert index (exclusive). + dwdp_size: Number of DWDP ranks. + device_id: CUDA device ordinal. + + Returns: + Ready-to-use WeightBuffer with all VA mappings established. + + Raises: + ValueError: If parameters are inconsistent. + """ + # Validate inputs + cls._validate_inputs(layer_weight_specs, handles, local_start, local_end, dwdp_size) + + # Create instance + buffer = cls( + layer_weight_specs=layer_weight_specs, + handles=handles, + local_start=local_start, + local_end=local_end, + dwdp_size=dwdp_size, + device_id=device_id, + ) + + try: + # Step 1: Compute layouts (uses pool_page_size as pool_granularity + # so that pre/post sizes are aligned to the pool page boundary). + layouts = buffer._compute_all_layouts() + + # Step 2: Compute slot sizes and create page pool. + # Because layouts already have pre/post sizes aligned to + # pool_page_size, the slot sizes are automatically aligned too. + buffer_slot_assignments = { + layer_idx: buffer.buffer_index_for_layer(layer_idx) + for layer_idx in layer_weight_specs.keys() + } + slot_sizes = compute_slot_sizes(layouts, buffer_slot_assignments) + + buffer._page_pool = PagePool.create( + slot_sizes, device_id, page_size=buffer._pool_page_size + ) + + # Step 3 & 4: Create composite VA mappings and tensor views + for layer_idx in buffer._moe_layer_indices: + buffer._setup_layer(layer_idx, layouts[layer_idx]) + + logger.info( + f"[WeightBuffer] Created for {len(buffer._moe_layer_indices)} layers, " + f"local experts [{local_start}, {local_end}), DWDP size {dwdp_size}" + ) + + return buffer + + except Exception: + buffer.release() + raise + + @staticmethod + def _validate_inputs( + layer_weight_specs: LayerWeightSpecs, + handles: MnnvlHandleSet, + local_start: int, + local_end: int, + dwdp_size: int, + ) -> None: + """Validate constructor inputs.""" + if local_start < 0: + raise ValueError(f"local_start must be non-negative, got {local_start}") + if local_end <= local_start: + raise ValueError( + f"local_end must be greater than local_start, got {local_end} <= {local_start}" + ) + if dwdp_size <= 0: + raise ValueError(f"dwdp_size must be positive, got {dwdp_size}") + + # Verify that every (layer_idx, name) in specs has a matching handle + for layer_idx, weight_specs in layer_weight_specs.items(): + for name in weight_specs.keys(): + key = (layer_idx, name) + if key not in handles.handles: + raise ValueError(f"Missing handle for {key}") + + def _compute_all_layouts(self) -> Dict[int, Dict[str, PageAlignedLayout]]: + """Compute page-aligned layouts for all (layer, weight) pairs. + + Uses ``_pool_page_size`` as the ``pool_granularity`` so that pre/post + region sizes are aligned to the pool page boundary. This ensures + each pool-backed region is an exact number of pool pages, eliminating + cuMemMap overlap between pool handles and the MNNVL handle. + """ + layouts: Dict[int, Dict[str, PageAlignedLayout]] = {} + + for layer_idx, weight_specs in self._layer_weight_specs.items(): + layouts[layer_idx] = {} + + for name, spec in weight_specs.items(): + handle_size = self._handles.get_size(layer_idx, name) + + layout = PageAlignedLayout.compute( + expert_bytes=spec.expert_bytes, + num_experts=spec.num_experts, + local_start=self._local_start, + local_end=self._local_end, + granularity=self._granularity, + handle_phys_size=handle_size, + pool_granularity=self._pool_page_size, + ) + layouts[layer_idx][name] = layout + + return layouts + + def _setup_layer( + self, + layer_idx: int, + weight_layouts: Dict[str, PageAlignedLayout], + ) -> None: + """Set up composite VA tensor views for a single layer. + + Composite VA layout per weight: + + [ pre_region | mnnvl_region | post_region ] + (page pool) (MNNVL handle) (page pool) + + - pre_region : page-pool fabric pages for remote experts [0, local_start) + - mnnvl_region: zero-copy mapping of the MNNVL fabric handle + (contains local experts [local_start, local_end)) + - post_region : page-pool fabric pages for remote experts [local_end, N) + + Routing table budget (GB200): + ~928 total entries + - 696 used by Transport (3 peers × 4 weights × 58 layers, persistent) + - 232 used here (1 entry per weight per layer, persistent for lifetime) + = exactly 928 = 0 remaining. + + We achieve 1 routing entry per weight by calling set_access ONCE on the + full composite VA (va_base, total_size). Calling it three times + (once per sub-region) would consume 696 entries here, exhausting the table. + """ + weight_specs = self._layer_weight_specs[layer_idx] + buf_slot = self.buffer_index_for_layer(layer_idx) + + layer_state = LayerBufferState( + layer_idx=layer_idx, + va_base=0, + va_size=0, + layouts=weight_layouts, + tensors={}, + remote_slices={}, + mappings=[], + va_regions=[], + ) + + # Track page pool offsets within the slot — each weight name allocates + # its own pages sequentially within the slot. + page_pool_offset = 0 + + for name, layout in weight_layouts.items(): + spec = weight_specs[name] + handle = self._handles.get_handle(layer_idx, name) + + # Reserve one contiguous VA for this weight's full composite buffer. + va_base = reserve_va(layout.total_size, self._granularity) + all_mappings: List[Tuple[int, int]] = [] + + try: + # --- Map pre-region (page pool pages for remote experts before local) --- + if layout.pre_size > 0: + pre_mappings = self._page_pool.map_pages( + slot=buf_slot, + va_start=va_base, + size=layout.pre_size, + page_offset=page_pool_offset, + ) + all_mappings.extend(pre_mappings) + page_pool_offset += layout.pre_pages + + # --- Map MNNVL region (zero-copy local expert data) --- + mnnvl_va = va_base + layout.pre_size + map_handle(mnnvl_va, layout.mnnvl_size, handle, offset=0) + all_mappings.append((mnnvl_va, layout.mnnvl_size)) + + # --- Map post-region (page pool pages for remote experts after local) --- + if layout.post_size > 0: + post_va = mnnvl_va + layout.mnnvl_size + post_mappings = self._page_pool.map_pages( + slot=buf_slot, + va_start=post_va, + size=layout.post_size, + page_offset=page_pool_offset, + ) + all_mappings.extend(post_mappings) + page_pool_offset += layout.post_pages + + # --- Single set_access for the ENTIRE composite VA --- + # One cuMemSetAccess call = 1 routing table entry consumed. + # This is critical: calling it per sub-region would consume 3 + # entries per weight per layer (696 total) and exhaust the + # routing table on top of Transport's 696 persistent entries. + set_access(va_base, layout.total_size, self._device_id) + + # --- Create full [num_experts, ...] tensor view --- + # The VA layout is: + # va_base -> start of pre-region (pool) + # va_base + pre_size -> start of mnnvl (local data) + # va_base + pre_size + mnnvl_size -> start of post-region (pool) + # + # When pool_granularity > granularity, pre_size may be larger + # than page_start due to rounding. The difference is + # pre_padding bytes of unused pool-backed space at the end of + # the pre-region. The tensor view must start at + # va_base + pre_padding so that expert index 0 maps to the + # correct physical memory: + # expert[0] @ va_base + pre_padding + # expert[local_start] @ va_base + pre_size + data_offset + # = va_base + pre_padding + page_start + data_offset + # = va_base + pre_padding + local_start_bytes + tensor_start = va_base + layout.pre_padding + full_tensor = tensor_from_ptr( + ptr=tensor_start, + shape=spec.full_shape, + dtype=spec.dtype, + device_id=self._device_id, + ) + + except Exception: + # Best-effort cleanup on error + for va, size in all_mappings: + try: + unmap_va(va, size) + except Exception: + pass + try: + free_va(va_base, layout.total_size) + except Exception: + pass + raise + + # Accumulate into layer_state for release() cleanup: + # - mappings: all (va, size) sub-region mappings (for unmap_va) + # - va_regions: all (va_base, total_size) per-weight VA (for free_va) + layer_state.mappings.extend(all_mappings) + layer_state.va_regions.append((va_base, layout.total_size)) + + # Store the full tensor and compute remote slices. + layer_state.tensors[name] = full_tensor + remote_slices = self._compute_remote_slices(full_tensor, layout, spec.num_experts) + layer_state.remote_slices[name] = remote_slices + + self._layer_states[layer_idx] = layer_state + + def _create_tensor_view( + self, + va: int, + shape: Tuple[int, ...], + dtype: torch.dtype, + ) -> torch.Tensor: + """Create a tensor view over a VA region.""" + return tensor_from_ptr(va, shape, dtype, self._device_id) + + def _compute_remote_slices( + self, + full_tensor: torch.Tensor, + layout: PageAlignedLayout, + num_experts: int, + ) -> List[Tuple[torch.Tensor, int, int]]: + """Compute remote slices for P2P copy destinations. + + Returns: + List of (slice_tensor, expert_start, expert_end) for each remote region. + """ + slices = [] + + # Pre-remote region: experts [0, local_start) + if self._local_start > 0: + pre_slice = full_tensor[: self._local_start] + slices.append((pre_slice, 0, self._local_start)) + + # Post-remote region: experts [local_end, num_experts) + if self._local_end < num_experts: + post_slice = full_tensor[self._local_end :] + slices.append((post_slice, self._local_end, num_experts)) + + return slices + + # --- Tensor access (used by WeightManager at runtime) --- + + def get_full_tensor(self, layer_idx: int, name: str) -> torch.Tensor: + """Full [num_experts, ...] tensor view over composite VA. + + Local region [local_start:local_end] is zero-copy MNNVL. + Remote regions are backed by page pool (P2P copy target). + + Args: + layer_idx: Layer index. + name: Weight name. + + Returns: + Full tensor view. + + Raises: + KeyError: If layer/weight not found. + RuntimeError: If buffer has been released. + """ + if self._released: + raise RuntimeError("WeightBuffer has been released") + if layer_idx not in self._layer_states: + raise KeyError(f"Layer {layer_idx} not found") + if name not in self._layer_states[layer_idx].tensors: + raise KeyError(f"Weight '{name}' not found in layer {layer_idx}") + return self._layer_states[layer_idx].tensors[name] + + def get_remote_slices(self, layer_idx: int, name: str) -> List[Tuple[torch.Tensor, int, int]]: + """Get (slice_tensor, expert_start, expert_end) for each remote region. + + These are P2P copy destinations. WeightManager copies peer data here. + + Args: + layer_idx: Layer index. + name: Weight name. + + Returns: + List of (tensor_slice, start_idx, end_idx) for remote regions. + + Raises: + KeyError: If layer/weight not found. + RuntimeError: If buffer has been released. + """ + if self._released: + raise RuntimeError("WeightBuffer has been released") + if layer_idx not in self._layer_states: + raise KeyError(f"Layer {layer_idx} not found") + if name not in self._layer_states[layer_idx].remote_slices: + raise KeyError(f"Weight '{name}' not found in layer {layer_idx}") + return self._layer_states[layer_idx].remote_slices[name] + + # --- Edge info (used by orchestrator for one-time setup) --- + + def get_edge_info(self, layer_idx: int, name: str) -> EdgeInfo: + """Page-alignment edge info for setup-time peer data fill. + + Args: + layer_idx: Layer index. + name: Weight name. + + Returns: + EdgeInfo with data_offset, leading_edge, trailing_edge. + + Raises: + KeyError: If layer/weight not found. + """ + return self.get_layout(layer_idx, name).get_edge_info() + + def get_data_offset(self, layer_idx: int, name: str) -> int: + """Byte offset where local data starts within MNNVL handle. + + Args: + layer_idx: Layer index. + name: Weight name. + + Returns: + Data offset in bytes. + """ + return self.get_layout(layer_idx, name).data_offset + + def compute_peer_data_offset(self, layer_idx: int, name: str, peer_local_start: int) -> int: + """Compute data_offset for a peer rank given their local_start. + + layer_idx is required because different layers may have + different expert_bytes, affecting page alignment. + + Args: + layer_idx: Layer index. + name: Weight name. + peer_local_start: The peer rank's local_start expert index. + + Returns: + Data offset in bytes for the peer. + """ + layout = self.get_layout(layer_idx, name) + peer_start_bytes = peer_local_start * layout.expert_bytes + peer_page_start = (peer_start_bytes // self._granularity) * self._granularity + return peer_start_bytes - peer_page_start + + # --- Layout queries (diagnostics, tests) --- + + def get_layout(self, layer_idx: int, name: str) -> PageAlignedLayout: + """Page-aligned layout for a specific (layer, weight) pair. + + Args: + layer_idx: Layer index. + name: Weight name. + + Returns: + PageAlignedLayout for this weight. + + Raises: + KeyError: If layer/weight not found. + """ + if layer_idx not in self._layer_states: + raise KeyError(f"Layer {layer_idx} not found") + if name not in self._layer_states[layer_idx].layouts: + raise KeyError(f"Weight '{name}' not found in layer {layer_idx}") + return self._layer_states[layer_idx].layouts[name] + + @property + def granularity(self) -> int: + """CUDA VMM page granularity in bytes (typically 2MB on GB200).""" + return self._granularity + + @property + def pool_page_size(self) -> int: + """Pool page handle size in bytes (typically 16MB on GB200).""" + return self._pool_page_size + + @property + def layer_indices(self) -> List[int]: + """MoE layer indices managed by this buffer.""" + return self._moe_layer_indices.copy() + + @property + def local_start(self) -> int: + """First local expert index (inclusive).""" + return self._local_start + + @property + def local_end(self) -> int: + """Last local expert index (exclusive).""" + return self._local_end + + @property + def dwdp_size(self) -> int: + """Number of DWDP ranks.""" + return self._dwdp_size + + @property + def device_id(self) -> int: + """CUDA device ordinal.""" + return self._device_id + + def buffer_index_for_layer(self, layer_idx: int) -> int: + """Double buffer slot: 0 or 1 based on MoE order (odd/even). + + Args: + layer_idx: Layer index. + + Returns: + Buffer slot index (0 or 1). + """ + # Find position of this layer in the MoE layer sequence + if layer_idx in self._moe_layer_indices: + moe_order = self._moe_layer_indices.index(layer_idx) + else: + moe_order = layer_idx + + return moe_order % 2 + + def weight_names(self, layer_idx: int) -> List[str]: + """Get weight names for a specific layer. + + Args: + layer_idx: Layer index. + + Returns: + List of weight names. + """ + if layer_idx not in self._layer_weight_specs: + raise KeyError(f"Layer {layer_idx} not found") + return list(self._layer_weight_specs[layer_idx].keys()) + + # --- Lifecycle --- + + def release(self) -> None: + """Release all CUDA VMM resources. Idempotent. Safe from __del__.""" + if self._released: + return + + self._released = True + + # Unmap and free all VA regions + for layer_idx, state in self._layer_states.items(): + # First unmap all sub-region mappings (must precede free_va) + for va, size in state.mappings: + try: + unmap_va(va, size) + except Exception as e: + logger.warning(f"[WeightBuffer] Failed to unmap layer {layer_idx}: {e}") + + # Free each per-weight VA region + for va_base, va_size in state.va_regions: + try: + free_va(va_base, va_size) + except Exception as e: + logger.warning( + f"[WeightBuffer] Failed to free VA {va_base:#x} for layer {layer_idx}: {e}" + ) + + # Legacy fallback: free va_base/va_size if va_regions is empty + # (handles old-style single-VA LayerBufferState from older code paths) + if not state.va_regions and state.va_base != 0: + try: + free_va(state.va_base, state.va_size) + except Exception as e: + logger.warning(f"[WeightBuffer] Failed to free VA for layer {layer_idx}: {e}") + + self._layer_states.clear() + + # Release page pool + if self._page_pool is not None: + self._page_pool.release() + self._page_pool = None + + logger.debug("[WeightBuffer] Released all resources") + + def __del__(self): + """Clean up on destruction.""" + try: + self.release() + except Exception: + pass + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.release() + return False + + # --- Debug utilities --- + + def debug_info(self) -> Dict: + """Get debug information about the buffer state. + + Returns: + Dictionary with debug information. + """ + info = { + "local_range": (self._local_start, self._local_end), + "dwdp_size": self._dwdp_size, + "device_id": self._device_id, + "granularity": self._granularity, + "pool_page_size": self._pool_page_size, + "num_layers": len(self._moe_layer_indices), + "layer_indices": self._moe_layer_indices, + "released": self._released, + } + + if self._page_pool is not None: + info["page_pool"] = { + "page_size": self._page_pool.page_size, + "slot0_pages": self._page_pool.num_pages(0), + "slot1_pages": self._page_pool.num_pages(1), + "slot0_size": self._page_pool.slot_size(0), + "slot1_size": self._page_pool.slot_size(1), + } + + return info diff --git a/tensorrt_llm/_torch/modules/dwdp/weight_manager.py b/tensorrt_llm/_torch/modules/dwdp/weight_manager.py new file mode 100644 index 000000000000..b559de70523a --- /dev/null +++ b/tensorrt_llm/_torch/modules/dwdp/weight_manager.py @@ -0,0 +1,519 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DWDPWeightManager: Runtime P2P prefetch and weight binding orchestrator. + +The WeightManager is the runtime scheduler for DWDP expert weight prefetch. +It runs INDEPENDENTLY per instance (no cross-instance sync at runtime) and +manages: + - Async P2P copies from peer MNNVL handles into WeightBuffer's remote slices + - Bidirectional event sync (prefetch_events, consume_events) with zero CPU blocking + - param.data binding for MoE forward pass + +Design invariants: + - NEVER calls torch.cuda.synchronize() — all sync via stream.wait_event() + - NEVER allocates/frees memory during inference — all buffers pre-allocated + - Only copies remote experts — local experts are zero-copy MNNVL + - Double buffer with bidirectional events prevents RAW and WAR hazards + +Timeline visualization: + compute_stream: [forward L3] --wait(prefetch[0])--> [forward L4] --wait(prefetch[1])--> [forward L5] + copy_stream: --wait(consume[0])--> [P2P L4] --wait(consume[1])--> [P2P L5] +""" + +from __future__ import annotations + +import bisect +import logging +from typing import Dict, List, Optional, Tuple + +import torch + +# Try to use tensorrt_llm logger if available, otherwise use standard logging +try: + from tensorrt_llm.logger import logger +except ImportError: + logger = logging.getLogger(__name__) + +from .specs import PeerRanges, lookup_owner +from .weight_buffer import WeightBuffer + +# Slice granularity for the batched prefetch path: 2 MiB. Each peer's +# contribution to a layer is split into chunks of this size and the chunks +# are interleaved across peers in a single ``cudaMemcpyBatchAsync`` call +# so that any moment in flight touches multiple NVLink links, limiting +# single-link contention. Ported from the IPC-era DWDP contention +# optimization (PR #12974) to the VA-based pipeline. +MEMCPY_BATCH_SLICE_BYTES = 1 << 21 + + +class DWDPWeightManager: + """Runtime P2P copy orchestrator for DWDP expert weight prefetch. + + Manages async P2P copies from peer MNNVL handles into WeightBuffer's + remote slices using a double-buffered event protocol. All synchronization + is done via CUDA events and stream.wait_event() — the CPU is never blocked. + + Attributes: + weight_buffer: WeightBuffer with composite VA layout for all MoE layers. + dwdp_rank: This instance's DWDP rank (0..dwdp_size-1). + dwdp_size: Total number of DWDP ranks. + """ + + __slots__ = ( + "_weight_buffer", + "_peer_views", + "_peer_ranges", + "_moe_layer_indices", + "_moe_layer_set", + "_weight_names", + "_dwdp_rank", + "_dwdp_size", + "_experts_per_rank", + "_copy_stream", + "_prefetch_events", + "_consume_events", + "_transport", + "_use_batched_prefetch", + "_batched_copy_plans", + ) + + def __init__( + self, + weight_buffer: WeightBuffer, + peer_views: Dict[Tuple[int, int, str], torch.Tensor], + peer_ranges: PeerRanges, + moe_layer_indices: List[int], + weight_names: List[str], + dwdp_rank: int, + dwdp_size: int, + contention_opt: bool = False, + ) -> None: + """Initialize the DWDPWeightManager. + + Creates a dedicated copy stream and pre-allocates all CUDA events for + the double-buffer protocol. Both consume_events are recorded immediately + so the first prefetch does not stall waiting for a never-recorded event. + + Args: + weight_buffer: WeightBuffer with composite VA layout for all MoE layers. + peer_views: Mapping of (peer_rank, layer_idx, weight_name) to tensor + views into peer's MNNVL handle. These are IMMUTABLE — the source + GPU never writes to them during inference. + peer_ranges: Per-peer ``(local_start, local_end_capped)`` tuples + indexed by DWDP rank. ``prefetch_layer`` resolves the owner of + a remote expert id with ``lookup_owner(expert_id, peer_ranges)``, + which handles non-uniform partition (tail-rank padding) and + redundancy (overlapping ranges) uniformly. + moe_layer_indices: Sorted list of decoder layer indices that are MoE + layers (e.g., layers 3..60 in a typical MoE model where the first + few layers are dense). + weight_names: Weight parameter names to prefetch + (e.g., ["gate_up_proj", "down_proj"]). + dwdp_rank: This instance's DWDP rank (0..dwdp_size-1). + dwdp_size: Total number of DWDP ranks. + contention_opt: If True, ``prefetch_layer`` issues a single + ``cudaMemcpyBatchAsync`` per layer with sub-slices interleaved + across peers in ``MEMCPY_BATCH_SLICE_BYTES`` (2 MiB) chunks + to reduce single-NVLink-link contention. Plans are cached + per layer since peer ``data_ptr()`` and destination + ``data_ptr()`` are stable after initialization. + + Raises: + ValueError: If dwdp_rank is out of range or dwdp_size is invalid. + """ + if dwdp_size <= 0: + raise ValueError(f"dwdp_size must be positive, got {dwdp_size}") + if dwdp_rank < 0 or dwdp_rank >= dwdp_size: + raise ValueError(f"dwdp_rank must be in [0, {dwdp_size}), got {dwdp_rank}") + + self._weight_buffer = weight_buffer + self._peer_views = peer_views + self._peer_ranges = peer_ranges + self._moe_layer_indices = sorted(moe_layer_indices) + self._moe_layer_set = set(self._moe_layer_indices) + self._weight_names = list(weight_names) + self._dwdp_rank = dwdp_rank + self._dwdp_size = dwdp_size + + # Storage chunk size of every peer's MNNVL handle. Uniform across + # ranks (Phase 1 made ``num_experts_per_worker`` the storage size for + # every rank) — the variability lives in the *valid* sub-range + # tracked by ``_peer_ranges``. + self._experts_per_rank = weight_buffer.local_end - weight_buffer.local_start + + # Dedicated CUDA stream for P2P copy operations. + # Using a separate stream allows copy/compute overlap. + device = torch.device("cuda", weight_buffer.device_id) + self._copy_stream = torch.cuda.Stream(device=device) + + # Double-buffer events: one per buffer slot (0 and 1). + # prefetch_events[i]: recorded on copy_stream after P2P copy into slot i. + # Waited on by compute_stream (RAW — must finish copy before read). + # consume_events[i]: recorded on compute_stream after forward using slot i. + # Waited on by copy_stream (WAR — must finish read before overwrite). + self._prefetch_events: List[torch.cuda.Event] = [torch.cuda.Event() for _ in range(2)] + self._consume_events: List[torch.cuda.Event] = [torch.cuda.Event() for _ in range(2)] + + # Initialize consume_events as "already signaled" so the first prefetch + # does not stall. We record them on the current (default) stream, which + # establishes them as completed from the copy_stream's perspective. + current_stream = torch.cuda.current_stream(device) + for event in self._consume_events: + event.record(current_stream) + + # Batched contention-optimized prefetch. Plans are stable after + # initialization (peer data pointers + destination slice pointers + # do not move during inference) so we cache one ``(dst_ptrs, + # src_ptrs, sizes)`` tuple per layer to avoid host-side rebuild on + # every prefetch. + self._use_batched_prefetch = contention_opt + self._batched_copy_plans: Dict[int, Tuple[List[int], List[int], List[int]]] = {} + + logger.info( + f"[DWDPWeightManager] Initialized: rank={dwdp_rank}/{dwdp_size}, " + f"moe_layers={len(self._moe_layer_indices)}, " + f"weights={self._weight_names}, " + f"experts_per_rank={self._experts_per_rank}, " + f"contention_opt={contention_opt}" + ) + + @property + def weight_buffer(self) -> WeightBuffer: + """The underlying WeightBuffer.""" + return self._weight_buffer + + @property + def dwdp_rank(self) -> int: + """This instance's DWDP rank.""" + return self._dwdp_rank + + @property + def dwdp_size(self) -> int: + """Total number of DWDP ranks.""" + return self._dwdp_size + + def is_moe_layer(self, layer_idx: int) -> bool: + """Check if a given layer index is a MoE layer. + + Args: + layer_idx: Decoder layer index to check. + + Returns: + True if layer_idx is a MoE layer managed by this instance. + """ + return layer_idx in self._moe_layer_set + + def next_moe_layer(self, layer_idx: int) -> Optional[int]: + """Return the next MoE layer index after layer_idx. + + Uses binary search for O(log n) lookup. + + Args: + layer_idx: Current layer index. + + Returns: + The next MoE layer index strictly greater than layer_idx, + or None if layer_idx is at or past the last MoE layer. + """ + if not self._moe_layer_indices: + return None + pos = bisect.bisect_right(self._moe_layer_indices, layer_idx) + if pos < len(self._moe_layer_indices): + return self._moe_layer_indices[pos] + return None + + def first_moe_layer(self) -> int: + """Return the first MoE layer index. + + Returns: + The smallest MoE layer index. + + Raises: + IndexError: If there are no MoE layers. + """ + if not self._moe_layer_indices: + raise IndexError("No MoE layers configured") + return self._moe_layer_indices[0] + + def prefetch_layer(self, layer_idx: int) -> None: + """Kick off async P2P copies for all remote expert slices of a layer. + + Switches to the copy_stream, waits for the consume_event on the target + buffer slot (WAR hazard — ensures compute has finished reading the slot), + then issues P2P copies from peer MNNVL views into the WeightBuffer's + remote slices. Finally, records a prefetch_event so the compute stream + can wait for the copy to complete. + + Two implementations dispatch on ``self._use_batched_prefetch`` (set + from ``DwdpConfig.contention_opt``): the default per-slice path + issues one ``torch.Tensor.copy_`` per peer chunk, while the batched + path collapses all chunks into a single ``cudaMemcpyBatchAsync`` + call whose sub-slices round-robin across peers in 2 MiB chunks to + limit single-NVLink-link contention. + + This method returns immediately — all GPU work is enqueued on + copy_stream and does not block the CPU. + + Args: + layer_idx: MoE layer index to prefetch. Must be in moe_layer_indices. + + Raises: + KeyError: If layer_idx is not a valid MoE layer. + """ + if layer_idx not in self._moe_layer_set: + raise KeyError( + f"Layer {layer_idx} is not a MoE layer. Valid layers: {self._moe_layer_indices}" + ) + + buf_idx = self._weight_buffer.buffer_index_for_layer(layer_idx) + + with torch.cuda.stream(self._copy_stream): + # WAR hazard: wait until compute finishes reading this buffer slot + # before overwriting it with new P2P data. + self._copy_stream.wait_event(self._consume_events[buf_idx]) + + if self._use_batched_prefetch: + self._prefetch_layer_batched(layer_idx) + else: + self._prefetch_layer_per_slice(layer_idx) + + # RAW signal: record event so compute_stream knows copy is done. + self._prefetch_events[buf_idx].record(self._copy_stream) + + logger.debug( + f"[DWDPWeightManager] Prefetch enqueued: layer={layer_idx}, buf_slot={buf_idx}" + ) + + def _prefetch_layer_per_slice(self, layer_idx: int) -> None: + """Default prefetch: one ``torch.Tensor.copy_`` per peer chunk.""" + for name in self._weight_names: + remote_slices = self._weight_buffer.get_remote_slices(layer_idx, name) + for dst_slice, expert_start, expert_end in remote_slices: + # A remote slice may span multiple peer ranks. Walk + # the destination range and dispatch each contiguous + # sub-chunk to the peer that owns it. ``lookup_owner`` + # picks the lowest-rank owner (matters under redundancy + # where multiple peers cover the same expert id). + cursor = expert_start + dst_offset = 0 + while cursor < expert_end: + peer_rank = lookup_owner(cursor, self._peer_ranges) + peer_local_start, peer_local_end = self._peer_ranges[peer_rank] + local_offset = cursor - peer_local_start + # Stop at the first of: end of the destination + # remote slice, or end of this peer's valid range. + # Crossing the latter means the next expert is + # owned by a different peer (or by *us*, which + # would be a bug since this is the *remote* slice). + chunk_end = min(expert_end, peer_local_end) + n = chunk_end - cursor + + peer_key = (peer_rank, layer_idx, name) + src_tensor = self._peer_views[peer_key] + dst_slice[dst_offset : dst_offset + n].copy_( + src_tensor[local_offset : local_offset + n] + ) + dst_offset += n + cursor = chunk_end + + def _build_batched_prefetch_copy_plan( + self, layer_idx: int + ) -> Tuple[List[int], List[int], List[int]]: + """Build a per-layer ``cudaMemcpyBatchAsync`` plan. + + Strategy: collect every ``(dst_ptr, src_ptr, size_bytes)`` triple + from the same per-slice / per-peer walk used by + ``_prefetch_layer_per_slice``, then split each peer's contiguous + chunks into ``MEMCPY_BATCH_SLICE_BYTES`` (2 MiB) sub-slices and + interleave them across peers in round-robin order. Interleaving + is what limits single-NVLink-link contention — at any moment the + in-flight memcpys touch every peer link rather than saturating one. + + Returns: + Three parallel lists ``(dst_ptrs, src_ptrs, sizes)`` of the + same length, suitable for ``cudaMemcpyBatchAsync``. + """ + # Step 1: collect per-peer ``(dst_ptr, src_ptr, total_size)`` triples. + peer_triples: Dict[int, List[Tuple[int, int, int]]] = {} + for name in self._weight_names: + remote_slices = self._weight_buffer.get_remote_slices(layer_idx, name) + for dst_slice, expert_start, expert_end in remote_slices: + cursor = expert_start + dst_offset = 0 + while cursor < expert_end: + peer_rank = lookup_owner(cursor, self._peer_ranges) + peer_local_start, peer_local_end = self._peer_ranges[peer_rank] + local_offset = cursor - peer_local_start + chunk_end = min(expert_end, peer_local_end) + n = chunk_end - cursor + + peer_key = (peer_rank, layer_idx, name) + src_tensor = self._peer_views[peer_key] + dst_sub = dst_slice[dst_offset : dst_offset + n] + src_sub = src_tensor[local_offset : local_offset + n] + size_bytes = dst_sub.numel() * dst_sub.element_size() + + peer_triples.setdefault(peer_rank, []).append( + (dst_sub.data_ptr(), src_sub.data_ptr(), size_bytes) + ) + + dst_offset += n + cursor = chunk_end + + # Step 2: split each per-peer triple into 2 MiB sub-slices. + peer_subslices: Dict[int, List[Tuple[int, int, int]]] = {} + for peer, triples in peer_triples.items(): + sub_list: List[Tuple[int, int, int]] = [] + for dst_ptr, src_ptr, total in triples: + offset = 0 + while offset < total: + sz = min(MEMCPY_BATCH_SLICE_BYTES, total - offset) + sub_list.append((dst_ptr + offset, src_ptr + offset, sz)) + offset += sz + peer_subslices[peer] = sub_list + + # Step 3: round-robin across peers. At each round, take the next + # sub-slice from every peer that still has one, then advance. + # Peers naturally drop out as their sub-slice lists exhaust. + dst_ptrs: List[int] = [] + src_ptrs: List[int] = [] + sizes: List[int] = [] + cursors = {peer: 0 for peer in peer_subslices} + while cursors: + for peer in list(cursors): + idx = cursors[peer] + if idx < len(peer_subslices[peer]): + d, s, sz = peer_subslices[peer][idx] + dst_ptrs.append(d) + src_ptrs.append(s) + sizes.append(sz) + cursors[peer] = idx + 1 + else: + del cursors[peer] + + return dst_ptrs, src_ptrs, sizes + + def _prefetch_layer_batched(self, layer_idx: int) -> None: + """Batched prefetch: one ``cudaMemcpyBatchAsync`` per layer.""" + plan = self._batched_copy_plans.get(layer_idx) + if plan is None: + plan = self._build_batched_prefetch_copy_plan(layer_idx) + self._batched_copy_plans[layer_idx] = plan + + dst_ptrs, src_ptrs, sizes = plan + if not dst_ptrs: + return + + # Imported lazily because ``cuda.bindings.runtime`` is a heavy + # binding and the default per-slice path does not need it. + from cuda.bindings import runtime as cudart + + attr = cudart.cudaMemcpyAttributes() + attr.srcAccessOrder = cudart.cudaMemcpySrcAccessOrder.cudaMemcpySrcAccessOrderStream + (err,) = cudart.cudaMemcpyBatchAsync( + dst_ptrs, + src_ptrs, + sizes, + len(dst_ptrs), + [attr], + [0], + 1, + self._copy_stream.cuda_stream, + ) + if err != cudart.cudaError_t.cudaSuccess: + raise RuntimeError( + f"cudaMemcpyBatchAsync failed for DWDP prefetch (layer=" + f"{layer_idx}, copies={len(dst_ptrs)}): {err}" + ) + + def wait_and_bind(self, backend_module: torch.nn.Module, layer_idx: int) -> None: + """Wait for prefetch completion and bind weight tensors to the module. + + On the compute stream: + 1. Wait for prefetch_events[buf_idx] (RAW — copy must finish before read). + 2. Bind each weight's full tensor (local zero-copy + remote P2P data) + as the param.data of the backend module's corresponding attribute. + 3. Record consume_events[other_buf] (WAR signal for the OTHER buffer + slot, telling copy_stream it is safe to overwrite that slot). + + The WAR event is recorded for the OTHER buffer slot because by the time + this forward pass completes, the copy_stream may already be writing to + the alternate slot for the next-next layer's prefetch. + + Design rationale — implicit compute-done signal via stream in-order semantics: + The IPC-era predecessor of this code carried per-layer compute_events + (O(N_moe), ~116 events for DSv3 with 58 MoE layers and ping-pong slots) + to signal "kernel(L) finished, slot is reusable". This implementation + replaces them with per-slot consume_events recorded inside this very + method (4 events total, independent of N_moe). + + The simplification relies on CUDA stream in-order semantics: an event + recorded on a stream fires only after every prior enqueue on that + stream has completed. Because step 3 above (record consume_events[ + other_buf]) is enqueued on compute_stream AFTER step 2's binding — + which is in turn ordered before the next kernel(L) launch on the same + stream — the consume event for slot S cannot fire until every kernel + that read slot S has finished. This gives the same WAR ordering as + an explicit per-layer compute_event would, with O(1) bookkeeping. + + Implication for future work: if num_buffers ever grows beyond 2 (e.g. + to support cross-node prefetch with deeper pipelines), per-slot events + may need to become per-slot event LISTS, and the "next kernel is + enqueued on the same stream" invariant must be preserved. + + Args: + backend_module: The MoE backend module whose weight parameters will + be rebound. Must have attributes matching self._weight_names + (e.g., backend_module.gate_up_proj, backend_module.down_proj). + layer_idx: MoE layer index whose weights to bind. + + Raises: + KeyError: If layer_idx is not a valid MoE layer. + AttributeError: If backend_module lacks a required weight attribute. + """ + if layer_idx not in self._moe_layer_set: + raise KeyError( + f"Layer {layer_idx} is not a MoE layer. Valid layers: {self._moe_layer_indices}" + ) + + buf_idx = self._weight_buffer.buffer_index_for_layer(layer_idx) + other_buf = 1 - buf_idx + + compute_stream = torch.cuda.current_stream( + torch.device("cuda", self._weight_buffer.device_id) + ) + + # RAW hazard: wait until P2P copy into this slot is complete + # before the forward pass reads the data. + compute_stream.wait_event(self._prefetch_events[buf_idx]) + + # Bind full [num_experts, ...] tensors to the backend module's parameters. + # The tensor is a composite view: local region is zero-copy MNNVL, + # remote regions contain the freshly P2P-copied expert weights. + for name in self._weight_names: + full_tensor = self._weight_buffer.get_full_tensor(layer_idx, name) + param = getattr(backend_module, name) + param.data = full_tensor + + # WAR signal for the OTHER buffer slot: after this forward pass + # finishes consuming the current slot, the copy_stream is free to + # overwrite the other slot for the next-next layer's prefetch. + self._consume_events[other_buf].record(compute_stream) + + logger.debug( + f"[DWDPWeightManager] Bound weights: layer={layer_idx}, " + f"buf_slot={buf_idx}, signal_consume_slot={other_buf}" + ) diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index ea733543424a..2839e58a6faf 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -190,6 +190,16 @@ def __init__( **kwargs, ) + # ========== Optional DWDP integration ========== + # Must run BEFORE _create_comm_strategy_auto so the factory can skip + # alltoall strategies for DWDP (VA path swaps param.data; the backend + # reads from its own weight attrs, no comm strategy needed). + self.dwdp_manager = get_global_dwdp_manager() + self.enable_dwdp = False + if self.dwdp_manager is not None and self._should_enable_dwdp(): + self.enable_dwdp = True + self.dwdp_manager.add_layer(layer_idx=self.layer_idx) + # ========== Create Communication Strategy ========== self.comm = self._create_comm_strategy_auto() @@ -216,19 +226,6 @@ def __init__( # Validate configuration self.validate_config() - # ========== Optional DWDP integration ========== - self.dwdp_manager = get_global_dwdp_manager() - self.dwdp_handle_collector = None - self.dwdp_rank = None - self.enable_dwdp = False - if self.dwdp_manager is not None and self._should_enable_dwdp(): - self.enable_dwdp = True - self.dwdp_handle_collector = self.dwdp_manager.add_layer( - layer_idx=self.layer_idx, - ) - self.dwdp_rank = self.dwdp_manager.dwdp_rank - self.backend.dwdp_handle_collector = self.dwdp_handle_collector - # Mark as _weights_removed to skip ConfigurableMoE's post_load_weights in model_loader # The backend's post_load_weights will be called directly by model_loader # This avoids duplicate post_load_weights calls (once for ConfigurableMoE, once for backend) @@ -503,7 +500,7 @@ def __enter__(self): def __exit__(self, *exc_info): self.destroy() - def _create_comm_strategy_auto(self) -> Communication: + def _create_comm_strategy_auto(self) -> Optional[Communication]: """ Auto-create the best communication strategy based on hardware and configuration @@ -512,9 +509,15 @@ def _create_comm_strategy_auto(self) -> Communication: host-side comm entirely; layering Communication.dispatch / combine on top of the fused exchange would double-count traffic and break the in-kernel NVLink barrier semantics. + + DWDP VA path: returns None — there is no expert parallelism from the + backend's perspective (fixup_moe_backends sets ep_size=1 / slot_start=0 + so the kernel sees full weights via param.data pointer swap). """ if self.backend.scheduler_kind == MoESchedulerKind.FUSED_COMM: return None + if self.enable_dwdp: + return None return CommunicationFactory.create_strategy( model_config=self.model_config, num_experts=self.num_experts, @@ -559,6 +562,13 @@ def forward_impl( else: output_dtype = x.dtype + # DWDP: wait for prefetch to complete and swap backend weight param.data + # to the composite (full-experts) tensor. Per-layer, not per-chunk — + # owned at the wrapper so the scheduler does not run it twice (e.g. + # external-comm may enter via single- or multi-chunk paths). + if self.enable_dwdp: + self.dwdp_manager.wait_and_bind(self.backend, self.layer_idx) + outputs = self.scheduler.forward( x, router_logits, diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py index 9f033d5997b0..8abe42efa267 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py @@ -47,18 +47,17 @@ class NvFp4WeightView: """Bundles all NVFP4 weight tensors for MoE computation. - Provides a unified interface for both non-DWDP and DWDP paths: - - Non-DWDP: each list contains 1 element (local weight). - - DWDP: each list contains N elements (one per DWDP rank), - where the local rank's entry holds the actual model weight - and other ranks' entries hold prefetched buffer tensors. + Under the VA-based DWDP pipeline ``param.data`` is swapped to a + composite [num_experts, ...] tensor before the kernel call, so every + field is a single tensor — the bundle is just a convenient grouping + that lets the runner forward a single object instead of six. """ - w3_w1_weight: List[torch.Tensor] - fc1_weight_scale: List[torch.Tensor] - fc1_global_scale: List[torch.Tensor] - w2_weight: List[torch.Tensor] - fc2_weight_scale: List[torch.Tensor] - fc2_global_scale: List[torch.Tensor] + w3_w1_weight: torch.Tensor + fc1_weight_scale: torch.Tensor + fc1_global_scale: torch.Tensor + w2_weight: torch.Tensor + fc2_weight_scale: torch.Tensor + fc2_global_scale: torch.Tensor expert_size_per_partition: int slot_start: int @@ -465,14 +464,14 @@ def __init__( self.event_dict[key] = torch.cuda.Event() def _build_local_weight_view(self) -> NvFp4WeightView: - """Build weight view for non-DWDP path (single-element lists).""" + """Build the weight view from this backend's per-layer weights.""" return NvFp4WeightView( - w3_w1_weight=[self.w3_w1_weight], - fc1_weight_scale=[self.quant_scales.fc1_weight_block], - fc1_global_scale=[self.quant_scales.fc1_global], - w2_weight=[self.w2_weight], - fc2_weight_scale=[self.quant_scales.fc2_weight_block], - fc2_global_scale=[self.quant_scales.fc2_global], + w3_w1_weight=self.w3_w1_weight, + fc1_weight_scale=self.quant_scales.fc1_weight_block, + fc1_global_scale=self.quant_scales.fc1_global, + w2_weight=self.w2_weight, + fc2_weight_scale=self.quant_scales.fc2_weight_block, + fc2_global_scale=self.quant_scales.fc2_global, expert_size_per_partition=self.expert_size_per_partition, slot_start=self.slot_start, ) @@ -542,16 +541,15 @@ def run_moe_nvfp4( enable_alltoall: bool = False, weight_view: Optional[NvFp4WeightView] = None, ) -> torch.Tensor: - """NVFP4 MoE computation with unified interface. + """NVFP4 MoE computation. - Handles both non-DWDP and DWDP paths transparently: - - Non-DWDP (single-element weight lists): uses run_moe_nvfp4_impl. - Supports both fused-finalize and non-fused-finalize paths. - - DWDP (multi-element weight lists): uses run_moe_nvfp4_impl_dwdp. - Requires fused-finalize. + Uses the single-tensor ``run_moe_nvfp4_impl`` path. (The former + multi-B DWDP path was removed once DWDP switched to VA: VA swaps + ``param.data`` to a full [num_experts, ...] tensor, so the single- + tensor kernel is sufficient.) Args: - weight_view: Bundled weight tensors. If None, local weights are used. + weight_view: Bundled weight tensors. Must not be None. """ assert self.has_nvfp4 assert weight_view is not None @@ -569,8 +567,7 @@ def run_moe_nvfp4( effective_top_k = token_selected_experts.size(-1) - is_dwdp = len(weight_view.w3_w1_weight) > 1 - forward_impl = self.run_moe_nvfp4_impl_dwdp if is_dwdp else self.run_moe_nvfp4_impl + forward_impl = self.run_moe_nvfp4_impl tuner = AutoTuner.get() runner = CuteDslFusedMoENvfp4Runner( @@ -636,10 +633,10 @@ def run_moe_nvfp4_impl( # For non-gated (Relu2): weights are plain, output is N. x, x_sf = torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( input=x.view(torch.float4_e2m1fn_x2), - weight=weight_view.w3_w1_weight[0].view(torch.float4_e2m1fn_x2), + weight=weight_view.w3_w1_weight.view(torch.float4_e2m1fn_x2), input_scale=x_sf.view(torch.uint8), - weight_scale=weight_view.fc1_weight_scale[0].view(torch.uint8), - alpha=weight_view.fc1_global_scale[0], + weight_scale=weight_view.fc1_weight_scale.view(torch.uint8), + alpha=weight_view.fc1_global_scale, tile_idx_to_group_idx=tile_idx_to_expert_idx, tile_idx_to_mn_limit=tile_idx_to_mn_limit, permuted_idx_to_expanded_idx=permuted_idx_to_expanded_idx, @@ -673,12 +670,10 @@ def run_moe_nvfp4_impl( torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( input=x.view(torch.float4_e2m1fn_x2), - weight=[weight_view.w2_weight[0].view(torch.float4_e2m1fn_x2)], + weight=weight_view.w2_weight.view(torch.float4_e2m1fn_x2), input_scale=x_sf.view(torch.uint8), - weight_scale=[ - weight_view.fc2_weight_scale[0].view(torch.uint8) - ], - alpha=[weight_view.fc2_global_scale[0]], + weight_scale=weight_view.fc2_weight_scale.view(torch.uint8), + alpha=weight_view.fc2_global_scale, output=moe_output, tile_idx_to_group_idx=tile_idx_to_expert_idx, tile_idx_to_mn_limit=tile_idx_to_mn_limit, @@ -695,10 +690,10 @@ def run_moe_nvfp4_impl( else: x = torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_blackwell( input=x.view(torch.float4_e2m1fn_x2), - weight=weight_view.w2_weight[0].view(torch.float4_e2m1fn_x2), + weight=weight_view.w2_weight.view(torch.float4_e2m1fn_x2), input_scale=x_sf.view(torch.uint8), - weight_scale=weight_view.fc2_weight_scale[0].view(torch.uint8), - alpha=weight_view.fc2_global_scale[0], + weight_scale=weight_view.fc2_weight_scale.view(torch.uint8), + alpha=weight_view.fc2_global_scale, tile_idx_to_group_idx=tile_idx_to_expert_idx, num_non_exiting_tiles=num_non_exiting_tiles, num_experts=self.num_slots, @@ -716,109 +711,6 @@ def run_moe_nvfp4_impl( ) return moe_output - def run_moe_nvfp4_impl_dwdp( - self, - x: torch.Tensor, - token_selected_experts: torch.Tensor, - token_final_scales: Optional[torch.Tensor], - x_sf: torch.Tensor, - moe_output: torch.Tensor, - weight_view: NvFp4WeightView, - enable_alltoall: bool = False, - tile_size: int = 128, - ) -> torch.Tensor: - """DWDP NVFP4 MoE implementation using multi-B list ops. - - Requires fused-finalize since the non-fused FC2 op does not support - multiple B weight tensors. - """ - assert self.use_fused_finalize, ( - "DWDP requires fused finalize (cute_dsl_nvfp4_grouped_gemm_blackwell " - "does not support multiple B weight tensors)") - output_dtype = torch.bfloat16 - effective_top_k = token_selected_experts.size(1) - esp = weight_view.expert_size_per_partition - slot_start = weight_view.slot_start - - tile_idx_to_expert_idx, tile_idx_to_mn_limit, expanded_idx_to_permuted_idx, permuted_idx_to_expanded_idx, total_num_padded_tokens, num_non_exiting_tiles = torch.ops.trtllm.moe_sort( - token_selected_experts=token_selected_experts, - token_final_scales=token_final_scales, - num_experts=self.num_slots, - top_k=effective_top_k, - local_expert_offset=slot_start, - local_num_experts=esp, - tile_tokens_dim=tile_size, - ) - - self.event_dict[EventType.Main].record() - moe_output.record_stream( - self.aux_stream_dict[AuxStreamType.MoeOutputMemset]) - - x, x_sf = torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( - input=x.view(torch.float4_e2m1fn_x2), - weight=[ - w.view(torch.float4_e2m1fn_x2) for w in weight_view.w3_w1_weight - ], - input_scale=x_sf.view(torch.uint8), - weight_scale=[ - ws.view(torch.uint8) for ws in weight_view.fc1_weight_scale - ], - alpha=weight_view.fc1_global_scale, - tile_idx_to_group_idx=tile_idx_to_expert_idx, - tile_idx_to_mn_limit=tile_idx_to_mn_limit, - permuted_idx_to_expanded_idx=permuted_idx_to_expanded_idx, - num_non_exiting_tiles=num_non_exiting_tiles, - global_sf=self.fc2_input_scale, - num_experts=self.num_slots, - top_k=effective_top_k, - num_local_experts=esp, - local_expert_offset=slot_start, - tile_size=tile_size, - activation_type=self.activation_type, - ) - - with torch.cuda.stream( - self.aux_stream_dict[AuxStreamType.MoeOutputMemset]): - self.event_dict[EventType.Main].wait() - torch.ops.trtllm.moe_output_memset_inplace( - input=moe_output, - tile_idx_to_mn_limit=tile_idx_to_mn_limit, - expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, - permuted_idx_to_expanded_idx=permuted_idx_to_expanded_idx, - num_non_exiting_tiles=num_non_exiting_tiles, - tile_tokens_dim=tile_size, - top_k=effective_top_k, - ep_size=self.mapping.moe_ep_size, - enable_alltoall=enable_alltoall, - ) - self.event_dict[EventType.MoeOutputMemset].record() - self.event_dict[EventType.MoeOutputMemset].wait() - - torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( - input=x.view(torch.float4_e2m1fn_x2), - weight=[ - w.view(torch.float4_e2m1fn_x2) for w in weight_view.w2_weight - ], - input_scale=x_sf.view(torch.uint8), - weight_scale=[ - ws.view(torch.uint8) for ws in weight_view.fc2_weight_scale - ], - alpha=weight_view.fc2_global_scale, - output=moe_output, - tile_idx_to_group_idx=tile_idx_to_expert_idx, - tile_idx_to_mn_limit=tile_idx_to_mn_limit, - permuted_idx_to_expanded_idx=permuted_idx_to_expanded_idx, - num_non_exiting_tiles=num_non_exiting_tiles, - token_final_scales=token_final_scales, - num_experts=self.num_slots, - top_k=effective_top_k, - num_local_experts=esp, - local_expert_offset=slot_start, - tile_size=tile_size, - output_dtype=output_dtype, - ) - return moe_output - def run_moe_fp8_block_scales( self, x: torch.Tensor, @@ -933,8 +825,7 @@ def run_moe( """ # Execute MoE computation if self.has_nvfp4: - weight_view = kwargs.get( - "dwdp_weight_view") or self._build_local_weight_view() + weight_view = self._build_local_weight_view() result = self.run_moe_nvfp4( x=x, token_selected_experts=token_selected_experts, @@ -999,6 +890,3 @@ def load_weights(self, allow_partial_loading: bool = False): super().load_weights(weights, allow_partial_loading=allow_partial_loading) - dwdp_handle_collector = getattr(self, "dwdp_handle_collector", None) - if dwdp_handle_collector is not None: - dwdp_handle_collector.register_weights(self) diff --git a/tensorrt_llm/_torch/modules/fused_moe/interface.py b/tensorrt_llm/_torch/modules/fused_moe/interface.py index 8af319ad7a3a..92acca88768d 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/interface.py +++ b/tensorrt_llm/_torch/modules/fused_moe/interface.py @@ -47,7 +47,6 @@ def _warn_and_return(reason: str) -> Tuple[bool, Optional[str]]: from ...model_config import ModelConfig -from ...pyexecutor.dwdp import get_global_dwdp_manager from ...utils import (ActivationType, AuxStreamType, Fp4QuantizedTensor, get_model_extra_attrs, is_gated_activation, is_torch_compiling) @@ -391,26 +390,80 @@ def __init__( self.initial_global_assignments = list(range(self.num_experts)) self.allreduce = None - # Override expert layout if DWDP is enabled + # Override expert layout when DWDP is enabled. This must run before + # create_weights() (which is invoked later by ConfigurableMoE) so the + # fused MoE backend allocates ``num_experts_per_worker`` slots per + # rank — not ``num_experts // ep_size``. fixup_moe_backends() at + # setup_dwdp() time later promotes these to the full composite-VA + # view (``ep_size = 1``, ``slot_start = 0``, + # ``slot_end = num_experts``); the earlier override is still + # required because ``mapping.moe_ep_size`` is now 1, so the + # un-overridden default would be ``expert_size_per_partition = + # num_experts`` (each rank allocating storage for every expert). self._init_dwdp_expert_layout() self._init_perfect_router() def _init_dwdp_expert_layout(self): - """Override expert layout when DWDP is enabled.""" + """Override expert layout when DWDP is enabled. + + Plumbs ``num_experts_per_worker`` (storage size) and + ``start_expert_id`` (storage start) from the active + ``DwdpManager``. This is a no-op when DWDP is not enabled + (``get_global_dwdp_manager()`` returns ``None``). + + For the uniform partition case (``num_prefetch_experts == + num_experts_per_worker == num_experts // dwdp_size``) this is + mathematically equivalent to the legacy ``ep_size = dwdp_size`` + layout. It additionally enables: + + * Non-uniform partition (``dwdp_size`` does not divide + ``num_experts``): user picks ``num_prefetch_experts < num_experts_per_worker`` + so that ``(dwdp_size - 1) * num_prefetch_experts + num_experts_per_worker + == num_experts`` exactly. Adjacent ranks' valid ranges + overlap by ``num_experts_per_worker - num_prefetch_experts`` + experts; ``_validate_partition_config`` rejects + configurations whose last-rank end exceeds ``num_experts`` + because the GB200 cuMemMap-with-fabric-handle ABI requires + ``mnnvl_size == handle_phys_size`` (no partial mapping), so + tail-padded storage cannot be partially mapped into a + ``num_experts``-sized composite VA. + * Redundancy (``num_prefetch_experts < num_experts_per_worker`` + with ``(dwdp_size - 1) * stride + size == num_experts``): + consecutive ranks' ranges overlap; peer-side + ``lookup_owner`` (Phase 2) picks the lowest-rank owner so + reads of shared experts are deterministic. + + DWDP and MoE EPLB are mutually exclusive — DWDP swaps + ``param.data`` to a composite-VA tensor at runtime, which the + EPLB rebalancer would clobber. + """ + from tensorrt_llm._torch.pyexecutor.dwdp import get_global_dwdp_manager + dwdp_manager = get_global_dwdp_manager() if dwdp_manager is None: return assert self.layer_load_balancer is None, ( "DWDP and EPLB (MoE load balancer) cannot be used together. " "Disable one of dwdp_config or moe_load_balancer.") + self.num_slots = self.num_experts self.expert_size_per_partition = dwdp_manager.num_experts_per_worker dwdp_size = dwdp_manager.dwdp_size + # Routing-side expert assignment: distribute ``num_experts`` round-robin + # across DWDP ranks. Independent of the storage layout — the gate uses + # this to map expert ids to ranks, while DWDP composite VA handles the + # actual weight access. self.initial_global_assignments = [ (ep_rank * self.num_experts // dwdp_size + local_slot_id) % self.num_experts for ep_rank in range(dwdp_size) for local_slot_id in range(self.expert_size_per_partition) ] + # Storage range: ``[start, start + size)``. Phase 2's strict + # validation guarantees ``slot_end <= num_experts``, so every + # storage slot maps to a valid expert id. ``len(initial_local_expert_ids) + # == expert_size_per_partition`` is preserved, which is required + # by the per-slot weight-scale copy in + # ``quantization.load_quant_scales``. self.slot_start = dwdp_manager.start_expert_id self.slot_end = self.slot_start + self.expert_size_per_partition self.initial_local_expert_ids = list( diff --git a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py index dd13a8a80905..407794b82cf1 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py +++ b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py @@ -693,7 +693,7 @@ def _get_backend_kwargs( Backend-specific kwargs: - Cutlass: is_sf_swizzled, enable_alltoall, tuner_*, moe_output - - CuteDSL: enable_alltoall, moe_output, dwdp_weight_view + - CuteDSL: enable_alltoall, moe_output - DeepGemm: workspace - TRTLLMGen: router_logits, do_finalize, moe_output """ @@ -729,11 +729,6 @@ def _get_backend_kwargs( all_rank_num_tokens=all_rank_num_tokens, output_dtype=output_dtype ) - if moe.enable_dwdp: - kwargs["dwdp_weight_view"] = moe.dwdp_manager.build_weight_view( - moe.layer_idx, moe.backend - ) - elif moe.backend.__class__ == DeepGemmFusedMoE: if workspace is not None: kwargs["workspace"] = workspace diff --git a/tensorrt_llm/_torch/pyexecutor/dwdp.py b/tensorrt_llm/_torch/pyexecutor/dwdp.py index cd86ef75c8f0..d5a7fa7ad55f 100644 --- a/tensorrt_llm/_torch/pyexecutor/dwdp.py +++ b/tensorrt_llm/_torch/pyexecutor/dwdp.py @@ -1,34 +1,52 @@ -from typing import Dict, List, Optional, Tuple +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""DwdpManager facade for VA-based DWDP. + +This module owns three concerns and delegates the rest: + * Lifecycle / global singleton (``__enter__`` / ``__exit__`` + + ``set_global_dwdp_manager`` / ``get_global_dwdp_manager``) + * DWDP MPI sub-communicator creation (``_create_dwdp_comm``) + * Layer index registration (SSOT: ``add_layer`` → ``_registered_layers``) + plus runtime entry points that forward to ``DWDPWeightManager`` + (``prefetch_first_layers``, ``wait_and_bind``, + ``record_compute_and_prefetch_next``) + +The heavy lifting — MNNVL handle allocation, composite VA layout, +double-buffer scheduling — lives in ``tensorrt_llm._torch.modules.dwdp`` +and is wired up by ``setup_dwdp()`` during ``setup(model)``. +""" + +from __future__ import annotations + +from typing import List, Optional import torch import torch.nn as nn -from cuda.bindings import driver as cuda_driver -from cuda.bindings import runtime as cudart from mpi4py.MPI import COMM_WORLD from tensorrt_llm._torch.distributed import MPIDist -from tensorrt_llm._utils import global_mpi_rank, nvtx_range +from tensorrt_llm._torch.modules.dwdp import DWDPWeightManager, setup_dwdp +from tensorrt_llm._utils import global_mpi_rank from tensorrt_llm.llmapi.llm_args import DwdpConfig - -# Parameter names to collect handles for -WEIGHT_PARAMS = ["w3_w1_weight", "w2_weight"] -BIAS_PARAMS = ["w3_w1_bias", "w2_bias"] -# Quant scale params vary by quantization method -QUANT_SCALE_PARAMS = [ - "w3_w1_weight_scale", - "w2_weight_scale", # NVFP4/MXFP4 - "fc31_alpha", - "fc2_alpha", # NVFP4 alpha -] - -# Default slice granularity for batched memcpy (2 MiB). -MEMCPY_BATCH_SLICE_BYTES = 1 << 21 - +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping _global_dwdp_manager: Optional["DwdpManager"] = None -def set_global_dwdp_manager(manager: "DwdpManager"): +def set_global_dwdp_manager(manager: Optional["DwdpManager"]) -> None: global _global_dwdp_manager _global_dwdp_manager = manager @@ -37,650 +55,196 @@ def get_global_dwdp_manager() -> Optional["DwdpManager"]: return _global_dwdp_manager -def check_cuda_error(err, context: str = ""): - """Check CUDA error.""" - if err != cudart.cudaError_t.cudaSuccess: - raise RuntimeError(f"CUDA error in {context}: {err}") - - -class DwdpLayerHandleCollector: - """ - Dwdp Layer Handle Collector for IPC handle coordination and prefetch buffer management. - """ - - def __init__( - self, - layer_idx: int, - ): - self.layer_idx = layer_idx - - # Local IPC handles: param_name -> handle_bytes - self.local_ipc_handles: Dict[str, bytes] = {} - # Local pointers: param_name -> data_ptr (for verification) - self.local_ptrs: Dict[str, int] = {} - # Local offsets: param_name -> offset from allocation base - # IPC handle points to allocation base, we need offset to get actual tensor data - self.local_offsets: Dict[str, int] = {} - # Parameter shapes: param_name -> shape (without expert dim) - self.param_shapes: Dict[str, torch.Size] = {} - # Parameter dtypes: param_name -> dtype - self.param_dtypes: Dict[str, torch.dtype] = {} - # Peer pointers: (peer_rank, param_name) -> ptr (already adjusted with offset) - self.peer_ptrs: Dict[Tuple[int, str], int] = {} - - def register_weights(self, module: nn.Module): - """ - Register weights from a MoE module and create IPC handles. - - Called after module.load_weights() completes. - - Args: - module: The MoE module with loaded weights - """ - params_to_register = [] - # Weights (check if present and not None) - for param_name in WEIGHT_PARAMS: - if hasattr(module, param_name) and getattr(module, param_name, None) is not None: - params_to_register.append(param_name) - # Bias (optional) - if hasattr(module, "bias"): - params_to_register.extend(BIAS_PARAMS) - # Quant scales (optional, depends on quant method) - for param_name in QUANT_SCALE_PARAMS: - if hasattr(module, param_name) and getattr(module, param_name, None) is not None: - params_to_register.append(param_name) - - # Register each parameter - for param_name in params_to_register: - param = getattr(module, param_name) - if isinstance(param, nn.Parameter): - param = param.data - if param is None: - continue - if not param.is_cuda or not param.is_contiguous(): - raise ValueError(f"Parameter {param_name} is not on GPU or is not contiguous") - self._register_param(param_name, param) - - def _register_param(self, param_name: str, param: torch.Tensor): - # Get IPC handle - note: handle points to the CUDA allocation base, not tensor's data_ptr - tensor_ptr = param.data_ptr() - err, handle = cudart.cudaIpcGetMemHandle(tensor_ptr) - check_cuda_error(err, f"get handle for {param_name}") - - # Get allocation base address using Driver API cuMemGetAddressRange - # This returns the actual base address and size of the CUDA allocation - # cudaPointerGetAttributes.devicePointer returns the input pointer, not base! - err, alloc_base, alloc_size = cuda_driver.cuMemGetAddressRange(tensor_ptr) - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"cuMemGetAddressRange failed for {param_name}: {err}") - - # Calculate offset from allocation base - # Convert CUdeviceptr to int for arithmetic - offset = tensor_ptr - int(alloc_base) - - self.local_ipc_handles[param_name] = bytes(handle.reserved) - self.local_ptrs[param_name] = tensor_ptr - self.local_offsets[param_name] = offset - self.param_shapes[param_name] = param.shape[1:] - self.param_dtypes[param_name] = param.dtype - - def get_peer_ptr(self, peer_rank: int, param_name: str) -> int: - """Get pointer to parameter on peer rank.""" - return self.peer_ptrs[(peer_rank, param_name)] - - def cleanup(self): - """Clean up peer handles.""" - for _, ptr in self.peer_ptrs.items(): - cudart.cudaIpcCloseMemHandle(ptr) - self.peer_ptrs.clear() - - -class DwdpPrefetchBuffer: - """ - Ping-pong buffer for expert weight prefetching. - - Buffer Selection Strategy: - - Even layers (0, 2, 4, ...) use buffer[0] - - Odd layers (1, 3, 5, ...) use buffer[1] - - This ensures layer N-1's prefetch doesn't overwrite layer N's data - - Synchronization Strategy: - - prefetch_events[buffer_idx][layer_idx]: Recorded when prefetch completes - Waited by forward() before using prefetched data - - compute_events[buffer_idx][layer_idx]: Recorded when forward() completes - Waited by next prefetch before overwriting buffer - - Buffer Layout (organized by rank): - - buffers[buffer_idx][param_name] = List[Optional[Tensor]] - - len(list) == dwdp_size - - list[peer_rank] = Tensor[num_prefetch_experts, ...] for peer_rank != dwdp_rank - - list[dwdp_rank] = None (local weight used directly, not prefetched) - """ - - def __init__( - self, - dwdp_size: int, - dwdp_rank: int, - num_experts_per_worker: int, - num_prefetch_experts: int, - num_layers: int, - first_moe_layer_idx: int, - param_shapes: Dict[str, torch.Size], - param_dtypes: Dict[str, torch.dtype], - ): - self.dwdp_size = dwdp_size - self.num_prefetch_experts = num_prefetch_experts - self.num_experts_per_worker = num_experts_per_worker - self.num_layers = num_layers - self.first_moe_layer_idx = first_moe_layer_idx - self.num_buffers = 2 # Ping-pong - self.dwdp_rank = dwdp_rank - - self.param_shapes = param_shapes - self.param_dtypes = param_dtypes - - self.device = torch.cuda.current_device() - - # buffers[buffer_idx][param_name] = List[Optional[Tensor]] - # list[peer_rank] contains prefetched weights from that rank - # list[dwdp_rank] = None (local weights used directly) - self.buffers: List[Dict[str, List[Optional[torch.Tensor]]]] = [] - - for _ in range(self.num_buffers): - buffer = {} - for param_name, shape in param_shapes.items(): - dtype = param_dtypes[param_name] - # Pre-allocate list of length dwdp_size, one slot per rank - # tensor_list[dwdp_rank] = None (local weights used directly) - # tensor_list[peer_rank] = Tensor for prefetched weights from peer - tensor_list: List[Optional[torch.Tensor]] = [None] * dwdp_size - for peer_rank in range(dwdp_size): - if peer_rank != dwdp_rank: - buffer_shape = (self.num_prefetch_experts,) + tuple(shape) - tensor_list[peer_rank] = torch.empty( - buffer_shape, - dtype=dtype, - device=self.device, - ) - buffer[param_name] = tensor_list - self.buffers.append(buffer) - - self.max_layer_idx = num_layers + first_moe_layer_idx - self.prefetch_events: List[List[torch.cuda.Event]] = [ - [torch.cuda.Event() for _ in range(self.max_layer_idx // self.num_buffers + 1)] - for _ in range(self.num_buffers) - ] - self.compute_events: List[List[torch.cuda.Event]] = [ - [torch.cuda.Event() for _ in range(self.max_layer_idx // self.num_buffers + 1)] - for _ in range(self.num_buffers) - ] - self.prefetch_stream = torch.cuda.Stream(device=self.device) - - def initialize_compute_events(self): - for buffer_idx in range(self.num_buffers): - self.compute_events[buffer_idx][0].record(torch.cuda.current_stream()) - - def record_prefetch_event(self, layer_idx: int): - self.prefetch_events[layer_idx % self.num_buffers][layer_idx // self.num_buffers].record( - self.prefetch_stream - ) - - def record_compute_event(self, layer_idx: int): - self.compute_events[layer_idx % self.num_buffers][layer_idx // self.num_buffers].record( - torch.cuda.current_stream() - ) - - def wait_prefetch_event(self, layer_idx: int): - torch.cuda.current_stream().wait_event( - self.prefetch_events[layer_idx % self.num_buffers][layer_idx // self.num_buffers] - ) - - def wait_compute_event(self, layer_idx: int): - self.prefetch_stream.wait_event( - self.compute_events[layer_idx % self.num_buffers][layer_idx // self.num_buffers] - ) - - class DwdpManager: - """ - Dwdp Manager for IPC handle coordination and prefetch buffer management. + """Lifecycle facade that plumbs the VA-based DWDP pipeline. - This manager: - - Tracks IPC handles for all MoE layers across Context workers - - Manages double-buffered prefetch buffers for remote expert weights - - Provides expert tensor routing (local vs. prefetched) + Construction creates the DWDP MPI sub-communicator. ``add_layer`` is the + Single Source of Truth for MoE layer indices — ``configurable_moe`` + registers each MoE layer during model construction, and ``setup(model)`` + passes that list to ``setup_dwdp()`` so ``collect_moe_params`` never has + to re-discover layers by walking the model tree. + After ``setup()`` returns, the runtime methods (``prefetch_first_layers``, + ``wait_and_bind``, ``record_compute_and_prefetch_next``) forward to the + ``DWDPWeightManager`` instance produced by ``setup_dwdp``. """ def __init__( self, config: DwdpConfig, - dist: Optional[object] = None, - ): + dist: object, + mapping: Mapping, + ) -> None: + if not isinstance(dist, MPIDist): + raise RuntimeError("DWDP requires MPI backend (MPIDist)") + if not mapping.dwdp_enabled: + raise RuntimeError( + f"DwdpManager requires mapping.dwdp_enabled (dwdp_size > 1); " + f"got dwdp_size={mapping.dwdp_size}" + ) + + # Validate DwdpConfig topology against the actual MPI launch. + # `num_groups` declares the user's intended topology — that the launch + # contains `num_groups × dwdp_size` CTX worker ranks, partitioned into + # `num_groups` independent DWDP groups of `dwdp_size` ranks each. + # Without this validation `num_groups` is a dead schema field, which + # is how it was treated up through commit `1fbc0d49`. + if config.num_groups <= 0: + raise ValueError(f"DwdpConfig.num_groups must be positive, got {config.num_groups}") + self.rank = global_mpi_rank() + group_id = self.rank // mapping.dwdp_size + if group_id >= config.num_groups: + raise ValueError( + f"DwdpManager rank {self.rank} computes group_id={group_id} " + f"(rank // dwdp_size={mapping.dwdp_size}) which is >= " + f"DwdpConfig.num_groups ({config.num_groups}). The launch " + f"started more CTX workers than fit in {config.num_groups} × " + f"{mapping.dwdp_size} = {config.num_groups * mapping.dwdp_size} " + f"ranks, or num_groups in the YAML is set too small." + ) + expected_min_world = config.num_groups * mapping.dwdp_size + actual_world = COMM_WORLD.Get_size() + if actual_world < expected_min_world: + raise ValueError( + f"DwdpConfig declares {config.num_groups} groups × dwdp_size " + f"{mapping.dwdp_size} = {expected_min_world} CTX workers, but " + f"MPI world size is only {actual_world}. Either the launch's " + f"--ntasks is too small, or num_groups is set too high." + ) + self.config = config self.dist = dist - self.dwdp_size = config.dwdp_size + self._mapping = mapping + self.dwdp_size = mapping.dwdp_size + self.dwdp_rank = mapping.dwdp_rank self.num_experts_per_worker = config.num_experts_per_worker self.num_groups = config.num_groups - self.use_batched_prefetch = config.contention_opt + self.num_prefetch_experts = config.num_prefetch_experts + + # Per-rank expert range. ``num_prefetch_experts`` is the rank-to-rank + # stride and ``num_experts_per_worker`` is the (uniform across ranks) + # storage chunk size. When the user requests redundancy + # (``num_prefetch_experts < num_experts_per_worker``) consecutive + # ranks' ranges overlap; when the partition is non-uniform (e.g. + # dwdp_size = 3 over 256 experts) the tail rank's + # ``[start, end)`` runs past ``num_experts`` and is later capped at + # weight-load time. ``_init_dwdp_expert_layout`` reads + # ``start_expert_id`` to install the correct ``slot_start`` / + # ``slot_end`` on every fused MoE backend before ``create_weights``. + self.start_expert_id = self.dwdp_rank * self.num_prefetch_experts + self.end_expert_id = self.start_expert_id + self.num_experts_per_worker - if config.contention_opt: - global MEMCPY_BATCH_SLICE_BYTES + self.dwdp_group = self._create_dwdp_comm() - self._init_dwdp_group() + # SSOT for MoE layer indices; populated by configurable_moe.add_layer() + self._registered_layers: List[int] = [] - # Per-layer IPC handle collectors (indexed by layer_idx) - self.ipc_collectors: List[DwdpLayerHandleCollector] = [] + # Set by setup(model); None until then + self._weight_manager: Optional[DWDPWeightManager] = None - # Prefetch buffer (initialized later in create_py_executor) - self.prefetch_buffer: Optional[DwdpPrefetchBuffer] = None - # Batched prefetch copy plans are stable once peer handles and prefetch - # buffers are initialized, so cache them per layer to avoid host rebuild. - self.batched_prefetch_copy_plans: Dict[int, Tuple[List[int], List[int], List[int]]] = {} - # Auto-detected from first add_layer() call - self.first_moe_layer_idx: Optional[int] = None + # ------------------------------------------------------------------ + # DWDP MPI group + # ------------------------------------------------------------------ - # Peer expert ranges: (peer_rank, (start_expert_id, end_expert_id)) - self.peer_expert_ranges: Dict[int, Tuple[int, int]] = {} + def _create_dwdp_comm(self): + """Create an MPI sub-communicator scoped to this rank's DWDP group. - self.dwdp_rank = self.rank % self.dwdp_size - self.num_prefetch_experts = config.num_prefetch_experts - self.start_expert_id = self.num_prefetch_experts * self.dwdp_rank - self.end_expert_id = self.start_expert_id + self.num_experts_per_worker + With num_groups=2, dwdp_size=4: + Group 0: ranks [0, 1, 2, 3] + Group 1: ranks [4, 5, 6, 7] + """ + group_id = self.rank // self.dwdp_size + group_start = group_id * self.dwdp_size + ranks = list(range(group_start, group_start + self.dwdp_size)) + return COMM_WORLD.Create_group(COMM_WORLD.group.Incl(ranks)) - def __enter__(self): + # ------------------------------------------------------------------ + # Global singleton lifecycle + # ------------------------------------------------------------------ + + def __enter__(self) -> "DwdpManager": + if get_global_dwdp_manager() is not None: + raise RuntimeError("DwdpManager already registered globally") set_global_dwdp_manager(self) return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: self.cleanup() set_global_dwdp_manager(None) return False - def _init_dwdp_group(self): - if not isinstance(self.dist, MPIDist): - raise RuntimeError("DWDP requires MPI backend (MPIDist)") - - self.rank = global_mpi_rank() - - # Calculate which group this rank belongs to - # With num_groups=2, dwdp_size=4: - # Group 0: ranks [0, 1, 2, 3] - # Group 1: ranks [4, 5, 6, 7] - self.group_id = self.rank // self.dwdp_size - group_start_rank = self.group_id * self.dwdp_size - ranks = list(range(group_start_rank, group_start_rank + self.dwdp_size)) - - new_group = COMM_WORLD.group.Incl(ranks) - self.dwdp_group = COMM_WORLD.Create_group(new_group) - def is_enabled(self) -> bool: return self.dwdp_size > 1 - def cleanup(self): - """Release all IPC handles and clean up resources.""" - for collector in self.ipc_collectors: - collector.cleanup() - self.ipc_collectors.clear() + def cleanup(self) -> None: + """Release MPI sub-communicator and reset weight manager. Idempotent.""" + self._weight_manager = None if self.dwdp_group is not None: self.dwdp_group.Free() self.dwdp_group = None - def add_layer( - self, - layer_idx: int, - ) -> "DwdpLayerHandleCollector": - """ - Add a new layer IPC handle collector. + # ------------------------------------------------------------------ + # Layer registration (SSOT) + setup + # ------------------------------------------------------------------ - Called from CuteDslFusedMoE.__init__() during model construction. - """ - if self.first_moe_layer_idx is None: - self.first_moe_layer_idx = layer_idx - collector = DwdpLayerHandleCollector(layer_idx=layer_idx) - self.ipc_collectors.append(collector) - return collector - - def exchange_all_handles(self): - """ - Exchange IPC handles with peer Context workers via Dwdp Group AllGather. - - Called after all weights are loaded, before creating prefetch buffer. - """ - - # Collect all local handles with explicit worker info - local_data = { - "dwdp_rank": self.dwdp_rank, - "expert_start_id": self.start_expert_id, - "expert_end_id": self.end_expert_id, - "ipc_collectors": [], - } - for collector in self.ipc_collectors: - local_data["ipc_collectors"].append( - { - "layer_idx": collector.layer_idx, - "handles": collector.local_ipc_handles, - "offsets": collector.local_offsets, - } - ) - - # AllGather from all Context workers in DWDP group - all_data = self.dwdp_group.allgather(local_data) + def add_layer(self, layer_idx: int) -> None: + """Register an MoE layer index. Called from configurable_moe.__init__.""" + if layer_idx in self._registered_layers: + logger.warning(f"[DwdpManager] Layer {layer_idx} already registered; ignoring") + return + self._registered_layers.append(layer_idx) - # Open handles from peer workers - for peer_data in all_data: - peer_rank = peer_data["dwdp_rank"] - self.peer_expert_ranges[peer_rank] = ( - peer_data["expert_start_id"], - peer_data["expert_end_id"], - ) + def setup(self, model: nn.Module) -> Optional[DWDPWeightManager]: + """Run the VA initialization pipeline on ``model``. - if peer_rank == self.dwdp_rank: - continue - for layer_idx, ipc_collector in enumerate(peer_data["ipc_collectors"]): - collector = self.ipc_collectors[layer_idx] - peer_offsets = ipc_collector["offsets"] - for param_name, handle_bytes in ipc_collector["handles"].items(): - # Reconstruct and open handle - handle = cudart.cudaIpcMemHandle_t() - handle.reserved = list(handle_bytes) - - err, base_ptr = cudart.cudaIpcOpenMemHandle( - handle, cudart.cudaIpcMemLazyEnablePeerAccess - ) - check_cuda_error(err, f"open handle rank={peer_rank}") - - # Apply offset to get actual tensor pointer - # IPC handle points to allocation base, offset gives us the tensor location - offset = peer_offsets[param_name] - actual_ptr = base_ptr + offset - collector.peer_ptrs[(peer_rank, param_name)] = actual_ptr - - def initialize_prefetch_buffer(self): - """ - Initialize the prefetch buffer. + Delegates to ``setup_dwdp`` (transport + weight buffer + weight + manager + MoE backend fixup). Stores the returned ``DWDPWeightManager`` + for subsequent runtime calls. - Called in create_py_executor() after model loading. + Called once from ``py_executor_creator`` after weight loading. """ - self.batched_prefetch_copy_plans.clear() - self.prefetch_buffer = DwdpPrefetchBuffer( - dwdp_size=self.dwdp_size, - dwdp_rank=self.dwdp_rank, + layer_indices = sorted(self._registered_layers) + device_id = torch.cuda.current_device() + self._weight_manager = setup_dwdp( + model=model, + mapping=self._mapping, + device_id=device_id, + comm=self.dwdp_group, + layer_indices=layer_indices, num_experts_per_worker=self.num_experts_per_worker, num_prefetch_experts=self.num_prefetch_experts, - num_layers=len(self.ipc_collectors), - first_moe_layer_idx=self.first_moe_layer_idx, - param_shapes=self.ipc_collectors[0].param_shapes, - param_dtypes=self.ipc_collectors[0].param_dtypes, - ) - self.prefetch_buffer.initialize_compute_events() - - def prefetch_first_layers(self): - """Prefetch the first num_buffers layers as warmup.""" - if self.prefetch_buffer is None: - raise RuntimeError("Prefetch buffer is not initialized") - start = self.first_moe_layer_idx - for layer_idx in range(start, start + self.prefetch_buffer.num_buffers): - self.prefetch_layer(layer_idx) - self.prefetch_buffer.record_prefetch_event(layer_idx) - - def build_weight_view(self, layer_idx: int, backend): - """Build NvFp4WeightView from prefetch buffer and local weights. - - Assembles weight tensors from all DWDP ranks: - - Peer ranks: uses prefetched buffer tensors - - Local rank: uses backend's actual model weights - - Args: - layer_idx: The MoE layer index. - backend: The CuteDslFusedMoE backend holding local model weights. - - Returns: - NvFp4WeightView with all weights assembled. - """ - from tensorrt_llm._torch.modules.fused_moe.fused_moe_cute_dsl import NvFp4WeightView - - buffer_data = self.wait_prefetch_and_get_buffer(layer_idx) - required_keys = ( - "w3_w1_weight", - "w3_w1_weight_scale", - "fc31_alpha", - "w2_weight", - "w2_weight_scale", - "fc2_alpha", - ) - missing_keys = [key for key in required_keys if key not in buffer_data] - if missing_keys: - raise ValueError( - f"DWDP buffer missing required keys {missing_keys} for layer {layer_idx}." - ) - - w3_w1_weight_list = buffer_data["w3_w1_weight"] - fc1_weight_scale_list = buffer_data["w3_w1_weight_scale"] - fc1_global_scale_list = buffer_data["fc31_alpha"] - w2_weight_list = buffer_data["w2_weight"] - fc2_weight_scale_list = buffer_data["w2_weight_scale"] - fc2_global_scale_list = buffer_data["fc2_alpha"] - - w3_w1_weight_list[self.dwdp_rank] = backend.w3_w1_weight - fc1_weight_scale_list[self.dwdp_rank] = backend.quant_scales.fc1_weight_block - fc1_global_scale_list[self.dwdp_rank] = backend.quant_scales.fc1_global - w2_weight_list[self.dwdp_rank] = backend.w2_weight - fc2_weight_scale_list[self.dwdp_rank] = backend.quant_scales.fc2_weight_block - fc2_global_scale_list[self.dwdp_rank] = backend.quant_scales.fc2_global - - return NvFp4WeightView( - w3_w1_weight=w3_w1_weight_list, - fc1_weight_scale=fc1_weight_scale_list, - fc1_global_scale=fc1_global_scale_list, - w2_weight=w2_weight_list, - fc2_weight_scale=fc2_weight_scale_list, - fc2_global_scale=fc2_global_scale_list, - expert_size_per_partition=backend.num_slots, - slot_start=0, - ) - - def wait_prefetch_and_get_buffer( - self, layer_idx: int - ) -> Optional[Dict[str, List[Optional[torch.Tensor]]]]: - """Wait for prefetch to complete and return the buffer for this layer. - - Returns: - Dict mapping param_name to List[Optional[Tensor]] where: - - list[peer_rank] = Tensor for prefetched weights from that peer - - list[dwdp_rank] = None (local weights used directly) - """ - if self.prefetch_buffer is None: - raise RuntimeError("Prefetch buffer is not initialized") - self.prefetch_buffer.wait_prefetch_event(layer_idx) - buffer_idx = layer_idx % self.prefetch_buffer.num_buffers - return self.prefetch_buffer.buffers[buffer_idx] - - def record_compute_and_prefetch_next(self, layer_idx: int): - """Record compute completion and trigger prefetch for layer_idx + num_buffers.""" - if self.prefetch_buffer is None: - raise RuntimeError("Prefetch buffer is not initialized") - # Record compute event for current layer - self.prefetch_buffer.record_compute_event(layer_idx) - - next_layer_idx = layer_idx + self.prefetch_buffer.num_buffers - if next_layer_idx >= self.prefetch_buffer.max_layer_idx: - return - # prefetch_layer handles stream internally: local copy on default stream, peer copy on prefetch stream - self.prefetch_layer(next_layer_idx, wait_compute_layer_idx=layer_idx) - self.prefetch_buffer.record_prefetch_event(next_layer_idx) - - def _get_prefetch_src_offset_from_peer(self, peer_rank: int) -> int: - """ - Calculate the source offset (in number of experts) to fetch from a peer. - - Returns: - src_offset: Offset into peer's local expert tensor to start copying from - - Example: 256 experts, rank0: [0, 200), rank1: [56, 256) - - rank0 needs [200, 256) from rank1: - src_offset = 200 - 56 = 144 (fetch last 56 experts from rank1) - - rank1 needs [0, 56) from rank0: - src_offset = 0 - 0 = 0 (fetch first 56 experts from rank0) - """ - peer_start, peer_end = self.peer_expert_ranges[peer_rank] - - # What I need = global - what I have - # From peer = what I need ∩ what peer has - if self.dwdp_rank < peer_rank: - # I'm earlier rank, need experts after my end (tail of peer's experts) - prefetch_end = peer_end - prefetch_start = prefetch_end - self.num_prefetch_experts - else: - # I'm later rank, need experts before my start (head of peer's experts) - prefetch_start = peer_start - - src_offset = prefetch_start - peer_start - return src_offset - - def _build_batched_prefetch_copy_plan( - self, - layer_idx: int, - collector: DwdpLayerHandleCollector, - buffer_idx: int, - ) -> Tuple[List[int], List[int], List[int]]: - """ - Build one batched memcpy plan for all remote peers in round-robin slice order. - - Each remote buffer is split into MEMCPY_BATCH_SLICE_BYTES slices and - interleaved across peers so that copies to different source ranks alternate, - limiting NVLink contention on any single link. - - Results are cached per layer_idx since peer pointers and buffer addresses - are stable after initialization. - """ - cached = self.batched_prefetch_copy_plans.get(layer_idx) - if cached is not None: - return cached - - dst_ptrs: List[int] = [] - src_ptrs: List[int] = [] - sizes: List[int] = [] - - for param_name, param_shape in collector.param_shapes.items(): - param_dtype = collector.param_dtypes[param_name] - expert_size = param_shape.numel() * param_dtype.itemsize - data_size = self.num_prefetch_experts * expert_size - - slice_offset = 0 - while slice_offset < data_size: - slice_size = min(MEMCPY_BATCH_SLICE_BYTES, data_size - slice_offset) - - # Round-robin across peers for this slice - for peer_offset in range(1, self.dwdp_size): - peer_rank = (self.dwdp_rank + peer_offset) % self.dwdp_size - src_expert_offset = self._get_prefetch_src_offset_from_peer(peer_rank) - - base_ptr = collector.get_peer_ptr(peer_rank, param_name) - src_ptr = base_ptr + src_expert_offset * expert_size - - dst_tensor = self.prefetch_buffer.buffers[buffer_idx][param_name][peer_rank] - if dst_tensor is None: - raise RuntimeError( - f"Missing prefetch buffer for peer_rank={peer_rank} param={param_name}" - ) - dst_ptr = dst_tensor.data_ptr() - - dst_ptrs.append(dst_ptr + slice_offset) - src_ptrs.append(src_ptr + slice_offset) - sizes.append(slice_size) - - slice_offset += slice_size - - copy_plan = (dst_ptrs, src_ptrs, sizes) - self.batched_prefetch_copy_plans[layer_idx] = copy_plan - return copy_plan - - def _prefetch_layer_default( - self, - layer_idx: int, - collector: DwdpLayerHandleCollector, - buffer_idx: int, - ): - """Original per-peer cudaMemcpyAsync prefetch path (baseline).""" - for peer_rank in range(self.dwdp_size): - if peer_rank == self.dwdp_rank: - continue - - src_expert_offset = self._get_prefetch_src_offset_from_peer(peer_rank) - - for param_name, param_shape in collector.param_shapes.items(): - param_dtype = collector.param_dtypes[param_name] - expert_size = param_shape.numel() * param_dtype.itemsize - - base_ptr = collector.get_peer_ptr(peer_rank, param_name) - src_ptr = base_ptr + src_expert_offset * expert_size - - dst_tensor = self.prefetch_buffer.buffers[buffer_idx][param_name][peer_rank] - dst_ptr = dst_tensor.data_ptr() - - data_size = self.num_prefetch_experts * expert_size - - (err,) = cudart.cudaMemcpyAsync( - dst_ptr, - src_ptr, - data_size, - cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice, - self.prefetch_buffer.prefetch_stream.cuda_stream, - ) - check_cuda_error( - err, f"prefetch layer {layer_idx} peer_rank {peer_rank} {param_name}" - ) - - def _prefetch_layer_batched( - self, - layer_idx: int, - collector: DwdpLayerHandleCollector, - buffer_idx: int, - ): - """Batched cudaMemcpyBatchAsync prefetch path (limited-contention opt).""" - dst_ptrs, src_ptrs, sizes = self._build_batched_prefetch_copy_plan( - layer_idx, collector, buffer_idx + contention_opt=self.config.contention_opt, ) - if not dst_ptrs: - return - - attr = cudart.cudaMemcpyAttributes() - attr.srcAccessOrder = cudart.cudaMemcpySrcAccessOrder.cudaMemcpySrcAccessOrderStream - (err,) = cudart.cudaMemcpyBatchAsync( - dst_ptrs, - src_ptrs, - sizes, - len(dst_ptrs), - [attr], - [0], - 1, - self.prefetch_buffer.prefetch_stream.cuda_stream, - ) - check_cuda_error(err, f"prefetch layer {layer_idx} batch") - - @nvtx_range("dwdp_prefetch_layer") - def prefetch_layer(self, layer_idx: int, wait_compute_layer_idx: Optional[int] = None): - """ - Prefetch layer data from peer ranks. - - Args: - layer_idx: The layer to prefetch - wait_compute_layer_idx: If provided, wait for this layer's compute to complete - before overwriting buffer (used when prefetching next layer) - - Note: Local weights are used directly by the kernel, no copy needed. - Peer copy runs on prefetch stream. - """ - moe_idx = layer_idx - self.first_moe_layer_idx - collector = self.ipc_collectors[moe_idx] - buffer_idx = layer_idx % self.prefetch_buffer.num_buffers - - with torch.cuda.stream(self.prefetch_buffer.prefetch_stream): - if wait_compute_layer_idx is not None: - self.prefetch_buffer.wait_compute_event(wait_compute_layer_idx) - - if self.use_batched_prefetch: - self._prefetch_layer_batched(layer_idx, collector, buffer_idx) - else: - self._prefetch_layer_default(layer_idx, collector, buffer_idx) + return self._weight_manager + + # ------------------------------------------------------------------ + # Runtime entry points (forwarded to DWDPWeightManager) + # ------------------------------------------------------------------ + + def prefetch_first_layers(self) -> None: + """Warm-up: kick off prefetch for the first MoE layers (double-buffer depth).""" + if self._weight_manager is None: + raise RuntimeError("DwdpManager.setup() has not been called yet") + first = self._weight_manager.first_moe_layer() + self._weight_manager.prefetch_layer(first) + second = self._weight_manager.next_moe_layer(first) + if second is not None: + self._weight_manager.prefetch_layer(second) + + def wait_and_bind(self, backend_module: nn.Module, layer_idx: int) -> None: + """Wait for prefetch and bind full-tensor views onto backend weights.""" + if self._weight_manager is None: + raise RuntimeError("DwdpManager.setup() has not been called yet") + self._weight_manager.wait_and_bind(backend_module, layer_idx) + + def record_compute_and_prefetch_next(self, layer_idx: int) -> None: + """After the current layer finishes compute, kick off the next layer's prefetch. + + The WAR signal for the "consumed" slot is recorded inside + ``wait_and_bind``; here we just schedule the next prefetch. + """ + if self._weight_manager is None: + raise RuntimeError("DwdpManager.setup() has not been called yet") + next_idx = self._weight_manager.next_moe_layer(layer_idx) + if next_idx is not None: + self._weight_manager.prefetch_layer(next_idx) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index a8fe091756d6..39de33f8f51a 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -13,7 +13,7 @@ import tensorrt_llm from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManagerType -from tensorrt_llm._utils import get_sm_version +from tensorrt_llm._utils import get_sm_version, global_mpi_rank from tensorrt_llm.llmapi.llm_args import (CapacitySchedulerPolicy, ContextChunkingPolicy, ExecutorMemoryType, @@ -432,6 +432,24 @@ def create_py_executor( "when only processing vision encoder inputs.") mapping = _get_mapping(llm_args.parallel_config.to_mapping()) + + # Bridge DwdpConfig -> Mapping: ParallelConfig.to_mapping() doesn't know + # about dwdp_size/dwdp_rank, so inject them here (before anything reads + # mapping.dwdp_enabled / moe_ep_rank). + # + # dwdp_rank MUST be derived from the global MPI rank, not mapping.rank: + # in disaggregated serving each context worker is its own TP=1 instance + # with mapping.rank=0, so `mapping.rank % dwdp_size` would make every + # worker think it is DWDP rank 0 (causing e.g. cuMemMap to self-import a + # peer handle and fail with CUDA_ERROR_NOT_SUPPORTED). + if llm_args.dwdp_config is not None and llm_args.dwdp_config.dwdp_size > 1: + dwdp_size = llm_args.dwdp_config.dwdp_size + dwdp_rank = global_mpi_rank() % dwdp_size + mapping = Mapping(**{ + **mapping.to_dict(), "dwdp_size": dwdp_size, + "dwdp_rank": dwdp_rank + }) + dist = Distributed.get(mapping) vm_pools = {} @@ -473,7 +491,9 @@ def create_py_executor( dwdp_manager: Optional[DwdpManager] = None if llm_args.dwdp_config is not None: assert mapping.tp_size == 1 and llm_args.dwdp_config.dwdp_size > 1, "DWDP requires TP=1 and dwdp_size > 1" - dwdp_manager = DwdpManager(config=llm_args.dwdp_config, dist=dist) + dwdp_manager = DwdpManager(config=llm_args.dwdp_config, + dist=dist, + mapping=mapping) dwdp_manager.__enter__() logger.info(f"Dwdp Manager initialized. Config: {llm_args.dwdp_config}") @@ -882,10 +902,10 @@ def drafting_loop_wrapper(model): max_seq_len = kv_cache_creator._max_seq_len update_sampler_max_seq_len(max_seq_len, sampler) - # Exchange IPC Handles and Initialize Dwdp Prefetch Buffer + # DWDP setup: MNNVL handle exchange + composite VA weight buffer + + # weight manager + MoE backend fixup (single entry point). if dwdp_manager is not None: - dwdp_manager.exchange_all_handles() - dwdp_manager.initialize_prefetch_buffer() + dwdp_manager.setup(model_engine.model) # Resource managers for speculative decoding # For user-specified drafters, use extra_resource_managers in PyTorchBackend config diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index c361686370ca..2c3aad405890 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -2838,11 +2838,13 @@ class DwdpConfig(StrictBaseModel): DWDP accelerates the context (prefill) phase of disaggregated MoE serving by combining data parallelism with NVLink-based expert weight sharing. Each worker holds a subset of experts locally and asynchronously prefetches - the remaining experts from peer workers via CUDA IPC, enabling fully - asynchronous execution across ranks without synchronization barriers. + the remaining experts from peer workers via CUDA VMM + MNNVL fabric + handles (composite virtual-address layout with zero-copy local mapping and + double-buffered P2P remote regions), enabling fully asynchronous execution + across ranks without synchronization barriers. Currently supported with the CuteDSL MoE backend and NVFP4 quantization - on NVLink-connected multi-GPU systems. + on MNNVL-connected multi-GPU systems (e.g., GB200). """ dwdp_size: int = Field(default=1, description="The number of GPUs per DWDP group.") diff --git a/tensorrt_llm/mapping.py b/tensorrt_llm/mapping.py index 74d3e02ee7e7..b5cabcd506e0 100644 --- a/tensorrt_llm/mapping.py +++ b/tensorrt_llm/mapping.py @@ -57,7 +57,19 @@ def __init__( attn_tp_size=-1, attn_cp_size=-1, enable_attention_dp=False, - enable_lm_head_tp_in_adp=False): + enable_lm_head_tp_in_adp=False, + dwdp_size=0, + dwdp_rank=0): + # Validate and store DWDP configuration + if dwdp_size < 0 or dwdp_size == 1: + raise ValueError( + f"dwdp_size must be 0 (disabled) or >= 2, got {dwdp_size}") + if dwdp_size > 1 and (dwdp_rank < 0 or dwdp_rank >= dwdp_size): + raise ValueError( + f"dwdp_rank must be in [0, {dwdp_size}), got {dwdp_rank}") + self._dwdp_size = dwdp_size + self._dwdp_rank = dwdp_rank + # set default values for non-moe cases # or where only one MOE parallelism size is specified if moe_cluster_size == -1: @@ -80,15 +92,32 @@ def __init__( moe_world_size = tp_size if cp_type == CpType.ULYSSES else tp_size * cp_size - if moe_tp_size == -1 and moe_ep_size == -1: - moe_tp_size = moe_world_size // moe_cluster_size + # DWDP override: when dwdp_size > 1, decouple the fused-MoE EP layout + # from DWDP. Setting ``moe_ep_size = 1`` makes the fused MoE backend + # see a single (full) expert table; the DWDP-specific layout + # (``slot_start`` / ``slot_end`` / ``expert_size_per_partition``) is + # installed by ``_init_dwdp_expert_layout`` at the end of + # ``MoE.__init__`` using ``DwdpManager.start_expert_id`` and + # ``num_experts_per_worker``. This is what unlocks dwdp_size values + # that do not divide ``num_experts`` evenly (e.g. dwdp_size = 3, 5) + # and the IPC-era redundancy mode where adjacent peer ranges overlap. + if dwdp_size > 1: + moe_tp_size = 1 moe_ep_size = 1 + moe_cluster_size = 1 + self._dwdp_moe_ep_rank = 0 + else: + self._dwdp_moe_ep_rank = 0 + + if moe_tp_size == -1 and moe_ep_size == -1: + moe_tp_size = moe_world_size // moe_cluster_size + moe_ep_size = 1 - elif moe_tp_size == -1: - moe_tp_size = moe_world_size // (moe_ep_size * moe_cluster_size) + elif moe_tp_size == -1: + moe_tp_size = moe_world_size // (moe_ep_size * moe_cluster_size) - elif moe_ep_size == -1: - moe_ep_size = moe_world_size // (moe_tp_size * moe_cluster_size) + elif moe_ep_size == -1: + moe_ep_size = moe_world_size // (moe_tp_size * moe_cluster_size) if attn_tp_size == -1 and attn_cp_size == -1: if cp_type == CpType.ULYSSES: @@ -117,8 +146,16 @@ def __init__( f"but got {world_size} != {tp_size} * {pp_size} * {cp_size}.") moe_tp_ep_size = moe_tp_size * moe_ep_size - self.moe_tp_cluster_ep_size = moe_tp_ep_size * moe_cluster_size - if self.moe_tp_cluster_ep_size != moe_world_size: + if dwdp_size > 1: + # DWDP: with moe_ep_size = 1, moe_tp_cluster_ep_size from the + # fused-MoE perspective is just ``moe_cluster_size``. The DWDP + # group is orthogonal to MoE TP/EP world, so skip the legacy + # equality check below. + self.moe_tp_cluster_ep_size = moe_cluster_size + else: + self.moe_tp_cluster_ep_size = moe_tp_ep_size * moe_cluster_size + if not (dwdp_size + > 1) and self.moe_tp_cluster_ep_size != moe_world_size: raise ValueError( "moe_tp_size * moe_ep_size * moe_cluster_size must equal to moe_world_size, " f"but got {self.moe_tp_cluster_ep_size} != {moe_world_size}") @@ -177,7 +214,9 @@ def __eq__(self, other): and self.moe_ep_size == other.moe_ep_size and self.attn_tp_size == other.attn_tp_size and self.attn_cp_size == other.attn_cp_size - and self.cp_config == other.cp_config) + and self.cp_config == other.cp_config + and self._dwdp_size == other._dwdp_size + and self._dwdp_rank == other._dwdp_rank) def __hash__(self): return hash(( @@ -195,6 +234,8 @@ def __hash__(self): # note: we do not allow updating cp_config after initialization tuple(sorted(self.cp_config.items())), tuple(self.pp_partition) if self.pp_partition is not None else (), + self._dwdp_size, + self._dwdp_rank, )) @property @@ -221,8 +262,22 @@ def moe_cluster_rank(self): @property def moe_ep_rank(self): + if self._dwdp_size > 1: + return self._dwdp_moe_ep_rank return self.tp_rank % self.moe_ep_size + @property + def dwdp_size(self) -> int: + return self._dwdp_size + + @property + def dwdp_rank(self) -> int: + return self._dwdp_rank + + @property + def dwdp_enabled(self) -> bool: + return self._dwdp_size > 1 + @property def moe_cluster_group(self): return self.moe_cluster_groups[self.pp_rank * self.moe_tp_size + @@ -390,6 +445,8 @@ def to_dict(self): 'cp_config': self.cp_config, 'enable_attention_dp': self.enable_attention_dp, 'enable_lm_head_tp_in_adp': self.enable_lm_head_tp_in_adp, + 'dwdp_size': self._dwdp_size, + 'dwdp_rank': self._dwdp_rank, } @@ -513,7 +570,9 @@ def __init__( attn_tp_size=-1, attn_cp_size=-1, enable_attention_dp=False, - enable_lm_head_tp_in_adp=False): + enable_lm_head_tp_in_adp=False, + dwdp_size=0, + dwdp_rank=0): super().__init__(world_size=world_size, rank=rank, gpus_per_node=gpus_per_node, @@ -528,7 +587,9 @@ def __init__( attn_tp_size=attn_tp_size, attn_cp_size=attn_cp_size, enable_attention_dp=enable_attention_dp, - enable_lm_head_tp_in_adp=enable_lm_head_tp_in_adp) + enable_lm_head_tp_in_adp=enable_lm_head_tp_in_adp, + dwdp_size=dwdp_size, + dwdp_rank=dwdp_rank) def repurpose_helix_cp_to_tp(self): # In helix parallelism, CP is relevant only for the attention layer. These ranks are repurposed to TP @@ -548,7 +609,9 @@ def repurpose_helix_cp_to_tp(self): moe_ep_size=self.moe_ep_size, # attn_tp_size, attn_cp_size shall be set in the constructor of Mapping. enable_attention_dp=self.enable_attention_dp, - enable_lm_head_tp_in_adp=self.enable_lm_head_tp_in_adp) + enable_lm_head_tp_in_adp=self.enable_lm_head_tp_in_adp, + dwdp_size=self._dwdp_size, + dwdp_rank=self._dwdp_rank) # DeviceMesh specific methods @property diff --git a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py index ce1644c04711..cb98648cfab8 100644 --- a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py @@ -21,7 +21,7 @@ from tensorrt_llm.llmapi.llm_args import LlmArgs from tensorrt_llm.llmapi.tokenizer import load_hf_tokenizer -from ..conftest import llm_models_root, skip_pre_blackwell +from ..conftest import llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell from ..trt_test_alternative import popen from .accuracy_core import LlmapiAccuracyTestHarness from .test_disaggregated_serving import ( @@ -181,6 +181,7 @@ class TestDwdpDeepSeekV3Lite(LlmapiAccuracyTestHarness): @pytest.mark.skip_less_device(4) @skip_pre_blackwell + @skip_post_blackwell_ultra def test_dwdp_accuracy(self): model_path = f"{llm_models_root()}/DeepSeek-V3-Lite/nvfp4_moe_only_mtp" @@ -280,3 +281,257 @@ def test_dwdp_accuracy(self): worker_config, frontend_config, model_path, total_gpus=4, max_workers=128 ) as llm: run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + + @pytest.mark.skip_less_device(4) + @skip_pre_blackwell + @skip_post_blackwell_ultra + def test_dwdp_accuracy_contention_opt(self): + """End-to-end accuracy test with ``contention_opt=True``. + + Mirrors ``test_dwdp_accuracy`` (uniform partition, dwdp_size=2, + GEN TP=2 on 4 GPU) but turns on the contention optimization so + that ``DWDPWeightManager.prefetch_layer`` exercises the batched + ``cudaMemcpyBatchAsync`` code path with 2 MiB sub-slices + round-robined across peers. Pass criterion is the same GSM8K + threshold (61.537) — the batched path must produce + bit-for-bit equivalent results to the per-slice default path, + since the only difference is the schedule / launch shape of + the same set of D2D copies. + """ + model_path = f"{llm_models_root()}/DeepSeek-V3-Lite/nvfp4_moe_only_mtp" + + ctx_port_0 = get_free_port() + ctx_port_1 = get_free_port() + gen_port = get_free_port() + serve_port = get_free_port() + + ctx_server_config = { + "num_instances": 2, + "urls": [ + f"localhost:{ctx_port_0}", + f"localhost:{ctx_port_1}", + ], + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "disable_overlap_scheduler": True, + "enable_autotuner": False, + "enable_chunked_prefill": False, + "cuda_graph_config": None, + "max_batch_size": 16, + "max_num_tokens": 8192, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.4, + "enable_block_reuse": False, + "enable_partial_reuse": False, + "tokens_per_block": 32, + }, + "cache_transceiver_config": { + "backend": "UCX", + "max_tokens_in_buffer": 8192, + }, + "moe_config": { + "backend": "CUTEDSL", + }, + "dwdp_config": { + "dwdp_size": 2, + "num_groups": 1, + "num_experts_per_worker": 36, + "num_prefetch_experts": 36, + "contention_opt": True, + }, + } + + gen_server_config = { + "num_instances": 1, + "urls": [f"localhost:{gen_port}"], + "tensor_parallel_size": 2, + "pipeline_parallel_size": 1, + "disable_overlap_scheduler": True, + "enable_autotuner": False, + "enable_chunked_prefill": False, + "cuda_graph_config": None, + "max_batch_size": 128, + "max_num_tokens": 1024, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "enable_partial_reuse": False, + "tokens_per_block": 32, + }, + "cache_transceiver_config": { + "backend": "UCX", + "max_tokens_in_buffer": 8192, + }, + "moe_config": { + "backend": "CUTEDSL", + }, + } + + worker_config = { + "model": model_path, + "hostname": "localhost", + "port": serve_port, + "backend": "pytorch", + "context_servers": ctx_server_config, + "generation_servers": gen_server_config, + } + + frontend_config = { + "backend": "pytorch", + "hostname": "localhost", + "port": serve_port, + "context_servers": { + "num_instances": 2, + "urls": [ + f"localhost:{ctx_port_0}", + f"localhost:{ctx_port_1}", + ], + }, + "generation_servers": { + "num_instances": 1, + "urls": [f"localhost:{gen_port}"], + }, + } + + with launch_dwdp_disaggregated_llm( + worker_config, frontend_config, model_path, total_gpus=4, max_workers=128 + ) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + + @pytest.mark.skip_less_device(4) + @skip_pre_blackwell + @skip_post_blackwell_ultra + def test_dwdp_accuracy_mode_b_overlap(self): + """End-to-end accuracy test for Phase 2's redundancy (overlap) path. + + DSv3-Lite has 72 experts. The companion ``test_dwdp_accuracy`` + uses the uniform Mode-B-equality config (size=36, stride=36, 0 + overlap). This test deliberately uses ``size=40, stride=32`` + (``(dwdp_size - 1) * stride + size = 1*32 + 40 == 72`` strict + equality, 8-expert overlap between rank 0 and rank 1) so that + the runtime exercises: + + * ``_init_dwdp_expert_layout`` overrides + ``expert_size_per_partition`` to 40 (not 72 // 2 = 36); + * peer ranges ``[(0, 40), (32, 72)]`` have a non-empty + 8-expert overlap, exercising ``lookup_owner``'s first-match + policy in fill_edge_bytes / weight_manager / scatter; + * ``_scatter_shards_to_full`` writes overlapping experts + (32..39) twice — once from rank 0 (lowest, wins) and once + from rank 1 — and values must agree because every rank that + owns expert ``e`` loaded ``e`` from the same checkpoint. + + Passing GSM8K at the standard threshold is sufficient evidence + that the runtime fetches every remote expert from the right + peer and that ``fixup_moe_backends`` reconstructs the gate-side + scale tensors correctly under overlap. + + Note: ``dwdp_size=2`` is kept (vs. the original dwdp=3 design) + to match the existing ``test_dwdp_accuracy``'s 4-GPU resource + footprint (2 CTX TP=1 + 1 GEN TP=2 = 4 GPUs) and to avoid an + unrelated UCX bug observed when running 3 CTX workers + (``libtensorrt_llm_ucx_wrapper.so`` bus error in the worker + progress thread; affects KV-cache transceiver, not DWDP). + """ + model_path = f"{llm_models_root()}/DeepSeek-V3-Lite/nvfp4_moe_only_mtp" + + ctx_port_0 = get_free_port() + ctx_port_1 = get_free_port() + gen_port = get_free_port() + serve_port = get_free_port() + + ctx_server_config = { + "num_instances": 2, + "urls": [ + f"localhost:{ctx_port_0}", + f"localhost:{ctx_port_1}", + ], + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "disable_overlap_scheduler": True, + "enable_autotuner": False, + "enable_chunked_prefill": False, + "cuda_graph_config": None, + "max_batch_size": 16, + "max_num_tokens": 8192, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.4, + "enable_block_reuse": False, + "enable_partial_reuse": False, + "tokens_per_block": 32, + }, + "cache_transceiver_config": { + "backend": "UCX", + "max_tokens_in_buffer": 8192, + }, + "moe_config": { + "backend": "CUTEDSL", + }, + "dwdp_config": { + "dwdp_size": 2, + "num_groups": 1, + # Mode B redundancy: stride=32 < size=40. Strict + # equality 1*32 + 40 == 72 is enforced by + # _validate_partition_config; 8-expert overlap between + # rank 0's range [0, 40) and rank 1's range [32, 72). + "num_experts_per_worker": 40, + "num_prefetch_experts": 32, + }, + } + + gen_server_config = { + "num_instances": 1, + "urls": [f"localhost:{gen_port}"], + "tensor_parallel_size": 2, + "pipeline_parallel_size": 1, + "disable_overlap_scheduler": True, + "enable_autotuner": False, + "enable_chunked_prefill": False, + "cuda_graph_config": None, + "max_batch_size": 128, + "max_num_tokens": 1024, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "enable_partial_reuse": False, + "tokens_per_block": 32, + }, + "cache_transceiver_config": { + "backend": "UCX", + "max_tokens_in_buffer": 8192, + }, + "moe_config": { + "backend": "CUTEDSL", + }, + } + + worker_config = { + "model": model_path, + "hostname": "localhost", + "port": serve_port, + "backend": "pytorch", + "context_servers": ctx_server_config, + "generation_servers": gen_server_config, + } + + frontend_config = { + "backend": "pytorch", + "hostname": "localhost", + "port": serve_port, + "context_servers": { + "num_instances": 2, + "urls": [ + f"localhost:{ctx_port_0}", + f"localhost:{ctx_port_1}", + ], + }, + "generation_servers": { + "num_instances": 1, + "urls": [f"localhost:{gen_port}"], + }, + } + + with launch_dwdp_disaggregated_llm( + worker_config, frontend_config, model_path, total_gpus=4, max_workers=128 + ) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 63d144aeac69..785e65d22078 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -101,6 +101,8 @@ accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-False] accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_nvfp4[False] diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 0b3cceab299f..2c4bc9268522 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -32,6 +32,10 @@ l0_a10: - unittest/_torch/executor/test_kv_cache_estimation.py - unittest/_torch/executor/test_kv_cache_budget_split.py - unittest/_torch/executor/test_disagg_index_mapper_early_release.py + - unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py + - unittest/_torch/modules/dwdp/test_dwdp_manager.py + - unittest/_torch/modules/dwdp/test_dwdp_mapping.py + - unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py # NOTE: this is a CPU-only test, but we do not have a dedicated job for this (and therefore no # test list either). - unittest/_torch/models/checkpoints/hf/test_weight_loader.py diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index ae0e24ff2d2b..9427774e2255 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -92,6 +92,8 @@ l0_gb200_multi_gpus: - accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-trtllm] - accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] TIMEOUT (90) - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy + - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt + - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_postquant diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 8d20bc229c5a..30da247dcaa2 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -4,7 +4,6 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6184575) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) -accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6094102) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph SKIP (https://nvbugs/5772995) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] SKIP (https://nvbugs/5346443) accuracy/test_llm_api.py::TestMistralNemo12B::test_fp8 SKIP (https://nvbugs/5413197) diff --git a/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_gather_grouped_gemm_act_fusion.py b/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_gather_grouped_gemm_act_fusion.py index 87d2d5b33ee3..b942f2d4139b 100644 --- a/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_gather_grouped_gemm_act_fusion.py +++ b/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_gather_grouped_gemm_act_fusion.py @@ -92,37 +92,6 @@ cvt_sf_M32x4xrm_K4xrk_L_to_MKL = kernel_module.cvt_sf_M32x4xrm_K4xrk_L_to_MKL -def split_groups_to_b_tensors( - num_groups: int, num_b_tensors: int -) -> Tuple[Tuple[int, ...], Tuple[Tuple[int, ...], ...]]: - """Split groups into multiple B tensors. - - :param num_groups: Total number of groups (experts) - :param num_b_tensors: Number of B tensors to split into - :return: Tuple of (b_tensor_l_sizes, groups_per_b_tensor) - - b_tensor_l_sizes: L size for each B tensor - - groups_per_b_tensor: Tuple of group indices for each B tensor - """ - # Distribute groups evenly across B tensors - base_groups_per_tensor = num_groups // num_b_tensors - remainder = num_groups % num_b_tensors - - b_tensor_l_sizes = [] - groups_per_b_tensor = [] - current_group = 0 - - for i in range(num_b_tensors): - # Add one extra group to first 'remainder' tensors - num_groups_in_tensor = base_groups_per_tensor + (1 if i < remainder else 0) - b_tensor_l_sizes.append(num_groups_in_tensor) - groups_per_b_tensor.append( - tuple(range(current_group, current_group + num_groups_in_tensor)) - ) - current_group += num_groups_in_tensor - - return tuple(b_tensor_l_sizes), tuple(groups_per_b_tensor) - - def create_mask(group_m_list, mma_tiler_m, permuted_m=None): """Create mask and group mapping for contiguous grouped GEMM. @@ -406,8 +375,6 @@ def create_tensors( sf_vec_size, mma_tiler_m, permuted_m=None, - b_tensor_l_sizes=None, - groups_per_b_tensor=None, ): """Create tensors for contiguous grouped GEMM with gather operation and SwiGLU fusion. @@ -416,7 +383,7 @@ def create_tensors( Returns tensors including: - A: Input matrix (MxKx1) - - B: Weight matrix with interleaved up/gate weights (NxKxL) or list of tensors for multi-B + - B: Weight matrix with interleaved up/gate weights (NxKxL) - C: Output matrix (Mx(N/2)x1), N is halved due to SwiGLU fusion - SFA, SFB: Scale factor matrices for A and B - SFC: Scale factor matrix for C (only when c_dtype is Float4E2M1FN) @@ -426,14 +393,9 @@ def create_tensors( :param mma_tiler_m: MMA tile size in M dimension (from mma_tiler_mn[0]), also used for alignment :param permuted_m: Optional padded M dimension for cuda_graph support. - :param b_tensor_l_sizes: Optional tuple of L sizes for multi-B tensor mode. - :param groups_per_b_tensor: Optional tuple of group indices for each B tensor. """ torch.manual_seed(1111) - # Determine if multi-B tensor mode - multi_b_mode = b_tensor_l_sizes is not None - ( valid_m, aligned_group_m_list, @@ -467,71 +429,21 @@ def create_tensors( divisibility=32 if ab_dtype == cutlass.Float4E2M1FN else 16, ) - if multi_b_mode: - # Multi-B tensor mode: create multiple B tensors - b_torch_cpu_list = [] - b_tensor_list = [] - b_torch_gpu_list = [] - sfb_torch_cpu_list = [] - sfb_tensor_list = [] - sfb_torch_gpu_list = [] - alpha_torch_cpu_list = [] - alpha_tensor_list = [] - - for l_size in b_tensor_l_sizes: - # Create alpha for this B tensor - alpha_torch_cpu = torch.randn((l_size,), dtype=torch.float32) - alpha_torch_cpu_list.append(alpha_torch_cpu) - alpha = from_dlpack(alpha_torch_cpu.cuda()).mark_layout_dynamic() - alpha_tensor_list.append(alpha) - - # Create B tensor - b_torch_cpu = cutlass_torch.matrix(l_size, n, k, b_major == "n", cutlass.Float32) - b_tensor, b_torch_gpu = cutlass_torch.cute_tensor_like( - b_torch_cpu, ab_dtype, is_dynamic_layout=True, assumed_align=16 - ) - b_tensor.mark_compact_shape_dynamic( - mode=1 if b_major == "k" else 0, - stride_order=(2, 0, 1) if b_major == "k" else (2, 1, 0), - divisibility=32 if ab_dtype == cutlass.Float4E2M1FN else 16, - ) - b_torch_cpu_list.append(b_torch_cpu) - b_tensor_list.append(b_tensor) - b_torch_gpu_list.append(b_torch_gpu) - - # Create SFB tensor - sfb_torch_cpu, sfb_tensor, sfb_torch_gpu = create_scale_factor_tensor( - l_size, n, k, sf_vec_size, sf_dtype - ) - sfb_torch_cpu_list.append(sfb_torch_cpu) - sfb_tensor_list.append(sfb_tensor) - sfb_torch_gpu_list.append(sfb_torch_gpu) - - b_tensor = b_tensor_list - b_torch_cpu = b_torch_cpu_list - b_torch_gpu = b_torch_gpu_list - sfb_tensor = sfb_tensor_list - sfb_torch_cpu = sfb_torch_cpu_list - sfb_torch_gpu = sfb_torch_gpu_list - alpha_torch_cpu = alpha_torch_cpu_list - alpha = alpha_tensor_list - else: - # Single B tensor mode - alpha_torch_cpu = torch.randn((num_groups,), dtype=torch.float32) - alpha = from_dlpack(alpha_torch_cpu.cuda()).mark_layout_dynamic() + alpha_torch_cpu = torch.randn((num_groups,), dtype=torch.float32) + alpha = from_dlpack(alpha_torch_cpu.cuda()).mark_layout_dynamic() - b_torch_cpu = cutlass_torch.matrix(num_groups, n, k, b_major == "n", cutlass.Float32) - b_tensor, b_torch_gpu = cutlass_torch.cute_tensor_like( - b_torch_cpu, ab_dtype, is_dynamic_layout=True, assumed_align=16 - ) - b_tensor.mark_compact_shape_dynamic( - mode=1 if b_major == "k" else 0, - stride_order=(2, 0, 1) if b_major == "k" else (2, 1, 0), - divisibility=32 if ab_dtype == cutlass.Float4E2M1FN else 16, - ) - sfb_torch_cpu, sfb_tensor, sfb_torch_gpu = create_scale_factor_tensor( - num_groups, n, k, sf_vec_size, sf_dtype - ) + b_torch_cpu = cutlass_torch.matrix(num_groups, n, k, b_major == "n", cutlass.Float32) + b_tensor, b_torch_gpu = cutlass_torch.cute_tensor_like( + b_torch_cpu, ab_dtype, is_dynamic_layout=True, assumed_align=16 + ) + b_tensor.mark_compact_shape_dynamic( + mode=1 if b_major == "k" else 0, + stride_order=(2, 0, 1) if b_major == "k" else (2, 1, 0), + divisibility=32 if ab_dtype == cutlass.Float4E2M1FN else 16, + ) + sfb_torch_cpu, sfb_tensor, sfb_torch_gpu = create_scale_factor_tensor( + num_groups, n, k, sf_vec_size, sf_dtype + ) # Use tensor_m (permuted_m if provided) for scale factor A sfa_torch_cpu, sfa_tensor, sfa_torch_gpu = create_scale_factor_tensor_unswizzled( @@ -619,7 +531,6 @@ def run( permuted_m: int = None, use_cupti: bool = False, raster_along_m: bool = False, - num_b_tensors: int = None, **kwargs, ): """Run contiguous grouped GEMM with gather operation and SwiGLU fusion for FC1 layer. @@ -630,17 +541,10 @@ def run( 3. Optional Quant: When c_dtype is Float4E2M1FN, generates SFC and quantizes output Note: Output C has N/2 columns since SwiGLU combines pairs of (up, gate) from interleaved B weights. - - :param num_b_tensors: If specified, enables multi-B tensor mode (2-4 tensors). """ - # Determine if multi-B tensor mode - multi_b_mode = num_b_tensors is not None - print("Running Blackwell Persistent Contiguous Grouped GEMM with Gather test:") print(f"nkl: {nkl}") print(f"group_m_list: {group_m_list}") - if multi_b_mode: - print(f"Multi-B tensor mode: {num_b_tensors} B tensors") print( f"AB dtype: {ab_dtype}, C dtype: {c_dtype}, Acc dtype: {acc_dtype}, " f"Scale factor dtype: {sf_dtype}, SF Vec size: {sf_vec_size}" @@ -665,14 +569,6 @@ def run( if not torch.cuda.is_available(): raise RuntimeError("GPU is required to run this example!") - # Split groups into multiple B tensors if multi-B mode - b_tensor_l_sizes = None - groups_per_b_tensor = None - if multi_b_mode: - b_tensor_l_sizes, groups_per_b_tensor = split_groups_to_b_tensors(num_groups, num_b_tensors) - print(f"b_tensor_l_sizes: {b_tensor_l_sizes}") - print(f"groups_per_b_tensor: {groups_per_b_tensor}") - # Skip unsupported testcase # Note: For grouped GEMM, we use mma_tiler_mn[0] as the m parameter for can_implement check # since individual group M values vary @@ -742,8 +638,6 @@ def run( sf_vec_size, mma_tiler_mn[0], # mma_tiler_m, also used for alignment permuted_m, - b_tensor_l_sizes=b_tensor_l_sizes, - groups_per_b_tensor=groups_per_b_tensor, ) # Configure gemm kernel @@ -754,7 +648,6 @@ def run( True, topk=1, raster_along_m=raster_along_m, - b_tensor_l_sizes=b_tensor_l_sizes if multi_b_mode else None, ) # Compute max active clusters on current device @@ -769,78 +662,40 @@ def run( current_stream = cuda.CUstream(torch_stream.cuda_stream) # Compile gemm kernel # sfc_tensor is optional and can be set as None (Python's None value) if not needed. - if multi_b_mode: - # Multi-B tensor mode: pass tuples - compiled_gemm = cute.compile( - gemm, - a_tensor, - tuple(b_tensor), - c_tensor, - sfa_tensor, - tuple(sfb_tensor), - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - tuple(alpha), - max_active_clusters, - current_stream, - ) - else: - # Single-B tensor mode - compiled_gemm = cute.compile( - gemm, - a_tensor, - b_tensor, - c_tensor, - sfa_tensor, - sfb_tensor, - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - alpha, - max_active_clusters, - current_stream, - ) + compiled_gemm = cute.compile( + gemm, + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + sfc_tensor, + norm_const_tensor, + tile_idx_to_expert_idx, + tile_idx_to_mn_limit, + token_id_mapping, + num_non_exiting_tiles, + alpha, + max_active_clusters, + current_stream, + ) # Execution - if multi_b_mode: - compiled_gemm( - a_tensor, - tuple(b_tensor), - c_tensor, - sfa_tensor, - tuple(sfb_tensor), - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - tuple(alpha), - current_stream, - ) - else: - compiled_gemm( - a_tensor, - b_tensor, - c_tensor, - sfa_tensor, - sfb_tensor, - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - alpha, - current_stream, - ) + compiled_gemm( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + sfc_tensor, + norm_const_tensor, + tile_idx_to_expert_idx, + tile_idx_to_mn_limit, + token_id_mapping, + num_non_exiting_tiles, + alpha, + current_stream, + ) torch.cuda.synchronize() # Compute reference result @@ -859,25 +714,8 @@ def run( end = start + group_m res_a = a_torch_cpu_f32[token_id_mapping_cpu[start:end]] - if multi_b_mode: - # Find which B tensor this group belongs to - b_tensor_idx = None - local_group_idx = None - for b_idx, groups in enumerate(groups_per_b_tensor): - if i in groups: - b_tensor_idx = b_idx - local_group_idx = groups.index(i) - break - assert b_tensor_idx is not None, f"Group {i} not found in any B tensor" - res_b = torch.einsum( - "nk,nk->nk", - b_torch_cpu[b_tensor_idx][:, :, local_group_idx], - sfb_torch_cpu[b_tensor_idx][:, :, local_group_idx], - ) - alpha_val = alpha_torch_cpu[b_tensor_idx][local_group_idx] - else: - res_b = torch.einsum("nk,nk->nk", b_torch_cpu[:, :, i], sfb_torch_cpu[:, :, i]) - alpha_val = alpha_torch_cpu[i] + res_b = torch.einsum("nk,nk->nk", b_torch_cpu[:, :, i], sfb_torch_cpu[:, :, i]) + alpha_val = alpha_torch_cpu[i] gemm_result[0, start:end, :] = torch.einsum("mk,nk->mn", res_a, res_b) * alpha_val start = end @@ -1160,54 +998,30 @@ def generate_tensors(): sf_vec_size, mma_tiler_mn[0], # mma_tiler_m, also used for alignment permuted_m, - b_tensor_l_sizes=b_tensor_l_sizes, - groups_per_b_tensor=groups_per_b_tensor, ) - if multi_b_mode: - return cute.testing.JitArguments( - a_tensor, - tuple(b_tensor), - c_tensor, - sfa_tensor, - tuple(sfb_tensor), - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - tuple(alpha), - current_stream, - ) - else: - return cute.testing.JitArguments( - a_tensor, - b_tensor, - c_tensor, - sfa_tensor, - sfb_tensor, - sfc_tensor, - norm_const_tensor, - tile_idx_to_expert_idx, - tile_idx_to_mn_limit, - token_id_mapping, - num_non_exiting_tiles, - alpha, - current_stream, - ) + return cute.testing.JitArguments( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + sfc_tensor, + norm_const_tensor, + tile_idx_to_expert_idx, + tile_idx_to_mn_limit, + token_id_mapping, + num_non_exiting_tiles, + alpha, + current_stream, + ) workspace_count = 1 if use_cold_l2: # Calculate actual tensor_m used (with padding if permuted_m provided) tensor_m = permuted_m if permuted_m is not None else valid_m - if multi_b_mode: - b_bytes = sum(t.numel() * t.element_size() for t in b_torch_gpu) - sfb_bytes = sum(t.numel() * t.element_size() for t in sfb_torch_gpu) - alpha_bytes = sum(t.numel() * t.element_size() for t in alpha_torch_cpu) - else: - b_bytes = b_torch_gpu.numel() * b_torch_gpu.element_size() - sfb_bytes = sfb_torch_gpu.numel() * sfb_torch_gpu.element_size() - alpha_bytes = alpha_torch_cpu.numel() * alpha_torch_cpu.element_size() + b_bytes = b_torch_gpu.numel() * b_torch_gpu.element_size() + sfb_bytes = sfb_torch_gpu.numel() * sfb_torch_gpu.element_size() + alpha_bytes = alpha_torch_cpu.numel() * alpha_torch_cpu.element_size() one_workspace_bytes = ( a_torch_gpu.numel() * a_torch_gpu.element_size() + b_bytes @@ -1380,13 +1194,6 @@ def read_benchmark_file( parser.add_argument( "--raster_along_m", action="store_true", default=False, help="Raster along M dimension" ) - parser.add_argument( - "--num_b_tensors", - type=int, - default=None, - help="Number of B tensors to split into (for multi-B tensor test). " - "If specified, enables multi-B tensor mode. Must be 2, 3, or 4.", - ) args = parser.parse_args() @@ -1421,16 +1228,6 @@ def read_benchmark_file( if len(args.cluster_shape_mn) != 2: parser.error("--cluster_shape_mn must contain exactly 2 values") - if args.num_b_tensors is not None: - if args.num_b_tensors < 2 or args.num_b_tensors > 4: - parser.error("--num_b_tensors must be 2, 3, or 4") - n, k, num_groups = nkl - if num_groups < args.num_b_tensors: - parser.error( - f"--num_b_tensors ({args.num_b_tensors}) cannot be greater than " - f"number of groups ({num_groups})" - ) - exec_time = run( nkl, group_m_list, @@ -1452,7 +1249,6 @@ def read_benchmark_file( args.permuted_m, args.use_cupti, args.raster_along_m, - args.num_b_tensors, ) print(f"Execution time: {exec_time:.2f} us") print("PASS") diff --git a/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_grouped_gemm_finalize_fusion.py b/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_grouped_gemm_finalize_fusion.py index 671cdc5db765..f140b76367db 100644 --- a/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_grouped_gemm_finalize_fusion.py +++ b/tests/scripts/cute_dsl_kernels/run_blockscaled_contiguous_grouped_gemm_finalize_fusion.py @@ -297,7 +297,6 @@ def create_tensors( mma_tiler_mn, permuted_m=None, seq_len=None, - b_tensor_l_sizes=None, ): """Create tensors for contiguous grouped GEMM. @@ -305,7 +304,6 @@ def create_tensors( A matrix, C matrix, and scale factor A will be padded to this size. The kernel exits when tile_idx >= num_non_exiting_tiles. :param seq_len: Sequence length (number of output tokens for C tensor) - :param b_tensor_l_sizes: Optional tuple of L sizes for multi-B tensor mode. :return: Tuple of (a_tensor, b_tensor, out_tensor, sfa_tensor, sfb_tensor, tile_idx_to_expert_idx, num_non_exiting_tiles, alpha, a_torch_cpu, b_torch_cpu, out_torch_cpu, sfa_torch_cpu, sfb_torch_cpu, @@ -333,11 +331,6 @@ def create_tensors( """ torch.manual_seed(1111) - multi_b_mode = b_tensor_l_sizes is not None - if multi_b_mode: - total_l = sum(b_tensor_l_sizes) - if total_l != l: - raise ValueError(f"Sum of b_tensor_l_sizes ({total_l}) must equal total L ({l}).") alpha_torch_cpu = torch.ones((l,), dtype=torch.float32) * 0.1 ( @@ -401,50 +394,6 @@ def create_tensors( out_torch_gpu.fill_(0) - if multi_b_mode: - b_torch_cpu_list = [] - b_tensor_list = [] - b_torch_gpu_list = [] - sfb_torch_cpu_list = [] - sfb_tensor_list = [] - sfb_torch_gpu_list = [] - alpha_torch_cpu_list = [] - alpha_tensor_list = [] - - for l_size in b_tensor_l_sizes: - alpha_cpu = torch.ones((l_size,), dtype=torch.float32) * 0.1 - alpha_torch_cpu_list.append(alpha_cpu) - alpha_tensor_list.append(from_dlpack(alpha_cpu.cuda()).mark_layout_dynamic()) - - b_cpu = cutlass_torch.matrix(l_size, n, k, b_major == "n", cutlass.Float32) - b_tensor_i, b_torch_gpu_i = cutlass_torch.cute_tensor_like( - b_cpu, ab_dtype, is_dynamic_layout=True, assumed_align=16 - ) - b_tensor_i.mark_compact_shape_dynamic( - mode=1 if b_major == "k" else 0, - stride_order=(2, 0, 1) if b_major == "k" else (2, 1, 0), - divisibility=32 if ab_dtype == cutlass.Float4E2M1FN else 16, - ) - b_torch_cpu_list.append(b_cpu) - b_tensor_list.append(b_tensor_i) - b_torch_gpu_list.append(b_torch_gpu_i) - - sfb_cpu, sfb_tensor_i, sfb_torch_gpu_i = create_scale_factor_tensor( - l_size, n, k, sf_vec_size, sf_dtype - ) - sfb_torch_cpu_list.append(sfb_cpu) - sfb_tensor_list.append(sfb_tensor_i) - sfb_torch_gpu_list.append(sfb_torch_gpu_i) - - b_tensor = b_tensor_list - b_torch_cpu = b_torch_cpu_list - b_torch_gpu = b_torch_gpu_list - sfb_tensor = sfb_tensor_list - sfb_torch_cpu = sfb_torch_cpu_list - sfb_torch_gpu = sfb_torch_gpu_list - alpha = alpha_tensor_list - alpha_torch_cpu = alpha_torch_cpu_list - return ( a_tensor, b_tensor, @@ -560,7 +509,6 @@ def run( raster_along_m: bool = False, use_blkred: bool = False, use_cupti: bool = False, - b_tensor_l_sizes=None, **kwargs, ): """Prepare A/B/C tensors, launch GPU kernel, and reference checking. @@ -618,10 +566,7 @@ def run( # Unpack parameters n, k, l = nkl # noqa: E741 - multi_b_mode = b_tensor_l_sizes is not None - total_l = sum(b_tensor_l_sizes) if multi_b_mode else l - if multi_b_mode and total_l != l: - raise ValueError(f"Sum of b_tensor_l_sizes ({total_l}) must equal L ({l}).") + total_l = l if not torch.cuda.is_available(): raise RuntimeError("GPU is required to run this example!") @@ -686,7 +631,6 @@ def run( mma_tiler_mn, # cta_tile_m permuted_m, seq_len, # Pass seq_len as num_tokens for C tensor shape - b_tensor_l_sizes=b_tensor_l_sizes, ) # Calculate actual tensor_m used (with padding if permuted_m provided) @@ -713,7 +657,6 @@ def run( cluster_shape_mn, use_blkred=use_blkred, raster_along_m=raster_along_m, - b_tensor_l_sizes=b_tensor_l_sizes if multi_b_mode else None, ) # Compute max active clusters on current device @@ -726,27 +669,29 @@ def run( current_stream = cutlass_torch.default_stream() # Compile gemm kernel - if multi_b_mode: - compiled_gemm = cute.compile( - gemm, - a_tensor, - tuple(b_tensor), - out_tensor, - sfa_tensor, - tuple(sfb_tensor), - tile_idx_to_expert_idx, - num_non_exiting_tiles, - tile_idx_to_mn_limit, - tuple(alpha), - max_active_clusters, - current_stream, - permuted_idx_to_expanded_idx, - token_final_scales, - options="--opt-level 2", - ) - else: - compiled_gemm = cute.compile( - gemm, + compiled_gemm = cute.compile( + gemm, + a_tensor, + b_tensor, + out_tensor, + sfa_tensor, + sfb_tensor, + tile_idx_to_expert_idx, + num_non_exiting_tiles, + tile_idx_to_mn_limit, + alpha, + max_active_clusters, + current_stream, + permuted_idx_to_expanded_idx, + token_final_scales, + options="--opt-level 2", + ) + + # Compute reference result + if not skip_ref_check: + print("Verifying results...") + # Execution + compiled_gemm( a_tensor, b_tensor, out_tensor, @@ -756,48 +701,11 @@ def run( num_non_exiting_tiles, tile_idx_to_mn_limit, alpha, - max_active_clusters, current_stream, permuted_idx_to_expanded_idx, token_final_scales, - options="--opt-level 2", ) - # Compute reference result - if not skip_ref_check: - print("Verifying results...") - # Execution - if multi_b_mode: - compiled_gemm( - a_tensor, - tuple(b_tensor), - out_tensor, - sfa_tensor, - tuple(sfb_tensor), - tile_idx_to_expert_idx, - num_non_exiting_tiles, - tile_idx_to_mn_limit, - tuple(alpha), - current_stream, - permuted_idx_to_expanded_idx, - token_final_scales, - ) - else: - compiled_gemm( - a_tensor, - b_tensor, - out_tensor, - sfa_tensor, - sfb_tensor, - tile_idx_to_expert_idx, - num_non_exiting_tiles, - tile_idx_to_mn_limit, - alpha, - current_stream, - permuted_idx_to_expanded_idx, - token_final_scales, - ) - torch.cuda.synchronize() ref_result = verify_reference_result( a_torch_cpu, @@ -892,7 +800,6 @@ def generate_tensors(): mma_tiler_mn, # cta_tile_m permuted_m, seq_len, # Pass seq_len as num_tokens for C tensor shape - b_tensor_l_sizes=b_tensor_l_sizes, ) ( @@ -909,21 +816,6 @@ def generate_tensors(): final_scale_dtype, ) - if multi_b_mode: - return cute.testing.JitArguments( - a_tensor, - tuple(b_tensor), - out_tensor, - sfa_tensor, - tuple(sfb_tensor), - tile_idx_to_expert_idx, - num_non_exiting_tiles, - tile_idx_to_mn_limit, - tuple(alpha), - current_stream, - permuted_idx_to_expanded_idx, - token_final_scales, - ) return cute.testing.JitArguments( a_tensor, b_tensor, @@ -983,14 +875,6 @@ def parse_comma_separated_ints(s: str) -> Tuple[int, ...]: except ValueError: raise argparse.ArgumentTypeError("Invalid format. Expected comma-separated integers.") - def split_groups_to_b_tensors(num_groups: int, num_b_tensors: int) -> Tuple[int, ...]: - if num_b_tensors <= 0: - raise argparse.ArgumentTypeError("num_b_tensors must be positive.") - base = num_groups // num_b_tensors - remainder = num_groups % num_b_tensors - sizes = [base + (1 if i < remainder else 0) for i in range(num_b_tensors)] - return tuple(sizes) - def read_benchmark_file( filepath: str, ) -> Tuple[Tuple[int, int, int], Tuple[int, ...]]: @@ -1184,19 +1068,6 @@ def parse_benchmark_arg( help="Use CUPTI to measure execution time", ) - parser.add_argument( - "--num_b_tensors", - type=int, - default=1, - help="Number of B tensors for multi-B mode (default: 1).", - ) - parser.add_argument( - "--b_tensor_l_sizes", - type=parse_comma_separated_ints, - default=None, - help="Comma-separated L sizes for each B tensor (e.g., 8,8,16). Overrides --num_b_tensors.", - ) - args = parser.parse_args() # Process arguments to generate nkl and group_m_list @@ -1214,17 +1085,6 @@ def parse_benchmark_arg( if len(args.cluster_shape_mn) != 2: parser.error("--cluster_shape_mn must contain exactly 2 values") - _, _, l = nkl # noqa: E741 - b_tensor_l_sizes = None - if args.b_tensor_l_sizes is not None: - b_tensor_l_sizes = args.b_tensor_l_sizes - if args.num_b_tensors != 1 and args.num_b_tensors != len(b_tensor_l_sizes): - parser.error("--num_b_tensors must match length of --b_tensor_l_sizes") - if sum(b_tensor_l_sizes) != l: - parser.error("--b_tensor_l_sizes must sum to L") - elif args.num_b_tensors > 1: - b_tensor_l_sizes = split_groups_to_b_tensors(l, args.num_b_tensors) - exec_time = run( nkl, group_m_list, @@ -1249,7 +1109,6 @@ def parse_benchmark_arg( args.raster_along_m, args.use_blkred, args.use_cupti, - b_tensor_l_sizes=b_tensor_l_sizes, ) print("exec_time: ", exec_time) print("PASS") diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py new file mode 100644 index 000000000000..d5392e6890c8 --- /dev/null +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MPI-allgather refactor of DWDP's ``fixup_moe_backends``. + +These tests verify that the MPI-based allgather (replacing tekit's TCPDWDPStore) +produces the same semantics as the tekit original: + +- Each rank's local shard is contributed via comm.allgather. +- Ranks receive a list of shards (one per peer, indexed by rank). +- The function concatenates along dim 0 and updates the parameter in place. + +We mock the MPI communicator since the tests run single-process on one GPU. +The mock's ``allgather`` returns a pre-built shard list that the test provides +(simulating "what this rank would receive if 4 ranks each contributed their +shard"). This lets us cover the multi-rank semantics end-to-end without a +real MPI environment. +""" + +import unittest +from unittest.mock import MagicMock + +import torch +import torch.nn as nn + +from tensorrt_llm._torch.modules.dwdp.setup import ( + _allgather_e_score_correction_bias, + _allgather_expert_scales, + _scatter_shards_to_full, +) + + +def _make_mock_comm(all_shards): + """Create a mock MPI comm whose allgather() returns ``all_shards``. + + ``all_shards`` is a list of torch.Tensor objects, one per simulated rank. + The mock ignores the input argument and returns the list, which is the + semantic shape of a real ``Intracomm.allgather(local_shard)`` call. + """ + comm = MagicMock() + comm.allgather = MagicMock(side_effect=lambda _local: all_shards) + comm.Barrier = MagicMock() + comm.Get_rank = MagicMock(return_value=0) + comm.Get_size = MagicMock(return_value=len(all_shards)) + return comm + + +class TestAllgatherEScoreCorrectionBias(unittest.TestCase): + def _make_gate(self, bias_tensor, as_parameter=True): + """Minimal nn.Module with an e_score_correction_bias attribute.""" + gate = nn.Module() + if as_parameter: + gate.e_score_correction_bias = nn.Parameter(bias_tensor, requires_grad=False) + else: + gate.e_score_correction_bias = bias_tensor + return gate + + def test_missing_bias_is_noop(self): + gate = nn.Module() + comm = _make_mock_comm([torch.zeros(2)]) + _allgather_e_score_correction_bias( + gate, layer_idx=3, dwdp_rank=0, dwdp_size=4, num_experts_total=16, comm=comm + ) + comm.allgather.assert_not_called() + self.assertFalse( + hasattr(gate, "e_score_correction_bias") + and getattr(gate, "e_score_correction_bias") is not None + and isinstance(getattr(gate, "e_score_correction_bias"), (torch.Tensor, nn.Parameter)) + ) + + def test_allgather_sharded_parameter(self): + """Sharded bias: allgather 4 shards, concat, replace nn.Parameter.""" + dwdp_size = 4 + shard_size = 2 + num_experts_total = shard_size * dwdp_size + # Each rank's shard has unique values + shards = [ + torch.arange(r * shard_size, (r + 1) * shard_size, dtype=torch.float32) + for r in range(dwdp_size) + ] + expected = torch.cat(shards, dim=0) + + for dwdp_rank in range(dwdp_size): + with self.subTest(dwdp_rank=dwdp_rank): + gate = self._make_gate(shards[dwdp_rank].clone()) + comm = _make_mock_comm(shards) + + _allgather_e_score_correction_bias( + gate, + layer_idx=3, + dwdp_rank=dwdp_rank, + dwdp_size=dwdp_size, + num_experts_total=num_experts_total, + comm=comm, + ) + + comm.allgather.assert_called_once() + self.assertIsInstance(gate.e_score_correction_bias, nn.Parameter) + self.assertEqual(gate.e_score_correction_bias.shape, (num_experts_total,)) + torch.testing.assert_close(gate.e_score_correction_bias.data, expected) + + def test_allgather_sharded_plain_tensor(self): + """Sharded bias stored as plain Tensor (not Parameter) — update in-place.""" + dwdp_size = 2 + shard_size = 3 + num_experts_total = shard_size * dwdp_size + shards = [ + torch.arange(r * shard_size, (r + 1) * shard_size, dtype=torch.float32) + for r in range(dwdp_size) + ] + gate = self._make_gate(shards[0].clone(), as_parameter=False) + comm = _make_mock_comm(shards) + + _allgather_e_score_correction_bias( + gate, + layer_idx=3, + dwdp_rank=0, + dwdp_size=dwdp_size, + num_experts_total=num_experts_total, + comm=comm, + ) + + comm.allgather.assert_called_once() + self.assertEqual(gate.e_score_correction_bias.shape, (num_experts_total,)) + torch.testing.assert_close(gate.e_score_correction_bias, torch.cat(shards, dim=0)) + + +class TestAllgatherExpertScales(unittest.TestCase): + def _make_experts_module(self, params): + """Return an nn.Module with the given {name: tensor} parameters.""" + module = nn.Module() + for name, tensor in params.items(): + setattr(module, name, nn.Parameter(tensor, requires_grad=False)) + return module + + def test_skip_non_scale_params(self): + """Parameters without scale keywords are not touched.""" + experts_per_rank = 2 + module = self._make_experts_module( + { + "random_weight": torch.zeros(experts_per_rank, 4), + "fc31_alpha": torch.tensor([1.0, 2.0]), + } + ) + all_alpha_shards = [torch.tensor([1.0, 2.0]), torch.tensor([3.0, 4.0])] + comm = _make_mock_comm(all_alpha_shards) + + _allgather_expert_scales( + module, + layer_idx=3, + dwdp_rank=0, + dwdp_size=2, + comm=comm, + experts_per_rank=experts_per_rank, + ) + + # Called exactly once — only fc31_alpha matched. + self.assertEqual(comm.allgather.call_count, 1) + # random_weight unchanged (still size (2,4)) + self.assertEqual(module.random_weight.shape, (experts_per_rank, 4)) + # fc31_alpha concatenated to full size (4,) + self.assertEqual(module.fc31_alpha.shape, (4,)) + torch.testing.assert_close(module.fc31_alpha.data, torch.cat(all_alpha_shards, dim=0)) + + def test_allgather_multiple_scales(self): + """Two matching scale params: each gets its own allgather call.""" + experts_per_rank = 2 + dwdp_size = 2 + alpha_shards = [torch.tensor([1.0, 2.0]), torch.tensor([3.0, 4.0])] + beta_shards = [torch.tensor([[5.0]]), torch.tensor([[6.0]])] + # fc31_alpha has shape (2,); fc2_alpha has shape (2,1) — both match + # experts_per_rank=2 on dim 0. + module = self._make_experts_module( + { + "fc31_alpha": alpha_shards[0].clone(), + "fc2_alpha": torch.tensor([[5.0]]), # shape (1,1) — shape[0]=1 != 2 + } + ) + # Rewrite fc2_alpha with shape[0]=2 so it matches + module.fc2_alpha = nn.Parameter(torch.tensor([[1.0], [2.0]]), requires_grad=False) + # Two shards, each shape (2,1) + beta_shards = [torch.tensor([[1.0], [2.0]]), torch.tensor([[3.0], [4.0]])] + + # The mock returns the pre-set shard list for whichever param is being + # allgathered. named_parameters() order is insertion order — the + # ordering here is fc31_alpha first, fc2_alpha second. + call_idx = {"n": 0} + + def _side_effect(_local): + idx = call_idx["n"] + call_idx["n"] += 1 + return alpha_shards if idx == 0 else beta_shards + + comm = MagicMock() + comm.allgather = MagicMock(side_effect=_side_effect) + + _allgather_expert_scales( + module, + layer_idx=3, + dwdp_rank=0, + dwdp_size=dwdp_size, + comm=comm, + experts_per_rank=experts_per_rank, + ) + + self.assertEqual(comm.allgather.call_count, 2) + torch.testing.assert_close(module.fc31_alpha.data, torch.cat(alpha_shards, dim=0)) + torch.testing.assert_close(module.fc2_alpha.data, torch.cat(beta_shards, dim=0)) + + +class TestScatterShardsToFull(unittest.TestCase): + """Unit tests for ``_scatter_shards_to_full`` — the peer_ranges-aware + reconstruction of a full-size tensor from per-peer EP shards. Covers + the three partition modes that motivate Phase 2: + + * Uniform (size == stride == num_experts // dwdp_size): no overlap, + no padding; equivalent to ``torch.cat``. + * Non-uniform tail-padding (size * dwdp_size > num_experts): tail + shard's trailing entries are skipped via ``[:valid]``. + * Redundancy (size > stride): overlapping ranges; later peers + overwrite earlier ones at overlap indices but values agree + (per Phase 1 invariant: every rank that owns expert ``e`` loaded + ``e`` from the same checkpoint). + """ + + def test_uniform_partition(self): + # dwdp=4, 8 experts, size=stride=2. Each rank's shard covers + # exactly 2 experts; reconstruction = concat. + peer_ranges = [(0, 2), (2, 4), (4, 6), (6, 8)] + shards = [ + torch.tensor([0.0, 1.0]), + torch.tensor([2.0, 3.0]), + torch.tensor([4.0, 5.0]), + torch.tensor([6.0, 7.0]), + ] + full = _scatter_shards_to_full( + shards=shards, + peer_ranges=peer_ranges, + num_experts_total=8, + ref=shards[0], + ) + torch.testing.assert_close(full, torch.arange(8, dtype=torch.float32)) + + def test_mode_b_overlap_dwdp3(self): + # dwdp=3, 8 experts, size=4, stride=2: 2*2+4=8 (Mode B equality). + # Adjacent ranks overlap by 2 experts; shared values agree + # because every rank that owns expert ``e`` loaded ``e`` from + # the same checkpoint. + peer_ranges = [(0, 4), (2, 6), (4, 8)] + shards = [ + torch.tensor([0.0, 1.0, 2.0, 3.0]), # rank 0: experts 0,1,2,3 + torch.tensor([2.0, 3.0, 4.0, 5.0]), # rank 1: experts 2,3,4,5 (overlap rank 0 at 2,3) + torch.tensor([4.0, 5.0, 6.0, 7.0]), # rank 2: experts 4,5,6,7 (overlap rank 1 at 4,5) + ] + full = _scatter_shards_to_full( + shards=shards, + peer_ranges=peer_ranges, + num_experts_total=8, + ref=shards[0], + ) + torch.testing.assert_close(full, torch.arange(8, dtype=torch.float32)) + + def test_higher_dim_trailing_shape(self): + # Shards may have trailing dims (e.g., scale param shape (size, K)). + peer_ranges = [(0, 2), (2, 4)] + shards = [ + torch.tensor([[0.0, 0.5], [1.0, 1.5]]), + torch.tensor([[2.0, 2.5], [3.0, 3.5]]), + ] + full = _scatter_shards_to_full( + shards=shards, + peer_ranges=peer_ranges, + num_experts_total=4, + ref=shards[0], + ) + expected = torch.tensor([[0.0, 0.5], [1.0, 1.5], [2.0, 2.5], [3.0, 3.5]]) + torch.testing.assert_close(full, expected) + + +class TestAllgatherExpertScalesNonUniform(unittest.TestCase): + """Validate ``_allgather_expert_scales`` end-to-end with the new + Phase 2 ``peer_ranges`` argument: shards loaded under Mode B + overlap (size > stride) must reconstruct the full bias correctly, + with overlapping experts getting the same value from any peer.""" + + def _make_experts_module(self, params): + module = torch.nn.Module() + for name, tensor in params.items(): + setattr(module, name, torch.nn.Parameter(tensor, requires_grad=False)) + return module + + def test_mode_b_overlap_dwdp3_via_peer_ranges(self): + # dwdp=3, 256 experts, Mode B with size=86, stride=85 + # (2*85 + 86 = 256 — exact equality, 1-expert overlap between + # adjacent ranks). Shards loaded by each rank cover overlapping + # ranges; ``_scatter_shards_to_full`` writes the shared experts + # multiple times but the values agree (same checkpoint). + peer_ranges = [(0, 86), (85, 171), (170, 256)] + all_shards = [ + torch.arange(0, 86, dtype=torch.float32), # rank 0: [0..85] + torch.arange(85, 171, dtype=torch.float32), # rank 1: [85..170] + torch.arange(170, 256, dtype=torch.float32), # rank 2: [170..255] + ] + module = self._make_experts_module( + { + "fc31_alpha": torch.arange(0, 86, dtype=torch.float32), + } + ) + comm = _make_mock_comm(all_shards) + + _allgather_expert_scales( + module, + layer_idx=3, + dwdp_rank=0, + dwdp_size=3, + comm=comm, + experts_per_rank=86, + num_experts_total=256, + peer_ranges=peer_ranges, + ) + + comm.allgather.assert_called_once() + # Reconstructed alpha covers ALL 256 experts with their canonical values. + torch.testing.assert_close( + module.fc31_alpha.data, + torch.arange(0, 256, dtype=torch.float32), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_manager.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_manager.py new file mode 100644 index 000000000000..5bbab2dd59d3 --- /dev/null +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_manager.py @@ -0,0 +1,372 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the DwdpManager facade. + +DwdpManager is a thin lifecycle layer that: +- creates a DWDP MPI sub-communicator +- owns the Single Source of Truth for registered MoE layer indices +- forwards runtime calls to the DWDPWeightManager produced by setup_dwdp + +These tests mock out COMM_WORLD / global_mpi_rank / setup_dwdp so the full +lifecycle can be exercised on a single CPU without MPI / CUDA / MNNVL. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from tensorrt_llm._torch.distributed import MPIDist +from tensorrt_llm._torch.pyexecutor.dwdp import ( + DwdpManager, + get_global_dwdp_manager, + set_global_dwdp_manager, +) +from tensorrt_llm.llmapi.llm_args import DwdpConfig +from tensorrt_llm.mapping import Mapping + + +def _make_config(dwdp_size: int = 2) -> DwdpConfig: + return DwdpConfig( + dwdp_size=dwdp_size, + num_groups=1, + num_experts_per_worker=4, + num_prefetch_experts=4, + ) + + +def _make_mapping(dwdp_size: int = 2, dwdp_rank: int = 0) -> Mapping: + return Mapping( + world_size=dwdp_size, + rank=dwdp_rank, + tp_size=dwdp_size, + dwdp_size=dwdp_size, + dwdp_rank=dwdp_rank, + ) + + +def _configure_mock_comm_world(mock_comm_world, world_size: int = 1024): + """Configure a class-decorator-patched COMM_WORLD mock with sensible defaults. + + Tests that construct a real DwdpManager need both ``Create_group`` (returns + the dwdp sub-comm) and ``Get_size`` (used by num_groups validation) to be + set; raw MagicMock returns from ``Get_size`` would fail integer comparison. + """ + sub_comm = MagicMock() + mock_comm_world.Create_group.return_value = sub_comm + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = world_size + return sub_comm + + +@patch("tensorrt_llm._torch.pyexecutor.dwdp.setup_dwdp") +@patch("tensorrt_llm._torch.pyexecutor.dwdp.global_mpi_rank", return_value=0) +@patch("tensorrt_llm._torch.pyexecutor.dwdp.COMM_WORLD") +class TestDwdpManagerLifecycle(unittest.TestCase): + def setUp(self): + # Ensure no leftover global singleton from a prior test. + set_global_dwdp_manager(None) + + def tearDown(self): + set_global_dwdp_manager(None) + + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def test_init_stores_fields(self, mock_comm_world, _rank, _setup): + sub_comm = MagicMock() + mock_comm_world.Create_group.return_value = sub_comm + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + + mgr = DwdpManager( + config=_make_config(dwdp_size=2), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(dwdp_size=2, dwdp_rank=0), + ) + self.assertEqual(mgr.dwdp_size, 2) + self.assertEqual(mgr.dwdp_rank, 0) + self.assertEqual(mgr._registered_layers, []) + self.assertIs(mgr.dwdp_group, sub_comm) + self.assertIsNone(mgr._weight_manager) + # start_expert_id / end_expert_id derive from + # num_prefetch_experts (stride) and num_experts_per_worker (size). + # rank=0: start=0, end=4. + self.assertEqual(mgr.start_expert_id, 0) + self.assertEqual(mgr.end_expert_id, 4) + + def test_init_expert_range_uniform(self, mock_comm_world, _rank, _setup): + """Uniform partition (size == stride): each rank's range is a clean slice.""" + _configure_mock_comm_world(mock_comm_world) + # dwdp_rank=1 with stride=4 -> start=4, end=4+4=8. + mgr = DwdpManager( + config=_make_config(dwdp_size=2), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(dwdp_size=2, dwdp_rank=1), + ) + self.assertEqual(mgr.start_expert_id, 4) + self.assertEqual(mgr.end_expert_id, 8) + + def test_init_expert_range_redundancy(self, mock_comm_world, _rank, _setup): + """Redundancy partition (size > stride): adjacent ranges overlap.""" + _configure_mock_comm_world(mock_comm_world) + redundant_config = DwdpConfig( + dwdp_size=2, + num_groups=1, + num_experts_per_worker=6, # storage size + num_prefetch_experts=4, # stride < size -> 2-expert overlap + ) + # rank=0: [0, 6). rank=1 would be [4, 10) -> 2-expert overlap [4, 6). + mgr = DwdpManager( + config=redundant_config, + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(dwdp_size=2, dwdp_rank=0), + ) + self.assertEqual(mgr.start_expert_id, 0) + self.assertEqual(mgr.end_expert_id, 6) + # Stored config knobs match what's plumbed into setup_dwdp. + self.assertEqual(mgr.num_experts_per_worker, 6) + self.assertEqual(mgr.num_prefetch_experts, 4) + + # ------------------------------------------------------------------ + # num_groups topology validation (was a dead schema field through 1fbc0d49) + # ------------------------------------------------------------------ + + def test_init_accepts_multi_group_topology(self, mock_comm_world, _rank, _setup): + # num_groups=2, dwdp_size=2 -> 4 CTX ranks total. World >= 4 is fine. + _configure_mock_comm_world(mock_comm_world, world_size=8) + mgr = DwdpManager( + config=DwdpConfig( + dwdp_size=2, + num_groups=2, # valid: this rank (0) is in group 0 + num_experts_per_worker=4, + num_prefetch_experts=4, + ), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + self.assertEqual(mgr.num_groups, 2) + + # ------------------------------------------------------------------ + # Global singleton + enter/exit + # ------------------------------------------------------------------ + + def test_enter_registers_global(self, mock_comm_world, _rank, _setup): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + self.assertIsNone(get_global_dwdp_manager()) + mgr.__enter__() + self.assertIs(get_global_dwdp_manager(), mgr) + mgr.__exit__(None, None, None) + self.assertIsNone(get_global_dwdp_manager()) + + def test_cleanup_frees_comm_and_idempotent(self, mock_comm_world, _rank, _setup): + sub_comm = MagicMock() + mock_comm_world.Create_group.return_value = sub_comm + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + mgr.cleanup() + sub_comm.Free.assert_called_once() + self.assertIsNone(mgr.dwdp_group) + # Idempotent — second call must not raise + mgr.cleanup() + sub_comm.Free.assert_called_once() # still only one Free + + # ------------------------------------------------------------------ + # add_layer (SSOT) + # ------------------------------------------------------------------ + + def test_add_layer_appends_to_registered(self, mock_comm_world, _rank, _setup): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + mgr.add_layer(3) + mgr.add_layer(5) + mgr.add_layer(7) + self.assertEqual(mgr._registered_layers, [3, 5, 7]) + + def test_add_layer_duplicate_is_idempotent(self, mock_comm_world, _rank, _setup): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + mgr.add_layer(3) + mgr.add_layer(3) + self.assertEqual(mgr._registered_layers, [3]) + + # ------------------------------------------------------------------ + # setup(model) forwards to setup_dwdp with layer_indices SSOT + # ------------------------------------------------------------------ + + def test_setup_forwards_layer_indices(self, mock_comm_world, _rank, mock_setup): + sub_comm = MagicMock() + mock_comm_world.Create_group.return_value = sub_comm + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + fake_weight_manager = MagicMock() + mock_setup.return_value = fake_weight_manager + + mapping = _make_mapping() + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=mapping, + ) + # Registration order should be sorted before passing to setup_dwdp + mgr.add_layer(7) + mgr.add_layer(3) + mgr.add_layer(5) + + fake_model = MagicMock() + with patch( + "tensorrt_llm._torch.pyexecutor.dwdp.torch.cuda.current_device", + return_value=0, + ): + result = mgr.setup(fake_model) + + mock_setup.assert_called_once() + _, kwargs = mock_setup.call_args + self.assertIs(kwargs["model"], fake_model) + self.assertIs(kwargs["mapping"], mapping) + self.assertEqual(kwargs["device_id"], 0) + self.assertIs(kwargs["comm"], sub_comm) + self.assertEqual(kwargs["layer_indices"], [3, 5, 7]) # sorted SSOT + # Config-driven expert range fields are forwarded from DwdpConfig. + self.assertEqual(kwargs["num_experts_per_worker"], 4) + self.assertEqual(kwargs["num_prefetch_experts"], 4) + self.assertIs(result, fake_weight_manager) + self.assertIs(mgr._weight_manager, fake_weight_manager) + + # ------------------------------------------------------------------ + # Runtime forwards require setup() + # ------------------------------------------------------------------ + + def test_prefetch_first_layers_forwards(self, mock_comm_world, _rank, mock_setup): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + fake_wm = MagicMock() + fake_wm.first_moe_layer.return_value = 3 + fake_wm.next_moe_layer.return_value = 5 + mock_setup.return_value = fake_wm + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + with patch( + "tensorrt_llm._torch.pyexecutor.dwdp.torch.cuda.current_device", + return_value=0, + ): + mgr.setup(MagicMock()) + mgr.prefetch_first_layers() + # First MoE layer 3 + ping-pong depth -> also prefetch next (5) + self.assertEqual( + fake_wm.prefetch_layer.call_args_list, + [ + unittest.mock.call(3), + unittest.mock.call(5), + ], + ) + + def test_record_compute_and_prefetch_next_schedules_next( + self, mock_comm_world, _rank, mock_setup + ): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + fake_wm = MagicMock() + fake_wm.next_moe_layer.return_value = 9 + mock_setup.return_value = fake_wm + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + with patch( + "tensorrt_llm._torch.pyexecutor.dwdp.torch.cuda.current_device", + return_value=0, + ): + mgr.setup(MagicMock()) + + mgr.record_compute_and_prefetch_next(7) + fake_wm.next_moe_layer.assert_called_once_with(7) + fake_wm.prefetch_layer.assert_called_once_with(9) + + def test_record_compute_and_prefetch_next_last_layer_is_noop( + self, mock_comm_world, _rank, mock_setup + ): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + fake_wm = MagicMock() + fake_wm.next_moe_layer.return_value = None + mock_setup.return_value = fake_wm + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + with patch( + "tensorrt_llm._torch.pyexecutor.dwdp.torch.cuda.current_device", + return_value=0, + ): + mgr.setup(MagicMock()) + + mgr.record_compute_and_prefetch_next(60) + fake_wm.prefetch_layer.assert_not_called() + + def test_wait_and_bind_forwards(self, mock_comm_world, _rank, mock_setup): + mock_comm_world.Create_group.return_value = MagicMock() + mock_comm_world.group.Incl.return_value = MagicMock() + mock_comm_world.Get_size.return_value = 1024 + fake_wm = MagicMock() + mock_setup.return_value = fake_wm + mgr = DwdpManager( + config=_make_config(), + dist=MagicMock(spec=MPIDist), + mapping=_make_mapping(), + ) + with patch( + "tensorrt_llm._torch.pyexecutor.dwdp.torch.cuda.current_device", + return_value=0, + ): + mgr.setup(MagicMock()) + backend = MagicMock() + mgr.wait_and_bind(backend, 5) + fake_wm.wait_and_bind.assert_called_once_with(backend, 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_mapping.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_mapping.py new file mode 100644 index 000000000000..ee1ec2094133 --- /dev/null +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_mapping.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for DWDP (Distributed Weight Data Parallelism) extensions to Mapping. + +Covers: +- Constructor validation (dwdp_size must be 0 or >=2; dwdp_rank range) +- dwdp_enabled / dwdp_size / dwdp_rank accessors +- DWDP override of moe_tp_size / moe_ep_size / moe_cluster_size to 1 + (DWDP takes ownership of the expert layout via + ``_init_dwdp_expert_layout``; the fused MoE backend sees a single + full-table partition) +- moe_ep_rank is always 0 when DWDP is enabled +- to_dict/from_dict round-trip preserves DWDP fields +- __eq__ and __hash__ include DWDP fields +""" + +import unittest + +from tensorrt_llm.mapping import Mapping + + +class TestMappingDwdp(unittest.TestCase): + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def test_enabled_basic(self): + """dwdp_size=4 gives an enabled mapping with DWDP-managed MoE parallelism.""" + m = Mapping(world_size=4, rank=2, tp_size=4, dwdp_size=4, dwdp_rank=2) + self.assertTrue(m.dwdp_enabled) + self.assertEqual(m.dwdp_size, 4) + self.assertEqual(m.dwdp_rank, 2) + + # ------------------------------------------------------------------ + # MoE parallelism override + # ------------------------------------------------------------------ + + def test_override_moe_parallelism(self): + """DWDP enabled: moe_tp_size=1, moe_ep_size=1, cluster=1. + + ``moe_ep_size`` is forced to 1 so the fused-MoE backend bypasses + ``num_experts % ep_size == 0`` and sees a single (full) expert + table; the rank-specific layout is installed by + ``_init_dwdp_expert_layout`` from ``DwdpManager``. + """ + m = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=0) + self.assertEqual(m.moe_tp_size, 1) + self.assertEqual(m.moe_ep_size, 1) + self.assertEqual(m.moe_cluster_size, 1) + + def test_override_beats_explicit_moe_values(self): + """Even if user explicitly passes moe_tp/ep sizes, DWDP override wins.""" + m = Mapping( + world_size=4, + rank=0, + tp_size=4, + moe_tp_size=2, + moe_ep_size=2, + moe_cluster_size=1, + dwdp_size=4, + dwdp_rank=1, + ) + self.assertEqual(m.moe_tp_size, 1) + self.assertEqual(m.moe_ep_size, 1) + self.assertEqual(m.moe_cluster_size, 1) + + def test_non_dwdp_leaves_moe_alone(self): + """When DWDP disabled, existing MoE defaulting logic is preserved.""" + m = Mapping( + world_size=4, rank=0, tp_size=4, moe_tp_size=2, moe_ep_size=2, moe_cluster_size=1 + ) + self.assertEqual(m.moe_tp_size, 2) + self.assertEqual(m.moe_ep_size, 2) + + def test_moe_tp_cluster_ep_size_dwdp(self): + """moe_tp_cluster_ep_size collapses to moe_cluster_size with DWDP. + + With ``moe_ep_size = moe_tp_size = 1`` under DWDP, the legacy + composite product reduces to ``moe_cluster_size`` (the DWDP + group is orthogonal to the MoE TP/EP world). + """ + m = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=0) + self.assertEqual(m.moe_tp_cluster_ep_size, 1) + + def test_dwdp_supports_non_divisible_size(self): + """DWDP must accept dwdp_size values that don't divide num_experts. + + Pre-Phase-1 the override forced ``moe_ep_size = dwdp_size``, + which then triggered ``num_experts % ep_size == 0`` deep in the + fused MoE backend. The Phase 1 contract is that ``Mapping`` + construction with dwdp_size=3 / 5 is unconditionally valid; the + partition is later validated against the concrete + ``num_experts`` by ``DwdpManager``/``setup_dwdp``. + """ + for dwdp_size in (3, 5): + for dwdp_rank in range(dwdp_size): + m = Mapping( + world_size=1, rank=0, tp_size=1, dwdp_size=dwdp_size, dwdp_rank=dwdp_rank + ) + self.assertEqual(m.moe_ep_size, 1) + self.assertEqual(m.dwdp_size, dwdp_size) + self.assertEqual(m.dwdp_rank, dwdp_rank) + + # ------------------------------------------------------------------ + # moe_ep_rank conditional branch + # ------------------------------------------------------------------ + + def test_moe_ep_rank_zero_when_dwdp_enabled(self): + """DWDP on: moe_ep_rank is always 0 (matches moe_ep_size = 1). + + The actual rank-specific expert range comes from + ``DwdpManager.start_expert_id`` via ``_init_dwdp_expert_layout``, + not from ``moe_ep_rank``. + """ + for dwdp_rank in range(4): + m = Mapping(world_size=4, rank=dwdp_rank, tp_size=4, dwdp_size=4, dwdp_rank=dwdp_rank) + self.assertEqual(m.moe_ep_rank, 0) + + def test_moe_ep_rank_uses_tp_rank_when_disabled(self): + """DWDP off: moe_ep_rank falls back to tp_rank % moe_ep_size.""" + m = Mapping( + world_size=4, rank=0, tp_size=4, moe_tp_size=1, moe_ep_size=4, moe_cluster_size=1 + ) + self.assertEqual(m.moe_ep_rank, m.tp_rank % m.moe_ep_size) + + # ------------------------------------------------------------------ + # to_dict / from_dict round-trip + # ------------------------------------------------------------------ + + def test_to_dict_includes_dwdp(self): + m = Mapping(world_size=4, rank=1, tp_size=4, dwdp_size=4, dwdp_rank=1) + d = m.to_dict() + self.assertIn("dwdp_size", d) + self.assertIn("dwdp_rank", d) + self.assertEqual(d["dwdp_size"], 4) + self.assertEqual(d["dwdp_rank"], 1) + + def test_roundtrip_preserves_dwdp(self): + m1 = Mapping(world_size=4, rank=1, tp_size=4, dwdp_size=4, dwdp_rank=1) + m2 = Mapping(**m1.to_dict()) + self.assertEqual(m1, m2) + self.assertEqual(m2.dwdp_size, 4) + self.assertEqual(m2.dwdp_rank, 1) + self.assertTrue(m2.dwdp_enabled) + + def test_roundtrip_disabled(self): + m1 = Mapping(world_size=4, rank=0, tp_size=4) + m2 = Mapping(**m1.to_dict()) + self.assertEqual(m1, m2) + self.assertFalse(m2.dwdp_enabled) + + # ------------------------------------------------------------------ + # __eq__ / __hash__ include DWDP fields + # ------------------------------------------------------------------ + + def test_eq_distinguishes_dwdp_rank(self): + """Two otherwise-identical mappings with different dwdp_rank are not equal.""" + m1 = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=0) + m2 = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=1) + self.assertNotEqual(m1, m2) + + def test_eq_distinguishes_dwdp_enabled(self): + m_on = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=0) + m_off = Mapping(world_size=4, rank=0, tp_size=4) + self.assertNotEqual(m_on, m_off) + + def test_hash_distinguishes_dwdp(self): + """Hash must differ for different dwdp configurations. + + Probabilistic but deterministic here since we only tweak dwdp_rank. + """ + m1 = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=0) + m2 = Mapping(world_size=4, rank=0, tp_size=4, dwdp_size=4, dwdp_rank=1) + self.assertNotEqual(hash(m1), hash(m2)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py new file mode 100644 index 000000000000..4bd77cc6b678 --- /dev/null +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ``compute_peer_ranges`` and ``lookup_owner``. + +These two helpers are the core of Phase 2 (non-uniform / redundant +partition support). They replace the legacy +``cursor // experts_per_rank`` formula used everywhere a remote expert +needed to be mapped to its owning peer rank. + +Cases covered: + + * Uniform partition (size == stride == num_experts // dwdp_size): + behavior must match the legacy ``//`` formula exactly. + * Non-uniform tail padding (size * dwdp_size > num_experts): the tail + rank's valid range is capped at ``num_experts``. + * Redundancy (size > stride): adjacent ranges overlap; + ``lookup_owner`` picks the lowest-rank owner (deterministic). + * Boundary expert ids (0, num_experts - 1). + * Out-of-range expert ids raise. +""" + +import unittest + +from tensorrt_llm._torch.modules.dwdp.specs import compute_peer_ranges, lookup_owner + + +class TestComputePeerRanges(unittest.TestCase): + def test_uniform_dwdp4_matches_floor_div(self): + # 256 experts / 4 ranks = 64 each. Every rank's range starts at + # rank * 64 and ends at the next rank's start. + peer_ranges = compute_peer_ranges( + dwdp_size=4, + num_experts_per_worker=64, + num_prefetch_experts=64, + num_experts_total=256, + ) + self.assertEqual(peer_ranges, [(0, 64), (64, 128), (128, 192), (192, 256)]) + + def test_non_uniform_tail_padding_capped(self): + # Defensive: the function caps the last rank's end at + # ``num_experts_total`` even when ``(dwdp-1)*stride + size > + # num_experts``. In production this configuration is rejected + # upstream by ``_validate_partition_config`` (tail padding is + # incompatible with GB200 fabric-handle partial mapping), but + # ``compute_peer_ranges`` itself stays robust. + peer_ranges = compute_peer_ranges( + dwdp_size=3, + num_experts_per_worker=86, + num_prefetch_experts=86, + num_experts_total=256, + ) + self.assertEqual(peer_ranges, [(0, 86), (86, 172), (172, 256)]) + + def test_mode_b_dwdp3_overlap(self): + # dwdp=3, 256 experts, Mode B: size=86, stride=85 — the + # production recipe for ``num_experts not divisible by dwdp_size``. + # ``(dwdp-1)*stride + size == num_experts`` exactly, with 1 + # expert overlapping between adjacent ranks. + peer_ranges = compute_peer_ranges( + dwdp_size=3, + num_experts_per_worker=86, + num_prefetch_experts=85, + num_experts_total=256, + ) + self.assertEqual(peer_ranges, [(0, 86), (85, 171), (170, 256)]) + + def test_redundancy_overlap(self): + # dwdp=4, size=70, stride=62: 8-expert overlap between adjacent + # ranks. Last rank's valid end is min(186 + 70, 256) = 256. + peer_ranges = compute_peer_ranges( + dwdp_size=4, + num_experts_per_worker=70, + num_prefetch_experts=62, + num_experts_total=256, + ) + self.assertEqual(peer_ranges, [(0, 70), (62, 132), (124, 194), (186, 256)]) + # Verify overlap exists between every adjacent pair. + for r in range(len(peer_ranges) - 1): + cur_end = peer_ranges[r][1] + next_start = peer_ranges[r + 1][0] + self.assertGreater( + cur_end, next_start, f"Expected overlap between rank {r} and {r + 1}" + ) + + +class TestLookupOwner(unittest.TestCase): + def test_uniform_matches_floor_div(self): + # The whole point: lookup_owner must agree with the + # legacy formula on every uniform expert id. + peer_ranges = compute_peer_ranges( + dwdp_size=4, + num_experts_per_worker=64, + num_prefetch_experts=64, + num_experts_total=256, + ) + for expert_id in range(256): + self.assertEqual( + lookup_owner(expert_id, peer_ranges), + expert_id // 64, + f"mismatch at expert_id={expert_id}", + ) + + def test_mode_b_dwdp3_overlap(self): + # dwdp=3, 256 experts, Mode B (size=86, stride=85): production + # recipe for non-divisible dwdp_size. Adjacent ranks overlap by + # 1 expert; lookup_owner picks the lowest-rank owner for shared + # experts (deterministic). + peer_ranges = compute_peer_ranges( + dwdp_size=3, + num_experts_per_worker=86, + num_prefetch_experts=85, + num_experts_total=256, + ) + # Rank 0's exclusive range + self.assertEqual(lookup_owner(0, peer_ranges), 0) + self.assertEqual(lookup_owner(84, peer_ranges), 0) + # Boundary expert 85: in both rank 0 (0..85] inclusive end-1) + # and rank 1 (start at 85). lookup_owner picks lowest = rank 0. + self.assertEqual(lookup_owner(85, peer_ranges), 0) + # Rank 1's exclusive range + self.assertEqual(lookup_owner(86, peer_ranges), 1) + self.assertEqual(lookup_owner(169, peer_ranges), 1) + # Boundary 170 in rank 1 and rank 2 → rank 1. + self.assertEqual(lookup_owner(170, peer_ranges), 1) + # Rank 2's exclusive range + self.assertEqual(lookup_owner(171, peer_ranges), 2) + self.assertEqual(lookup_owner(255, peer_ranges), 2) + + def test_redundancy_picks_lowest_rank(self): + # dwdp=4, size=70, stride=62: experts in the overlap zone + # [62, 70) are owned by both rank 0 and rank 1; lookup_owner + # must consistently return the lowest rank (0) for these. + peer_ranges = compute_peer_ranges( + dwdp_size=4, + num_experts_per_worker=70, + num_prefetch_experts=62, + num_experts_total=256, + ) + # Experts in rank 0's exclusive range + self.assertEqual(lookup_owner(0, peer_ranges), 0) + self.assertEqual(lookup_owner(61, peer_ranges), 0) + # Overlap zone [62, 70): both rank 0 and rank 1 have it. + # First-match → rank 0. + for expert_id in range(62, 70): + self.assertEqual( + lookup_owner(expert_id, peer_ranges), + 0, + f"overlap should pick lowest peer at {expert_id}", + ) + # Past rank 0's range → rank 1 (or its overlap with rank 2) + self.assertEqual(lookup_owner(70, peer_ranges), 1) + self.assertEqual(lookup_owner(123, peer_ranges), 1) # in rank 1's exclusive + # Overlap [124, 132): rank 1 vs rank 2 → rank 1 + for expert_id in range(124, 132): + self.assertEqual(lookup_owner(expert_id, peer_ranges), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittest/_torch/thop/parallel/test_cute_dsl_moe.py b/tests/unittest/_torch/thop/parallel/test_cute_dsl_moe.py index 2a6585b558b4..0f918946841b 100644 --- a/tests/unittest/_torch/thop/parallel/test_cute_dsl_moe.py +++ b/tests/unittest/_torch/thop/parallel/test_cute_dsl_moe.py @@ -562,10 +562,10 @@ def test_nvfp4_grouped_gemm_finalize_blackwell( c = torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_blackwell( a, - [b], + b, a_sf, - [b_sf], - [alpha], + b_sf, + alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, @@ -600,35 +600,6 @@ def test_nvfp4_grouped_gemm_finalize_blackwell( match_ratio = torch.isclose(c, c_ref, rtol=1.6e-2, atol=1e-5).sum().item() / c.numel() assert match_ratio > 0.99 - if num_local_experts > 1: - split_sizes = (num_local_experts // 2, num_local_experts - num_local_experts // 2) - b_list = list(torch.split(b, split_sizes, dim=0)) - b_sf_list = list(torch.split(b_sf, split_sizes, dim=0)) - alpha_list = list(torch.split(alpha, split_sizes, dim=0)) - c_multi = torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_blackwell( - a, - b_list, - a_sf, - b_sf_list, - alpha_list, - tile_idx_to_group_idx, - tile_idx_to_mn_limit, - permuted_idx_to_expanded_idx, - num_non_exiting_tiles, - token_final_scales, - num_experts=num_experts, - top_k=top_k, - num_local_experts=num_local_experts, - local_expert_offset=0, - tile_size=tile_size, - output_dtype=torch.bfloat16, - scaling_vector_size=sf_vec_size, - ) - multi_match_ratio = ( - torch.isclose(c_multi, c_ref, rtol=1.6e-2, atol=1e-5).sum().item() / c_ref.numel() - ) - assert multi_match_ratio > 0.99 - @pytest.mark.skipif( get_sm_version() not in (100, 103), @@ -903,13 +874,13 @@ def test_nvfp4_gather_grouped_gemm_act_fusion_blackwell( global_sf = c_ref[:num_valid_permuted_tokens].abs().max().float() / (448 * 6) c_ref, c_sf_ref = torch.ops.trtllm.fp4_quantize(c_ref, 1 / global_sf, sf_vec_size, False) - # Call gather kernel (single-B via multi_b op with single-element lists) - c, c_sf = torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( + # Call gather kernel (single-B) + c, c_sf = torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( a, - [b_kernel], + b_kernel, a_sf_unswizzled, - [b_sf_kernel], - [alpha], + b_sf_kernel, + alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, permuted_idx_to_expanded_idx, @@ -956,46 +927,3 @@ def test_nvfp4_gather_grouped_gemm_act_fusion_blackwell( c_sf_valid = torch.cat(c_sf_valid) c_sf_ref_valid = torch.cat(c_sf_ref_valid) check_accuracy(c_sf_valid, c_sf_ref_valid, atol=1e-4, rtol=1e-4, percent=0.95) - - if num_local_experts > 1: - split_sizes = ( - num_local_experts // 2, - num_local_experts - num_local_experts // 2, - ) - b_kernel_list = list(torch.split(b_kernel, split_sizes, dim=0)) - b_sf_kernel_list = list(torch.split(b_sf_kernel, split_sizes, dim=0)) - alpha_list = list(torch.split(alpha, split_sizes, dim=0)) - c_multi, c_sf_multi = ( - torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell_multi_b( - a, - b_kernel_list, - a_sf_unswizzled, - b_sf_kernel_list, - alpha_list, - tile_idx_to_group_idx, - tile_idx_to_mn_limit, - permuted_idx_to_expanded_idx, - num_non_exiting_tiles, - torch.tensor([1 / global_sf], dtype=torch.float32, device="cuda"), - num_experts=num_experts, - top_k=top_k, - num_local_experts=num_local_experts, - local_expert_offset=0, - tile_size=tile_size, - scaling_vector_size=sf_vec_size, - activation_type=activation_type, - ) - ) - c_multi_valid = c_multi[:num_valid_permuted_tokens].view(torch.uint8)[valid_token_mask] - check_accuracy(c_multi_valid, c_ref_valid, atol=1e-4, rtol=1e-4, percent=0.95) - - c_sf_multi_unswizzled = unswizzle_sf( - c_sf_multi, max_num_permuted_tokens, interm_size, sf_vec_size - ) - c_sf_multi_valid = [] - for i in range(num_valid_permuted_tokens): - if i >= tile_idx_to_mn_limit_list[i // tile_size]: - continue - c_sf_multi_valid.append(c_sf_multi_unswizzled[i]) - c_sf_multi_valid = torch.cat(c_sf_multi_valid) - check_accuracy(c_sf_multi_valid, c_sf_ref_valid, atol=1e-4, rtol=1e-4, percent=0.95) From 06456e1a3a95718995c1c81c10fc0e4a02c0ff2c Mon Sep 17 00:00:00 2001 From: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:00:53 -0400 Subject: [PATCH 166/308] [TRTLLM-10947][perf] eagle3: use cudaMemcpy2DAsync custom op for hidden-state capture (#14479) Signed-off-by: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> --- cpp/tensorrt_llm/thop/CMakeLists.txt | 3 +- cpp/tensorrt_llm/thop/inplaceSliceCopyOp.cpp | 83 +++++++++++ tensorrt_llm/_torch/compilation/utils.py | 3 + tensorrt_llm/_torch/custom_ops/__init__.py | 9 ++ .../_torch/custom_ops/cpp_custom_ops.py | 4 + tensorrt_llm/_torch/speculative/eagle3.py | 9 +- .../test_inplace_slice_copy.py | 136 ++++++++++++++++++ 7 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 cpp/tensorrt_llm/thop/inplaceSliceCopyOp.cpp create mode 100644 tests/unittest/_torch/thop/parallel_hw_agnostic/test_inplace_slice_copy.py diff --git a/cpp/tensorrt_llm/thop/CMakeLists.txt b/cpp/tensorrt_llm/thop/CMakeLists.txt index 6849415992b7..90239c56f526 100644 --- a/cpp/tensorrt_llm/thop/CMakeLists.txt +++ b/cpp/tensorrt_llm/thop/CMakeLists.txt @@ -119,7 +119,8 @@ add_library( dsv3RopeOp.cpp fusedGemmAllreduceOp.cpp convertReqIndexToGlobalOp.cpp - trtllmGenQKVProcessOp.cpp) + trtllmGenQKVProcessOp.cpp + inplaceSliceCopyOp.cpp) set_property(TARGET th_common PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries( th_common PRIVATE ${TORCH_LIBRARIES} th_utils ${Python3_LIBRARIES} diff --git a/cpp/tensorrt_llm/thop/inplaceSliceCopyOp.cpp b/cpp/tensorrt_llm/thop/inplaceSliceCopyOp.cpp new file mode 100644 index 000000000000..8439d9a891ca --- /dev/null +++ b/cpp/tensorrt_llm/thop/inplaceSliceCopyOp.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/thop/thUtils.h" + +#include +#include + +namespace tensorrt_llm::torch_ext +{ + +// Copy src[:, :] into dest[:numTokens, dim1Start:dim1End] using cudaMemcpy2D. +// dest : 2-D contiguous CUDA tensor, shape [destRows, destCols] +// src : 2-D contiguous CUDA tensor, shape [numTokens, sliceWidth] where sliceWidth == dim1End - dim1Start +// dim1Start : first column index in dest to write into +// dim1End : one-past-last column index in dest to write into +// numTokens is inferred from src.size(0) +void inplaceSliceCopy(at::Tensor& dest, at::Tensor const& src, int64_t dim1Start, int64_t dim1End) +{ + CHECK_TH_CUDA(dest); + CHECK_TH_CUDA(src); + TORCH_CHECK(dest.get_device() == src.get_device(), "dest and src must be on the same CUDA device"); + TORCH_CHECK(dest.is_contiguous(), "dest must be contiguous"); + TORCH_CHECK(src.is_contiguous(), "src must be contiguous"); + TORCH_CHECK(dest.dim() == 2, "dest must be 2-D"); + TORCH_CHECK(src.dim() == 2, "src must be 2-D"); + TORCH_CHECK(dest.scalar_type() == src.scalar_type(), "dest and src must have the same dtype"); + + int64_t const numTokens = src.size(0); + int64_t const sliceWidth = dim1End - dim1Start; + TORCH_CHECK(dim1Start >= 0, "dim1Start must be non-negative"); + TORCH_CHECK(sliceWidth > 0, "dim1End must be greater than dim1Start"); + TORCH_CHECK(numTokens <= dest.size(0), "numTokens exceeds dest row count"); + TORCH_CHECK(dim1End <= dest.size(1), "dim1End exceeds dest column count"); + TORCH_CHECK(src.size(1) == sliceWidth, "src column count must equal dim1End - dim1Start"); + + if (numTokens == 0 || sliceWidth == 0) + { + return; + } + + int64_t const elemSize = dest.element_size(); + int64_t const destPitch = dest.size(1) * elemSize; // bytes per dest row + int64_t const srcPitch = src.size(1) * elemSize; // bytes per src row + int64_t const width = sliceWidth * elemSize; // bytes to copy per row + + char* destPtr = static_cast(dest.data_ptr()) + dim1Start * elemSize; + char const* srcPtr = static_cast(src.data_ptr()); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dest.get_device()); + TLLM_CUDA_CHECK(cudaMemcpy2DAsync( + destPtr, destPitch, srcPtr, srcPitch, width, static_cast(numTokens), cudaMemcpyDeviceToDevice, stream)); +} + +} // namespace tensorrt_llm::torch_ext + +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + // dest: destination tensor (mutated in-place) + // src: source tensor (numTokens inferred from src.size(0)) + // dim1_start: first column index in dest + // dim1_end: one-past-last column index in dest + m.def("inplace_slice_copy(Tensor(a!) dest, Tensor src, int dim1_start, int dim1_end) -> ()"); +} + +TORCH_LIBRARY_IMPL(trtllm, CUDA, m) +{ + m.impl("inplace_slice_copy", TORCH_FN(tensorrt_llm::torch_ext::inplaceSliceCopy)); +} diff --git a/tensorrt_llm/_torch/compilation/utils.py b/tensorrt_llm/_torch/compilation/utils.py index 51c71e25ee2e..39c05c16ab8a 100644 --- a/tensorrt_llm/_torch/compilation/utils.py +++ b/tensorrt_llm/_torch/compilation/utils.py @@ -121,6 +121,9 @@ def inplace_info(): torch.ops.trtllm.cute_dsl_bf16_gemm_blackwell.default: { 1: "output" }, + torch.ops.trtllm.inplace_slice_copy.default: { + 1: "dest" + } } if IS_CUDA_TILE_AVAILABLE: # cuda.tile availability depends on GPU capability thus runtime check. diff --git a/tensorrt_llm/_torch/custom_ops/__init__.py b/tensorrt_llm/_torch/custom_ops/__init__.py index 719709e85f88..1963ac61d418 100644 --- a/tensorrt_llm/_torch/custom_ops/__init__.py +++ b/tensorrt_llm/_torch/custom_ops/__init__.py @@ -1,3 +1,5 @@ +import torch + from ..cuda_tile_utils import IS_CUDA_TILE_AVAILABLE from ..cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE @@ -10,6 +12,12 @@ # modules.attention and must be imported from there. They are not re-exported here to # avoid circular imports: custom_ops must not depend on modules.attention. + +def inplace_slice_copy(dest: torch.Tensor, src: torch.Tensor, dim1_start: int, + dim1_end: int) -> None: + torch.ops.trtllm.inplace_slice_copy(dest, src, dim1_start, dim1_end) + + __all__ = [ 'IS_FLASHINFER_AVAILABLE', '_register_fake', @@ -20,6 +28,7 @@ 'copy_to_userbuffers', 'matmul_to_ub', 'IS_CUTLASS_DSL_AVAILABLE', + 'inplace_slice_copy', ] if IS_FLASHINFER_AVAILABLE: diff --git a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py index 644155e9a4a7..b31bb56230c6 100644 --- a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py @@ -204,6 +204,10 @@ def _(scores, scores_with_bias, n_group, topk_group, topk, dtype=scores_with_bias.dtype), scores.new_empty( shape, dtype=torch.int32) + @torch.library.register_fake("trtllm::inplace_slice_copy") + def _(dest, src, dim1_start, dim1_end): + pass + @torch.library.register_fake("trtllm::indexer_topk_prefill") def _(logits, row_starts, row_ends, indices, index_topk): # In-place operation, no return value (void function) diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index 10ecd90966cd..b5b0b9e24877 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -4,6 +4,7 @@ import torch from torch import nn +from tensorrt_llm._torch.custom_ops import inplace_slice_copy from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.mapping import Mapping @@ -457,13 +458,13 @@ def maybe_capture_hidden_states( layer_id: int, hidden_states: torch.Tensor, residual: Optional[torch.Tensor] = None) -> None: + for i, captured_layer_id in enumerate(self.layers_to_capture): if captured_layer_id == layer_id: - num_tokens = hidden_states.shape[0] to_save = hidden_states + residual if residual is not None else hidden_states - self.hidden_states[:num_tokens, i * self.hidden_size:(i + 1) * - self.hidden_size].copy_(to_save, - non_blocking=True) + inplace_slice_copy(self.hidden_states, to_save, + i * self.hidden_size, + (i + 1) * self.hidden_size) break diff --git a/tests/unittest/_torch/thop/parallel_hw_agnostic/test_inplace_slice_copy.py b/tests/unittest/_torch/thop/parallel_hw_agnostic/test_inplace_slice_copy.py new file mode 100644 index 000000000000..07b2df615c4e --- /dev/null +++ b/tests/unittest/_torch/thop/parallel_hw_agnostic/test_inplace_slice_copy.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for trtllm::inplace_slice_copy. + +Verifies that the cudaMemcpy2DAsync-backed op produces the same result as a +reference Python slice + Tensor.copy_, for the row-prefix / column-slice +write pattern used in EAGLE3 hidden-state capture. +""" + +import pytest +import torch + +import tensorrt_llm # noqa: F401 + + +def _reference(dest_shape, src, dim1_start, dim1_end, dtype): + dest = torch.zeros(dest_shape, dtype=dtype, device="cuda") + num_tokens = src.shape[0] + dest[:num_tokens, dim1_start:dim1_end].copy_(src) + return dest + + +def _run(dest_shape, src, dim1_start, dim1_end, dtype): + dest = torch.zeros(dest_shape, dtype=dtype, device="cuda") + torch.ops.trtllm.inplace_slice_copy(dest, src, dim1_start, dim1_end) + return dest + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) +def test_full_dest_full_width(dtype): + """num_tokens == dest.size(0) and slice == full dest width.""" + dest_shape = (16, 64) + src = torch.randn(16, 64, dtype=dtype, device="cuda") + out = _run(dest_shape, src, 0, 64, dtype) + ref = _reference(dest_shape, src, 0, 64, dtype) + torch.testing.assert_close(out, ref) + + +def test_partial_rows(): + """num_tokens < dest.size(0): trailing rows must stay zero.""" + dtype = torch.bfloat16 + dest_shape = (32, 64) + src = torch.randn(8, 64, dtype=dtype, device="cuda") + out = _run(dest_shape, src, 0, 64, dtype) + ref = _reference(dest_shape, src, 0, 64, dtype) + torch.testing.assert_close(out, ref) + assert torch.all(out[8:] == 0) + + +def test_column_slice_middle(): + """Write to a middle column band; flanking columns must stay zero.""" + dtype = torch.bfloat16 + dest_shape = (16, 96) + src = torch.randn(16, 32, dtype=dtype, device="cuda") + out = _run(dest_shape, src, 32, 64, dtype) + ref = _reference(dest_shape, src, 32, 64, dtype) + torch.testing.assert_close(out, ref) + assert torch.all(out[:, :32] == 0) + assert torch.all(out[:, 64:] == 0) + + +def test_layered_capture_pattern(): + """Mimic EAGLE3 hidden-state capture: write each layer into its band.""" + dtype = torch.bfloat16 + num_tokens, hidden_size, num_layers = 12, 48, 3 + dest_shape = (24, hidden_size * num_layers) + srcs = [ + torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") for _ in range(num_layers) + ] + + out = torch.zeros(dest_shape, dtype=dtype, device="cuda") + for i, s in enumerate(srcs): + torch.ops.trtllm.inplace_slice_copy(out, s, i * hidden_size, (i + 1) * hidden_size) + + ref = torch.zeros(dest_shape, dtype=dtype, device="cuda") + for i, s in enumerate(srcs): + ref[:num_tokens, i * hidden_size : (i + 1) * hidden_size].copy_(s) + + torch.testing.assert_close(out, ref) + + +def test_empty_src_is_noop(): + """num_tokens == 0 must not modify dest and must not raise.""" + dtype = torch.bfloat16 + dest_shape = (16, 64) + dest = torch.full(dest_shape, 7, dtype=dtype, device="cuda") + src = torch.empty(0, 32, dtype=dtype, device="cuda") + torch.ops.trtllm.inplace_slice_copy(dest, src, 16, 48) + assert torch.all(dest == 7) + + +def test_dtype_mismatch_raises(): + dest = torch.zeros(8, 32, dtype=torch.bfloat16, device="cuda") + src = torch.randn(8, 32, dtype=torch.float16, device="cuda") + with pytest.raises(RuntimeError): + torch.ops.trtllm.inplace_slice_copy(dest, src, 0, 32) + + +def test_out_of_bounds_raises(): + dtype = torch.bfloat16 + dest = torch.zeros(8, 32, dtype=dtype, device="cuda") + src = torch.randn(8, 8, dtype=dtype, device="cuda") + with pytest.raises(RuntimeError): + torch.ops.trtllm.inplace_slice_copy(dest, src, 28, 36) + + +def test_negative_dim1_start_raises(): + """A negative dim1_start would underflow the dest pointer.""" + dtype = torch.bfloat16 + dest = torch.zeros(8, 32, dtype=dtype, device="cuda") + src = torch.randn(8, 8, dtype=dtype, device="cuda") + with pytest.raises(RuntimeError): + torch.ops.trtllm.inplace_slice_copy(dest, src, -8, 0) + + +def test_device_mismatch_raises(): + """dest and src on different CUDA devices must be rejected.""" + if torch.cuda.device_count() < 2: + pytest.skip("requires >= 2 CUDA devices") + dtype = torch.bfloat16 + dest = torch.zeros(8, 32, dtype=dtype, device="cuda:0") + src = torch.randn(8, 32, dtype=dtype, device="cuda:1") + with pytest.raises(RuntimeError): + torch.ops.trtllm.inplace_slice_copy(dest, src, 0, 32) From 059de9cbe11956a1f274e5fa880c8ee4c40e12df Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 1 Jun 2026 09:01:32 -0700 Subject: [PATCH 167/308] [None][fix] PyExecutor Hang in Disagg TP Prefill (#14020) Signed-off-by: jthomson04 --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 57 ++++++++++++++----- .../_torch/executor/test_benchmark_disagg.py | 3 + 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 739484a59b64..dbfe693eed62 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -42,6 +42,7 @@ get_global_profiler, host_profiler_context) from ..distributed import Distributed +from ..distributed.communicator import ReduceOp from ..expert_statistic import ExpertStatistic from ..models.modeling_llama import Llama4ForConditionalGeneration from ..models.modeling_utils import DecoderModelForCausalLM @@ -1926,16 +1927,26 @@ def _executor_loop_pp(self): and req.py_disaggregated_params.schedule_style == DisaggScheduleStyle.GENERATION_FIRST for req in self.active_requests) - if num_fitting_reqs == 0 and not fitting_disagg_gen_init_requests: - if not all_gen_first: + # [disagg-ctx-deadlock-fix] Mirror of the OR-gated entry in + # _executor_loop: ensure every TP rank either calls + # _check_disagg_ctx_cache_transfer_status together or skips + # it together, so the internal allgather in + # CacheTransceiver::checkContextTransferStatus always has + # full quorum. With PP > 1 the schedule is broadcast from + # rank 0 so num_fitting_reqs should already be uniform, but + # has_any_inflight_requests is rank-local and could + # otherwise diverge. + local_need_check = (num_fitting_reqs == 0 and + not fitting_disagg_gen_init_requests) + any_need_check = self.dist.allreduce(int(local_need_check), + op=ReduceOp.MAX) + if any_need_check > 0: + if local_need_check and not all_gen_first: logger.warning( "num_fitting_reqs=0 and fitting_disagg_gen_init_requests is empty, may not have enough kvCache" ) self._check_disagg_ctx_cache_transfer_status(1) - elif self.async_transfer_manager.has_any_inflight_requests( - ): - # Non-blocking cleanup of completed/timed-out - # transfers to free KV blocks (see _executor_loop). + else: self._check_disagg_ctx_cache_transfer_status(0) self.num_scheduled_requests = scheduled_batch.batch_size @@ -2471,18 +2482,36 @@ def _prepare_and_schedule_batch(self): req.py_disaggregated_params and req.py_disaggregated_params. schedule_style == DisaggScheduleStyle.GENERATION_FIRST for req in self.active_requests) - if num_fitting_reqs == 0 and not fitting_disagg_gen_init_requests: - if not all_gen_first: + # [disagg-ctx-deadlock-fix] _check_disagg_ctx_cache_transfer_status + # internally invokes a TP-wide allgather inside + # CacheTransceiver::checkContextTransferStatus. Gating the call on + # rank-local `num_fitting_reqs` (which can drift between ranks by + # one block due to per-rank UCX/CUDA-event-sync timing variance) + # lets some ranks enter the allgather while others skip ahead to + # model_forward / kv_connector → cross-rank collective-mismatch + # deadlock. OR the decision across TP ranks: if ANY rank wants the + # call, ALL ranks call it. Ranks that don't locally need it use the + # non-blocking variant so the collective stays in sync without + # holding any individual rank. + local_need_check = (num_fitting_reqs == 0 + and not fitting_disagg_gen_init_requests) + any_need_check = self.dist.allreduce(int(local_need_check), + op=ReduceOp.MAX) + if any_need_check > 0: + if local_need_check and not all_gen_first: logger.warning( "num_fitting_reqs=0 and fitting_disagg_gen_init_requests is empty, may not have enough kvCache" ) + # Local conditions warrant a blocking wait for at least one + # in-flight transfer to complete so KV blocks can be freed. self._check_disagg_ctx_cache_transfer_status(1) - elif self.async_transfer_manager.has_any_inflight_requests(): - # Non-blocking cleanup of completed/timed-out transfers - # to free KV blocks. We avoid the blocking check because - # gen-first requests may be waiting for peer info (which - # would block indefinitely), but completed transfers must - # still be reaped so that KV cache can be reclaimed. + else: + # Either (a) a peer rank needed the call but we didn't, or + # (b) all active requests are gen-first so we don't + # actively block. In both cases the non-blocking variant + # still runs the internal allgather (keeping all ranks in + # sync) and reaps any already-completed transfers without + # blocking on un-finished ones. self._check_disagg_ctx_cache_transfer_status(0) # In gen-only benchmark mode, all requests must fit in KV cache diff --git a/tests/unittest/_torch/executor/test_benchmark_disagg.py b/tests/unittest/_torch/executor/test_benchmark_disagg.py index f801e8e0e162..41cbe241defb 100644 --- a/tests/unittest/_torch/executor/test_benchmark_disagg.py +++ b/tests/unittest/_torch/executor/test_benchmark_disagg.py @@ -505,6 +505,7 @@ def test_fetch_called_once_even_in_benchmark_disagg(self): ex.enable_attention_dp = False ex.num_fetch_requests = 0 ex.dist = Mock(rank=0, tp_size=1) + ex.dist.allreduce = Mock(side_effect=lambda v, op=None: v) ex.is_shutdown = False ex._is_warmup = False ex.enable_iter_perf_stats = False @@ -894,6 +895,7 @@ def _make_executor( ex.enable_attention_dp = False ex.num_fetch_requests = num_fetch_requests ex.dist = Mock(rank=0, tp_size=1) + ex.dist.allreduce.return_value = 0 ex.is_shutdown = False ex._is_warmup = False ex.enable_iter_perf_stats = False @@ -1040,6 +1042,7 @@ def _make_executor(self): ex.num_fetch_requests = 0 ex.max_num_active_requests = self.MAX_BATCH_SIZE ex.dist = Mock(rank=0, tp_size=self.TP_SIZE) + ex.dist.allreduce.return_value = 0 ex.is_shutdown = False ex._is_warmup = False ex.enable_iter_perf_stats = False From d5b19bdb169f203ed92c14ca32b1c8647c6f9c9f Mon Sep 17 00:00:00 2001 From: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:08:33 -0700 Subject: [PATCH 168/308] [https://nvbugs/6240561][fix] Autodeploy fix the deepseek accuracy drop (#14774) Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- .../transform/library/fuse_rope_mla.py | 19 +++++++++++-------- .../singlegpu/models/test_deepseek_custom.py | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py index d3d1563ed444..2e293ffbc745 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py @@ -231,8 +231,10 @@ def _compute_rotary_cos_sin_from_config( max_position = getattr(model_config, "max_position_embeddings", 8192) half_dim = qk_rope_head_dim // 2 + # Match the custom model's expression order; fused MLA is sensitive to RoPE table drift. + dim_indices = torch.arange(0, qk_rope_head_dim, 2, dtype=torch.float32) - inv_freq = 1.0 / (rope_theta ** (torch.arange(0, half_dim, dtype=torch.float32) / half_dim)) + inv_freq = 1.0 / (rope_theta ** (dim_indices / qk_rope_head_dim)) # Handle YaRN scaling if present rope_scaling = getattr(model_config, "rope_scaling", None) @@ -255,7 +257,7 @@ def _yarn_get_mscale(scale_val, m): # The model's softmax_scale already includes mscale^2, so the # cos/sin table must NOT duplicate it. numerator = _yarn_get_mscale(factor, mscale_factor) - denominator = _yarn_get_mscale(factor, mscale_all_dim) if mscale_all_dim else 1.0 + denominator = _yarn_get_mscale(factor, mscale_all_dim) mscale = numerator / denominator if denominator != 0 else 1.0 original_max = rope_scaling.get("original_max_position_embeddings", max_position) beta_fast = rope_scaling.get("beta_fast", 32) @@ -277,15 +279,16 @@ def _yarn_find_correction_dim(num_rotations, dim, base, max_pos): if low == high: high += 0.001 - smooth = (torch.arange(half_dim, dtype=torch.float32) - low) / (high - low) - smooth = smooth.clamp(0, 1) - freq_extra = inv_freq - freq_inter = inv_freq / factor - inv_freq = freq_inter * smooth + freq_extra * (1 - smooth) + inv_freq_mask = 1.0 - torch.clamp( + (torch.arange(half_dim, dtype=torch.float32) - low) / (high - low), 0, 1 + ) + freq_extra = 1.0 / (rope_theta ** (dim_indices / qk_rope_head_dim)) + freq_inter = 1.0 / (factor * rope_theta ** (dim_indices / qk_rope_head_dim)) + inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask t = torch.arange(max_position, dtype=torch.float32) freqs = torch.outer(t, inv_freq) - emb = torch.cat([freqs, freqs], dim=-1) + emb = torch.cat((freqs, freqs), dim=-1) cos_vals = emb.cos() * mscale sin_vals = emb.sin() * mscale result = torch.stack([cos_vals, sin_vals], dim=-1) diff --git a/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py b/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py index 91ab98297b6e..cf9c4eb8126c 100644 --- a/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py +++ b/tests/unittest/auto_deploy/singlegpu/models/test_deepseek_custom.py @@ -190,6 +190,7 @@ def _get_model_config(self): return config, None actual = _compute_rotary_cos_sin_from_config(Factory()).cpu() + assert torch.equal(actual, expected) torch.testing.assert_close(actual, expected, atol=3e-7, rtol=1e-4) From 02a65b57ec0e90393b74f77ef83de924fd825055 Mon Sep 17 00:00:00 2001 From: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:33:51 -0700 Subject: [PATCH 169/308] [#12702][feat] Autodeploy deprecate the legacy triton attention (#14194) Signed-off-by: Chenghao Zhang <211069071+nvchenghaoz@users.noreply.github.com> --- .claude/skills/ad-sharding-ir-port/SKILL.md | 2 +- .../cookbooks/gemma_4_trtllm_cookbook.ipynb | 2 +- .../model_registry/configs/gemma4_dense.yaml | 4 +- .../model_registry/configs/gemma4_e2b.yaml | 2 +- .../model_registry/configs/gemma4_moe.yaml | 4 +- .../configs/gemma4_moe_base.yaml | 4 +- .../auto_deploy/compile/piecewise_utils.py | 5 +- .../_torch/auto_deploy/custom_ops/README.md | 3 +- .../custom_ops/attention/__init__.py | 7 +- .../custom_ops/attention/triton_attention.py | 1721 ++++++++++++++--- .../triton_attention_with_kv_cache.py | 882 --------- .../attention/triton_paged_attention.py | 1578 --------------- .../models/custom/modeling_gemma4.py | 2 +- .../auto_deploy/transform/library/kvcache.py | 8 - .../defs/accuracy/test_llm_api_autodeploy.py | 4 +- .../test_lists/qa/llm_function_core.txt | 2 +- .../test_lists/test-db/l0_b200.yml | 4 +- .../test_lists/test-db/l0_h100.yml | 2 +- .../custom_ops/attention/test_attention_op.py | 111 -- ..._attention.py => test_triton_attention.py} | 259 ++- ...on_swa.py => test_triton_attention_swa.py} | 22 +- .../test_triton_attention_with_kv_cache.py | 724 ------- .../transformations/library/test_kv_cache.py | 22 +- .../library/test_kvcache_vswa_metadata.py | 10 +- .../library/test_shared_kv_attention.py | 50 +- 25 files changed, 1692 insertions(+), 3742 deletions(-) delete mode 100644 tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention_with_kv_cache.py delete mode 100644 tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py delete mode 100644 tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_attention_op.py rename tests/unittest/auto_deploy/singlegpu/custom_ops/attention/{test_triton_paged_attention.py => test_triton_attention.py} (90%) rename tests/unittest/auto_deploy/singlegpu/custom_ops/attention/{test_triton_paged_attention_swa.py => test_triton_attention_swa.py} (97%) delete mode 100644 tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_with_kv_cache.py diff --git a/.claude/skills/ad-sharding-ir-port/SKILL.md b/.claude/skills/ad-sharding-ir-port/SKILL.md index 1b2aa804d3e0..a2d0d2df59ec 100644 --- a/.claude/skills/ad-sharding-ir-port/SKILL.md +++ b/.claude/skills/ad-sharding-ir-port/SKILL.md @@ -263,6 +263,6 @@ You are NOT done until every row in the table is a yes-allowed category. - All four configurations of the **sharding equivalence test** (Step 11b) pass with the parsed `rel_rmse` strictly below the parsed `tol` from the same rank-0 log line. Report the per-cell `rel_rmse` and `tol` pair. - `world_size=1`: unsharded path; hints should not break correctness. -- `world_size=`: end-to-end run (Step 11a) at the maximum GPU count auto-detected on the machine (head-divisibility permitting; see Step 11). +- `world_size=`: end-to-end run (Step 11a) at the maximum GPU count auto-detected on the machine (head-divisibility permitting; see Step 11). - `apply_sharding_hints` node count vs expectation. - Optional: `shard_layers: ['moe']` to verify selective sharding. diff --git a/examples/auto_deploy/cookbooks/gemma_4_trtllm_cookbook.ipynb b/examples/auto_deploy/cookbooks/gemma_4_trtllm_cookbook.ipynb index 9dc9dee9d801..1ba86f5cdc7d 100644 --- a/examples/auto_deploy/cookbooks/gemma_4_trtllm_cookbook.ipynb +++ b/examples/auto_deploy/cookbooks/gemma_4_trtllm_cookbook.ipynb @@ -16,7 +16,7 @@ "- Base MoE (no chat template): [`google/gemma-4-26B-A4B`](https://huggingface.co/google/gemma-4-26B-A4B)\n", "\n", "**Bundled AutoDeploy YAML (this branch):**\n", - "- **Instruction:** `examples/auto_deploy/model_registry/configs/gemma4_moe.yaml` — text-only export path; `attn_backend: triton_paged` (head_dim 512 / paged KV, CUDA-graph friendly).\n", + "- **Instruction:** `examples/auto_deploy/model_registry/configs/gemma4_moe.yaml` — text-only export path; `attn_backend: triton` (head_dim 512 / paged KV, CUDA-graph friendly).\n", "- **Base:** `examples/auto_deploy/model_registry/configs/gemma4_moe_base.yaml` — same stack for the base checkpoint.\n", "\n", "`trtllm-serve` takes **one** YAML path via `--extra_llm_api_options` (or `--config`). The bundled MoE YAMLs omit `world_size`; add it (or copy the YAML and edit) so it matches your GPU count when you use tensor parallel or multi-GPU loading.\n" diff --git a/examples/auto_deploy/model_registry/configs/gemma4_dense.yaml b/examples/auto_deploy/model_registry/configs/gemma4_dense.yaml index 302e2da9b41f..0cba86f38348 100644 --- a/examples/auto_deploy/model_registry/configs/gemma4_dense.yaml +++ b/examples/auto_deploy/model_registry/configs/gemma4_dense.yaml @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # Gemma 4 dense (31B) — text-only AD export path. -# Uses triton paged attention backend: supports head_dim=512 (global_head_dim), +# Uses triton attention backend: supports head_dim=512 (global_head_dim), # paged KV cache, CUDA-graph-compatible, FlashDecoding for decode. model_factory: Gemma4ForConditionalGeneration tokenizer: google/gemma-4-31B-it -attn_backend: triton_paged +attn_backend: triton compile_backend: torch-cudagraph cuda_graph_config: batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/examples/auto_deploy/model_registry/configs/gemma4_e2b.yaml b/examples/auto_deploy/model_registry/configs/gemma4_e2b.yaml index b0fbfe194fdc..756035edbca2 100644 --- a/examples/auto_deploy/model_registry/configs/gemma4_e2b.yaml +++ b/examples/auto_deploy/model_registry/configs/gemma4_e2b.yaml @@ -7,7 +7,7 @@ model_factory: Gemma4ForConditionalGeneration # Use the instruction-tuned tokenizer/chat template for end-to-end prompting. tokenizer: google/gemma-4-E2B-it -attn_backend: triton_paged +attn_backend: triton compile_backend: torch-cudagraph cuda_graph_config: batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/examples/auto_deploy/model_registry/configs/gemma4_moe.yaml b/examples/auto_deploy/model_registry/configs/gemma4_moe.yaml index 701f2451487d..e5feb99f5bb0 100644 --- a/examples/auto_deploy/model_registry/configs/gemma4_moe.yaml +++ b/examples/auto_deploy/model_registry/configs/gemma4_moe.yaml @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # Gemma 4 MoE (26B total, 4B activated) — text-only AD export path. -# Uses triton paged attention backend: supports head_dim=512 (global_head_dim), +# Uses triton attention backend: supports head_dim=512 (global_head_dim), # paged KV cache, CUDA-graph-compatible, FlashDecoding for decode. model_factory: Gemma4ForConditionalGeneration tokenizer: google/gemma-4-26B-A4B-it -attn_backend: triton_paged +attn_backend: triton compile_backend: torch-cudagraph cuda_graph_config: batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/examples/auto_deploy/model_registry/configs/gemma4_moe_base.yaml b/examples/auto_deploy/model_registry/configs/gemma4_moe_base.yaml index 9e469676559f..522b83eb475e 100644 --- a/examples/auto_deploy/model_registry/configs/gemma4_moe_base.yaml +++ b/examples/auto_deploy/model_registry/configs/gemma4_moe_base.yaml @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # Gemma 4 MoE base (26B total, 4B activated) — text-only AD export path. -# Uses triton paged attention backend: supports head_dim=512 (global_head_dim), +# Uses triton attention backend: supports head_dim=512 (global_head_dim), # paged KV cache, CUDA-graph-compatible, FlashDecoding for decode. model_factory: Gemma4ForConditionalGeneration tokenizer: google/gemma-4-26B-A4B -attn_backend: triton_paged +attn_backend: triton compile_backend: torch-cudagraph cuda_graph_config: batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tensorrt_llm/_torch/auto_deploy/compile/piecewise_utils.py b/tensorrt_llm/_torch/auto_deploy/compile/piecewise_utils.py index affe3b8301f2..fe3093d1b014 100644 --- a/tensorrt_llm/_torch/auto_deploy/compile/piecewise_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/compile/piecewise_utils.py @@ -40,8 +40,7 @@ # Cached attention ops (grid depends on per-sequence lengths) _CACHED_ATTENTION_OPS = [ "auto_deploy::flashinfer_attention_mha_with_cache", - "auto_deploy::triton_attention_flattened_mha_with_cache", - "auto_deploy::triton_paged_mha_with_cache", + "auto_deploy::triton_mha_with_cache", "auto_deploy::torch_cached_attention_with_cache", "auto_deploy::trtllm_attention_mha_with_cache", # MLA attention variants @@ -74,7 +73,7 @@ _METADATA_PREP_OPS = [ "auto_deploy::flashinfer_attention_prepare_metadata", "auto_deploy::flashinfer_mla_prepare_metadata", - "auto_deploy::triton_paged_prepare_metadata", + "auto_deploy::triton_prepare_metadata", "auto_deploy::mamba_ssm_prepare_metadata", ] diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/README.md b/tensorrt_llm/_torch/auto_deploy/custom_ops/README.md index 98c4fa9ae7cf..2e0ad3e5a2ce 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/README.md +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/README.md @@ -18,7 +18,8 @@ The table below lists the operators grouped by category. | `torch.ops.auto_deploy.torch_cached_attention_with_cache` | PyTorch backend attention with KV cache management | | `torch.ops.auto_deploy.flashinfer_attention_mha_with_cache` | FlashInfer multi-head attention with KV cache support | | `torch.ops.auto_deploy.flashinfer_attention_prepare_metadata` | FlashInfer attention metadata preparation | -| `torch.ops.auto_deploy.triton_attention_flattened_mha_with_cache` | Triton flattened MHA with cache | +| `torch.ops.auto_deploy.triton_mha_with_cache` | Triton multi-head attention with paged KV cache support | +| `torch.ops.auto_deploy.triton_prepare_metadata` | Triton attention metadata preparation | #### MLA (Multi-head Latent Attention) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/__init__.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/__init__.py index 4d47fb4b7586..b76beb693b98 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/__init__.py @@ -20,9 +20,7 @@ - torch_backend_attention: PyTorch-based attention backend - flashinfer_attention: FlashInfer-based optimized attention - trtllm_attention: TRT-LLM thop.attention-based optimized attention -- triton_attention: Triton-based attention implementations -- triton_attention_with_kv_cache: Triton attention with KV cache support -- triton_paged_attention: Triton paged attention (two-stage flash-decode) with HND layout +- triton_attention: Triton attention with paged KV cache and two-stage flash-decode """ __all__ = [ @@ -31,7 +29,4 @@ "flashinfer_attention", "trtllm_attention", "triton_attention", - "triton_attention_with_kv_cache", - "triton_attention_with_paged_kv_cache", - "triton_paged_attention", ] diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention.py index fedbc1401d44..83e4911dea0e 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention.py @@ -13,13 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Custom ops for MHA/XQA attention.""" +"""Triton Attention + +This module provides a Triton-based paged attention implementation with: +- Combined KV cache with HND layout: [num_blocks, 2, num_kv_heads, page_size, head_dim] +- Context/prefill kernel with causal masking +""" import math -from typing import List, Optional +from typing import List, Literal, Optional, Tuple +import flashinfer import torch import triton +import triton.language as tl from torch._ops import OpOverloadPacket from torch._subclasses import FakeTensor from torch.fx import Node @@ -31,322 +38,1472 @@ AttentionDescriptor, AttentionLayout, AttentionRegistry, - BatchInfo, Constant, + KVPagedResourceHandler, MHACallable, + PrepareMetadataCallable, ResourceHandlerDict, - UnpagedResourceHandler, -) -from .triton_attention_with_kv_cache import ( - attention_kv_stage2, - context_attention_kv_flattened, - gqa_attention_kv_stage1, - update_kv_cache, ) +KV_LAYOUT: Literal["HND", "NHD"] = "HND" -def _decode_attention( - q: torch.Tensor, # [num_decode, num_heads, qk_head_dim] - k: torch.Tensor, # [num_decode, num_kv_heads, qk_head_dim] - v: torch.Tensor, # [num_decode, num_kv_heads, v_head_dim] - k_cache: torch.Tensor, - v_cache: torch.Tensor, - slot_idx: torch.Tensor, # [num_decode] - input_pos: torch.Tensor, # [num_decode] - scale: float, - out: torch.Tensor, # [num_decode, num_heads, v_head_dim] - sinks: Optional[torch.Tensor] = None, - sliding_window: Optional[int] = None, -) -> None: - """Handle decode phase - single token generation attention.""" - num_decode = q.shape[0] - n_heads, q_d_head = q.shape[-2:] - max_seq_len, n_kv_heads = k_cache.shape[1:3] - v_d_head = v.shape[-1] - device = q.device - - SEQ_BLOCK_SIZE = 64 - num_blocks = (max_seq_len + SEQ_BLOCK_SIZE - 1) // SEQ_BLOCK_SIZE - stage1_output_values = torch.empty( - num_decode, n_heads, num_blocks, v_d_head, device=device, dtype=torch.float32 +# Cache SM count to avoid repeated get_device_properties calls +_NUM_SMS: Optional[int] = None +_MIN_TL_DOT_K = 16 + + +def _get_num_sms() -> int: + """Get the number of SMs on the current GPU (cached).""" + global _NUM_SMS + if _NUM_SMS is None: + _NUM_SMS = torch.cuda.get_device_properties(0).multi_processor_count + return _NUM_SMS + + +def _get_page_block(page_size: int) -> int: + """Return the page tile width used by Triton dot kernels.""" + return max(_MIN_TL_DOT_K, 1 << (page_size - 1).bit_length()) + + +def _get_sm_scale(head_dim: int, scale: Optional[float]) -> float: + """Get softmax scale, computing default if not provided.""" + return scale if scale is not None else 1.0 / math.sqrt(head_dim) + + +@triton.jit +def _update_paged_kv_cache_kernel( + # Input K, V + k_ptr, + v_ptr, + # Metadata + batch_indices_ptr, + positions_ptr, + # KV cache + kv_cache_ptr, + # Page table + kv_indices_ptr, + kv_indptr_ptr, + # Constants + NUM_TOKENS: tl.constexpr, + N_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + PAGE_SIZE: tl.constexpr, + # Strides for kv_cache: [num_blocks, 2, num_kv_heads, page_size, head_dim] + cache_stride_block: tl.constexpr, + cache_stride_kv: tl.constexpr, + cache_stride_head: tl.constexpr, + cache_stride_token: tl.constexpr, +): + """Update combined KV cache with new tokens.""" + token_id = tl.program_id(axis=0) + head_id = tl.program_id(axis=1) + + if token_id >= NUM_TOKENS: + return + + batch_idx = tl.load(batch_indices_ptr + token_id) + position = tl.load(positions_ptr + token_id) + + page_idx_in_seq = position // PAGE_SIZE + offset_in_page = position % PAGE_SIZE + + page_start = tl.load(kv_indptr_ptr + batch_idx) + physical_page = tl.load(kv_indices_ptr + page_start + page_idx_in_seq) + + head_offsets = tl.arange(0, HEAD_DIM) + kv_offset = token_id * N_KV_HEADS * HEAD_DIM + head_id * HEAD_DIM + head_offsets + + k = tl.load(k_ptr + kv_offset) + v = tl.load(v_ptr + kv_offset) + + # Compute cache offset (use int64 to avoid overflow when physical_page * stride > 2^31) + cache_base = ( + physical_page.to(tl.int64) * cache_stride_block + + head_id * cache_stride_head + + offset_in_page.to(tl.int64) * cache_stride_token + + head_offsets ) - stage1_output_logsumexp = torch.empty( - num_decode, n_heads, num_blocks, device=device, dtype=torch.float32 - ) - float("inf") - update_kv_cache[(num_decode, n_kv_heads, 1)]( + tl.store(kv_cache_ptr + cache_base, k) + tl.store(kv_cache_ptr + cache_base + cache_stride_kv, v) + + +def update_paged_kv_cache( + k: torch.Tensor, + v: torch.Tensor, + batch_indices: torch.Tensor, + positions: torch.Tensor, + kv_cache: torch.Tensor, + kv_indices: torch.Tensor, + kv_indptr: torch.Tensor, +) -> None: + """Update the combined paged KV cache with new K, V tensors.""" + num_tokens, n_kv_heads, head_dim = k.shape + page_size = kv_cache.shape[3] + + if num_tokens == 0: + return + + grid = (num_tokens, n_kv_heads) + _update_paged_kv_cache_kernel[grid]( k, v, - None, - None, - k_cache, - v_cache, - input_pos, - slot_idx, - max_seq_len, - n_kv_heads, - q_d_head, - v_d_head, - 1, - GENERATE_ONLY=True, + batch_indices, + positions, + kv_cache, + kv_indices, + kv_indptr, + NUM_TOKENS=num_tokens, + N_KV_HEADS=n_kv_heads, + HEAD_DIM=head_dim, + PAGE_SIZE=page_size, + cache_stride_block=kv_cache.stride(0), + cache_stride_kv=kv_cache.stride(1), + cache_stride_head=kv_cache.stride(2), + cache_stride_token=kv_cache.stride(3), ) - HEAD_BLOCK_SIZE = max(16, triton.next_power_of_2(n_heads // n_kv_heads)) - gqa_attention_kv_stage1[ - ( - num_decode, - n_kv_heads, - num_blocks, + +# ============================================================================= +# FLASH DECODE - HELPERS +# ============================================================================= + + +def _get_num_splits(max_seq_len: int, batch_size: int, n_kv_heads: int, page_size: int) -> int: + """Compute optimal number of KV splits for FlashDecoding. + + With GQA batching, the grid is (batch, n_kv_heads, num_splits). + We want enough blocks to saturate the GPU. + """ + if max_seq_len <= 0: + return 1 + + num_sms = _get_num_sms() + existing_parallelism = batch_size * n_kv_heads + + # Already enough parallelism + if existing_parallelism >= num_sms * 2: + return 1 + + # Target ~4 waves of thread blocks + target_blocks = num_sms * 4 + num_splits = max(1, (target_blocks + existing_parallelism - 1) // existing_parallelism) + + # Cap splits so each block has at least 2 pages of work. + # With fewer pages, the per-block overhead (Q load, accumulator init, + # partial_o/lse store, plus stage2 reduction cost) dominates the useful + # compute (page-loop iterations). 2 pages is a conservative lower bound + # to keep the overhead-to-work ratio acceptable. + max_pages = max_seq_len // page_size + max_splits = max(1, max_pages // 2) + num_splits = min(num_splits, max_splits) + + # Round to next power of 2 for Triton compile caching + if num_splits > 1: + num_splits = 2 ** math.ceil(math.log2(num_splits)) + + return min(num_splits, 128) + + +@triton.autotune( + configs=[ + triton.Config({}, num_warps=2, num_stages=2), + triton.Config({}, num_warps=2, num_stages=3), + triton.Config({}, num_warps=4, num_stages=2), + triton.Config({}, num_warps=4, num_stages=3), + triton.Config({}, num_warps=8, num_stages=2), + triton.Config({}, num_warps=8, num_stages=3), + ], + key=["HEAD_DIM", "PAGE_SIZE", "PAGE_BLOCK", "HEAD_RATIO_PADDED", "SLIDING_WINDOW"], +) +@triton.jit +def _flash_decode_stage1_kernel( + # Query input + q_ptr, + # KV cache (combined) + kv_cache_ptr, + # Page table + kv_indices_ptr, + kv_indptr_ptr, + kv_last_page_len_ptr, + # Intermediate outputs + partial_o_ptr, + partial_lse_ptr, + # Q strides: [batch, n_heads, head_dim] + q_stride_batch: tl.constexpr, + q_stride_head: tl.constexpr, + # Partial output strides: [batch, n_heads, num_splits, head_dim] + po_stride_batch: tl.constexpr, + po_stride_head: tl.constexpr, + po_stride_split: tl.constexpr, + # Partial LSE strides: [batch, n_heads, num_splits] + plse_stride_batch: tl.constexpr, + plse_stride_head: tl.constexpr, + plse_stride_split: tl.constexpr, + # Cache strides: [num_blocks, 2, n_kv_heads, page_size, head_dim] + cache_stride_block: tl.constexpr, + cache_stride_kv: tl.constexpr, + cache_stride_head: tl.constexpr, + cache_stride_token: tl.constexpr, + # Constants + SM_SCALE: tl.constexpr, + N_HEADS: tl.constexpr, + N_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_BLOCK: tl.constexpr, + HEAD_RATIO: tl.constexpr, + HEAD_RATIO_PADDED: tl.constexpr, + NUM_SPLITS: tl.constexpr, + SLIDING_WINDOW: tl.constexpr = 0, +): + """ + Key optimizations: + - Loads KV once for HEAD_RATIO Q heads + - Iterates by page for contiguous memory access + - Splits KV sequence across blocks for GPU utilization + """ + batch_id = tl.program_id(axis=0) + kv_head_id = tl.program_id(axis=1) + split_id = tl.program_id(axis=2) + + # Get sequence info from page table + kv_page_start = tl.load(kv_indptr_ptr + batch_id) + kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) + num_pages = kv_page_end - kv_page_start + last_page_len = tl.load(kv_last_page_len_ptr + batch_id) + + # Sliding window: restrict attention to pages within the window. + # Compute the total sequence length and the first valid KV position. + seq_len = (num_pages - 1) * PAGE_SIZE + last_page_len + if SLIDING_WINDOW > 0: + first_valid_pos = tl.maximum(0, seq_len - SLIDING_WINDOW) + first_window_page = first_valid_pos // PAGE_SIZE + else: + first_valid_pos = 0 + first_window_page = 0 + + # Only split over pages within the window + window_pages = num_pages - first_window_page + pages_per_split = (window_pages + NUM_SPLITS - 1) // NUM_SPLITS + page_split_start = first_window_page + split_id * pages_per_split + page_split_end = tl.minimum(page_split_start + pages_per_split, num_pages) + + dhead_offsets = tl.arange(0, HEAD_DIM) + # Use padded range for Triton power-of-2 requirement; mask out-of-bounds heads + head_local = tl.arange(0, HEAD_RATIO_PADDED) + head_ids = kv_head_id * HEAD_RATIO + head_local + head_mask = head_local < HEAD_RATIO + + # Handle inactive splits (beyond the sequence's pages) + if page_split_start >= num_pages: + # Store zeros + -inf LSE for valid HEAD_RATIO Q heads only + po_offsets = ( + batch_id * po_stride_batch + + head_ids[:, None] * po_stride_head + + split_id * po_stride_split + + dhead_offsets[None, :] ) - ]( - q, - k_cache, - v_cache, - slot_idx, - input_pos, - stage1_output_values, - stage1_output_logsumexp, - num_blocks, - scale, - max_seq_len, + tl.store( + partial_o_ptr + po_offsets, + tl.zeros([HEAD_RATIO_PADDED, HEAD_DIM], dtype=tl.float32), + mask=head_mask[:, None], + ) + plse_offsets = ( + batch_id * plse_stride_batch + + head_ids * plse_stride_head + + split_id * plse_stride_split + ) + tl.store( + partial_lse_ptr + plse_offsets, + tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) + float("-inf"), + mask=head_mask, + ) + return + + # Load Q for HEAD_RATIO heads sharing this KV head: [HEAD_RATIO_PADDED, HEAD_DIM] + # Padded rows get zeros, producing zero attention scores (harmless, never stored) + q_offsets = ( + batch_id * q_stride_batch + head_ids[:, None] * q_stride_head + dhead_offsets[None, :] + ) + q_all = tl.load(q_ptr + q_offsets, mask=head_mask[:, None], other=0.0) + + acc = tl.zeros([HEAD_RATIO_PADDED, HEAD_DIM], dtype=tl.float32) + m_i = tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) + float("-inf") + l_i = tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) + + num_pages_this_split = page_split_end - page_split_start + for local_page_idx in range(num_pages_this_split): + page_idx = page_split_start + local_page_idx + physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) + + # Determine valid tokens in this page + is_last_page_of_seq = page_idx == (num_pages - 1) + valid_tokens = tl.where(is_last_page_of_seq, last_page_len, PAGE_SIZE) + + page_offsets = tl.arange(0, PAGE_BLOCK) + page_mask = page_offsets < valid_tokens + + # Compute cache offset (use int64 to avoid overflow when physical_page * stride > 2^31) + cache_base = ( + physical_page.to(tl.int64) * cache_stride_block + + kv_head_id * cache_stride_head + + page_offsets[:, None] * cache_stride_token + + dhead_offsets[None, :] + ) + page_mask_2d = page_mask[:, None] + + k = tl.load(kv_cache_ptr + cache_base, mask=page_mask_2d, other=0.0).to( + q_all.dtype + ) # [PAGE_BLOCK, HEAD_DIM]; cast from fp8 if kv cache is fp8 + v = tl.load( + kv_cache_ptr + cache_base + cache_stride_kv, + mask=page_mask_2d, + other=0.0, + ).to(k.dtype) # [PAGE_BLOCK, HEAD_DIM]; cast from fp8 if kv cache is fp8 + + # [HEAD_RATIO_PADDED, HEAD_DIM] @ [HEAD_DIM, PAGE_BLOCK] -> [HEAD_RATIO_PADDED, PAGE_BLOCK] + attn = tl.dot(q_all, tl.trans(k)) * SM_SCALE + + # Combine validity mask with sliding window mask + if SLIDING_WINDOW > 0: + global_pos = page_idx * PAGE_SIZE + page_offsets + window_mask = global_pos >= first_valid_pos + attn = tl.where(page_mask[None, :] & window_mask[None, :], attn, float("-inf")) + else: + attn = tl.where(page_mask[None, :], attn, float("-inf")) + + # Online softmax update (vectorized over HEAD_RATIO_PADDED) + m_ij = tl.max(attn, axis=1) # [HEAD_RATIO_PADDED] + m_i_new = tl.maximum(m_i, m_ij) + alpha = tl.exp(m_i - m_i_new) + p = tl.exp(attn - m_i_new[:, None]) # [HEAD_RATIO_PADDED, PAGE_BLOCK] + + # [HEAD_RATIO_PADDED, PAGE_BLOCK] @ [PAGE_BLOCK, HEAD_DIM] -> [HEAD_RATIO_PADDED, HEAD_DIM] + acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_i_new + + # Finalize: normalize and compute LSE + l_i_safe = tl.where(l_i == 0.0, 1.0, l_i) + partial_o_val = acc / l_i_safe[:, None] # [HEAD_RATIO_PADDED, HEAD_DIM] + lse_val = m_i + tl.log(l_i_safe) # [HEAD_RATIO_PADDED] + + # Store results for valid HEAD_RATIO Q heads only (masked 2D store) + po_offsets = ( + batch_id * po_stride_batch + + head_ids[:, None] * po_stride_head + + split_id * po_stride_split + + dhead_offsets[None, :] + ) + tl.store(partial_o_ptr + po_offsets, partial_o_val, mask=head_mask[:, None]) + + plse_offsets = ( + batch_id * plse_stride_batch + head_ids * plse_stride_head + split_id * plse_stride_split + ) + tl.store(partial_lse_ptr + plse_offsets, lse_val, mask=head_mask) + + +@triton.jit +def _flash_decode_stage2_kernel( + # Partial results + partial_o_ptr, + partial_lse_ptr, + # Final output + o_ptr, + # Partial output strides: [batch, n_heads, num_splits, head_dim] + po_stride_batch: tl.constexpr, + po_stride_head: tl.constexpr, + po_stride_split: tl.constexpr, + # Partial LSE strides: [batch, n_heads, num_splits] + plse_stride_batch: tl.constexpr, + plse_stride_head: tl.constexpr, + plse_stride_split: tl.constexpr, + # Output strides: [batch, n_heads, head_dim] + o_stride_batch: tl.constexpr, + o_stride_head: tl.constexpr, + # Constants + HEAD_DIM: tl.constexpr, + NUM_SPLITS: tl.constexpr, +): + """ + Each program combines results from all splits for one (batch, head) pair. + """ + batch_id = tl.program_id(axis=0) + head_id = tl.program_id(axis=1) + + dhead_offsets = tl.arange(0, HEAD_DIM) + + # Find global maximum LSE across splits for numerical stability + global_max_lse = float("-inf") + for split_id in range(NUM_SPLITS): + plse_offset = ( + batch_id * plse_stride_batch + head_id * plse_stride_head + split_id * plse_stride_split + ) + lse = tl.load(partial_lse_ptr + plse_offset) + global_max_lse = tl.maximum(global_max_lse, lse) + + # Guard: if all splits had -inf LSE (empty sequence), output zeros + o_offset = batch_id * o_stride_batch + head_id * o_stride_head + dhead_offsets + if global_max_lse == float("-inf"): + tl.store(o_ptr + o_offset, tl.zeros([HEAD_DIM], dtype=tl.float32)) + return + + # Weighted combination: weight_i = exp(lse_i - global_max) + acc = tl.zeros([HEAD_DIM], dtype=tl.float32) + total_weight = 0.0 + + for split_id in range(NUM_SPLITS): + plse_offset = ( + batch_id * plse_stride_batch + head_id * plse_stride_head + split_id * plse_stride_split + ) + lse = tl.load(partial_lse_ptr + plse_offset) + weight = tl.exp(lse - global_max_lse) + + po_base = batch_id * po_stride_batch + head_id * po_stride_head + split_id * po_stride_split + partial_o = tl.load(partial_o_ptr + po_base + dhead_offsets) + + acc += weight * partial_o + total_weight += weight + + # Normalize and store + total_weight = tl.where(total_weight == 0.0, 1.0, total_weight) + o = acc / total_weight + tl.store(o_ptr + o_offset, o) + + +def triton_decode( + q: torch.Tensor, + kv_cache: torch.Tensor, + kv_indices: torch.Tensor, + kv_indptr: torch.Tensor, + kv_last_page_len: torch.Tensor, + sm_scale: float, + sliding_window: Optional[int] = None, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Optimized paged decode with GQA batching + FlashDecoding + page-aligned iteration. + + Args: + q: Query tensor [batch_size, n_heads, head_dim] + kv_cache: Combined cache [num_blocks, 2, n_kv_heads, page_size, head_dim] + kv_indices: Physical page indices (flattened) + kv_indptr: Cumulative page counts [batch_size + 1] + kv_last_page_len: Valid tokens in last page [batch_size] + sm_scale: Softmax scale factor + sliding_window: If set, only attend to the last sliding_window tokens + out: Optional output tensor [batch_size, n_heads, head_dim] + + Returns: + Output tensor [batch_size, n_heads, head_dim] + """ + batch_size, n_heads, head_dim = q.shape + _, _, n_kv_heads, page_size, _ = kv_cache.shape + head_ratio = n_heads // n_kv_heads + head_ratio_padded = max(1, 2 ** math.ceil(math.log2(head_ratio))) if head_ratio > 1 else 1 + page_block = _get_page_block(page_size) + + max_pages = kv_indices.shape[0] + max_seq_len = max_pages * page_size + # Normalize sliding_window: None/non-positive → 0 (full attention) + sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 + + output = out if out is not None else torch.empty_like(q) + + if batch_size == 0: + return output + + # Use effective sequence length (capped by sliding window) for split-K heuristic + effective_seq_len = min(max_seq_len, sw) if sw > 0 else max_seq_len + num_splits = _get_num_splits(effective_seq_len, batch_size, n_kv_heads, page_size) + + # Allocate intermediate buffers for split-K + partial_o = torch.empty( + batch_size, n_heads, - n_kv_heads, - q_d_head, - v_d_head, - SEQ_BLOCK_SIZE, - HEAD_BLOCK_SIZE, - sliding_window if sliding_window is not None else -1, + num_splits, + head_dim, + dtype=torch.float32, + device=q.device, ) - has_sinks = sinks is not None - - attention_kv_stage2[(num_decode, n_heads, 1)]( - stage1_output_values, - stage1_output_logsumexp, - out, - input_pos, - num_blocks, + partial_lse = torch.empty( + batch_size, n_heads, - v_d_head, - SEQ_BLOCK_SIZE, - has_sinks, - sinks, + num_splits, + dtype=torch.float32, + device=q.device, + ) + + # Stage 1: GQA-batched parallel KV processing + _flash_decode_stage1_kernel[(batch_size, n_kv_heads, num_splits)]( + q, + kv_cache, + kv_indices, + kv_indptr, + kv_last_page_len, + partial_o, + partial_lse, + # Q strides + q.stride(0), + q.stride(1), + # Partial output strides + partial_o.stride(0), + partial_o.stride(1), + partial_o.stride(2), + # Partial LSE strides + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + # Cache strides + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + kv_cache.stride(3), + # Constants + SM_SCALE=sm_scale, + N_HEADS=n_heads, + N_KV_HEADS=n_kv_heads, + HEAD_DIM=head_dim, + PAGE_SIZE=page_size, + PAGE_BLOCK=page_block, + HEAD_RATIO=head_ratio, + HEAD_RATIO_PADDED=head_ratio_padded, + NUM_SPLITS=num_splits, + SLIDING_WINDOW=sw, + ) + + # Stage 2: Combine partial results + _flash_decode_stage2_kernel[(batch_size, n_heads)]( + partial_o, + partial_lse, + output, + # Partial output strides + partial_o.stride(0), + partial_o.stride(1), + partial_o.stride(2), + # Partial LSE strides + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + # Output strides + output.stride(0), + output.stride(1), + # Constants + HEAD_DIM=head_dim, + NUM_SPLITS=num_splits, + ) + + return output + + +# ============================================================================= +# TRITON KERNELS - CONTEXT/PREFILL (page-aligned, causal skip, autotuned) +# ============================================================================= +@triton.autotune( + configs=[ + triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=2), + triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=4), + triton.Config({"Q_BLOCK": 64}, num_stages=4, num_warps=4), + triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=4), + triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=8), + triton.Config({"Q_BLOCK": 128}, num_stages=3, num_warps=8), + ], + key=["HEAD_DIM", "PAGE_SIZE", "PAGE_BLOCK"], +) +@triton.jit +def _paged_context_kernel( + # Inputs + q_ptr, + kv_cache_ptr, + # Metadata + qo_indptr_ptr, + kv_indptr_ptr, + kv_indices_ptr, + kv_last_page_len_ptr, + seq_len_with_cache_ptr, + # Output + o_ptr, + # Strides + q_stride_token: tl.constexpr, + q_stride_head: tl.constexpr, + o_stride_token: tl.constexpr, + o_stride_head: tl.constexpr, + cache_stride_block: tl.constexpr, + cache_stride_kv: tl.constexpr, + cache_stride_head: tl.constexpr, + cache_stride_token: tl.constexpr, + # Autotuned + Q_BLOCK: tl.constexpr, + # Constants + SM_SCALE: tl.constexpr, + N_HEADS: tl.constexpr, + N_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_BLOCK: tl.constexpr, + SLIDING_WINDOW: tl.constexpr = 0, +): + """Context/prefill attention with paged KV cache, causal skip, and page-aligned iteration. + + Grid: (num_seq, n_heads, num_q_blocks) + + Optimizations: + - Page-aligned iteration: 1 scalar page table load per page, no div/mod, + contiguous KV memory access within each page. + - Causal skip: pages entirely beyond the last Q position are skipped, + saving ~50% of KV loads on average for causal attention. + - Autotuned Q_BLOCK, num_stages, num_warps for best tile/pipeline config. + """ + batch_id = tl.program_id(axis=0) + head_id = tl.program_id(axis=1) + q_block_id = tl.program_id(axis=2) + + HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS + kv_head_id = head_id // HEAD_RATIO + + q_start = tl.load(qo_indptr_ptr + batch_id) + q_end = tl.load(qo_indptr_ptr + batch_id + 1) + q_len = q_end - q_start + + kv_page_start = tl.load(kv_indptr_ptr + batch_id) + kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) + num_kv_pages = kv_page_end - kv_page_start + total_kv_len = tl.load(seq_len_with_cache_ptr + batch_id) + + cache_len = total_kv_len - q_len + + q_block_start = q_block_id * Q_BLOCK + q_offsets = q_block_start + tl.arange(0, Q_BLOCK) + q_mask = q_offsets < q_len + + if tl.sum(q_mask.to(tl.int32)) == 0: + return + + dhead_offsets = tl.arange(0, HEAD_DIM) + q_load_offsets = ( + (q_start + q_offsets[:, None]) * q_stride_token + + head_id * q_stride_head + + dhead_offsets[None, :] + ) + q_load_mask = q_mask[:, None] + q = tl.load(q_ptr + q_load_offsets, mask=q_load_mask, other=0.0) + + acc = tl.zeros([Q_BLOCK, HEAD_DIM], dtype=tl.float32) + m_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - float("inf") + l_i = tl.zeros([Q_BLOCK], dtype=tl.float32) + + page_offsets = tl.arange(0, PAGE_BLOCK) + + # Two-phase page loop: + # Phase 1 (full pages): pages entirely before the first Q position need no causal mask. + # First Q position in KV coords = q_block_start + cache_len. + # A page ending at (page_idx+1)*PAGE_SIZE - 1 is fully attended if that's <= first Q pos. + # Phase 2 (boundary pages): remaining pages up to last Q position need causal masking. + first_q_kv_pos = q_block_start + cache_len + max_q_pos = q_block_start + Q_BLOCK - 1 + cache_len + + # Number of full pages (all tokens in these pages are attended by all Q tokens) + num_full_pages = first_q_kv_pos // PAGE_SIZE + + # Sliding window: compute the first page within the window for Phase 1 pruning. + # Each query at position q_pos attends to KV in [q_pos - W + 1, q_pos]. + # The most restrictive query is the first one (q_block_start), so: + # first_valid_pos = max(0, first_q_kv_pos - SLIDING_WINDOW + 1) + if SLIDING_WINDOW > 0: + first_valid_pos = tl.maximum(0, first_q_kv_pos - SLIDING_WINDOW + 1) + first_window_page = first_valid_pos // PAGE_SIZE + else: + first_window_page = 0 + + # Check if this is a full Q block (no q_mask needed) + is_full_q_block = (q_block_start + Q_BLOCK) <= q_len + + # Phase 1: Full pages — no causal mask, no validity mask + # Process one page at a time with a clean inner loop + kv_head_offset = kv_head_id * cache_stride_head + local_kv = page_offsets[:, None] * cache_stride_token + dhead_offsets[None, :] + full_page_mask = page_offsets < PAGE_SIZE + + for page_idx in range(first_window_page, num_full_pages): + physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) + + # Use int64 to avoid overflow when physical_page * stride > 2^31 + page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset + + # When sliding window is active, the first window page may partially + # overlap the window boundary, requiring per-token masking. + # Use masked loads (like Phase 2) instead of block_ptr loads. + if SLIDING_WINDOW > 0: + k = tl.load( + kv_cache_ptr + page_base + local_kv, + mask=full_page_mask[:, None], + other=0.0, + ).to(q.dtype) # cast from fp8 if kv cache is fp8 + v = tl.load( + kv_cache_ptr + page_base + local_kv + cache_stride_kv, + mask=full_page_mask[:, None], + other=0.0, + ).to(q.dtype) # cast from fp8 if kv cache is fp8 + + qk = tl.dot(q, tl.trans(k)) * SM_SCALE + + # Per-query sliding window mask: each query position q_pos + # can attend to KV in [q_pos - W + 1, q_pos]. + kv_positions = page_idx * PAGE_SIZE + page_offsets[None, :] + q_kv_pos = q_offsets[:, None] + cache_len + sw_mask = (q_kv_pos - kv_positions) < SLIDING_WINDOW + full_mask_p1 = q_mask[:, None] & sw_mask & full_page_mask[None, :] + qk = tl.where(full_mask_p1, qk, float("-inf")) + else: + k = tl.load( + kv_cache_ptr + page_base + local_kv, + mask=full_page_mask[:, None], + other=0.0, + ).to(q.dtype) # cast from fp8 if kv cache is fp8 + v = tl.load( + kv_cache_ptr + page_base + local_kv + cache_stride_kv, + mask=full_page_mask[:, None], + other=0.0, + ).to(q.dtype) # cast from fp8 if kv cache is fp8 + + qk = tl.dot(q, tl.trans(k)) * SM_SCALE + + qk = tl.where(full_page_mask[None, :], qk, float("-inf")) + if not is_full_q_block: + qk = tl.where(q_mask[:, None], qk, float("-inf")) + + m_ij = tl.max(qk, axis=1) + m_i_new = tl.maximum(m_i, m_ij) + if SLIDING_WINDOW > 0: + # Guard against NaN when m_i == m_i_new == -inf (no valid tokens seen + # yet for a query whose window doesn't overlap this page at all). + alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) + p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) + else: + alpha = tl.exp(m_i - m_i_new) + p = tl.exp(qk - m_i_new[:, None]) + acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_i_new + + # Phase 2: Boundary pages — need causal mask and validity mask + # Pre-compute q_positions outside loop (invariant across pages) + q_positions_2d = q_offsets[:, None] + cache_len + + for page_idx in range(num_full_pages, num_kv_pages): + kv_base_pos = page_idx * PAGE_SIZE + + # Causal skip: if entire page is beyond last Q position, skip it. + if kv_base_pos <= max_q_pos: + physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) + valid_tokens = tl.minimum(PAGE_SIZE, total_kv_len - kv_base_pos) + page_mask = page_offsets < valid_tokens + + # Use int64 to avoid overflow when physical_page * stride > 2^31 + page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset + page_mask_2d = page_mask[:, None] + k = tl.load(kv_cache_ptr + page_base + local_kv, mask=page_mask_2d, other=0.0).to( + q.dtype + ) # cast from fp8 if kv cache is fp8 + v = tl.load( + kv_cache_ptr + page_base + local_kv + cache_stride_kv, + mask=page_mask_2d, + other=0.0, + ).to(q.dtype) # cast from fp8 if kv cache is fp8 + + qk = tl.dot(q, tl.trans(k)) * SM_SCALE + kv_positions = kv_base_pos + page_offsets[None, :] + causal_mask = q_positions_2d >= kv_positions + if SLIDING_WINDOW > 0: + sliding_mask = (q_positions_2d - kv_positions) < SLIDING_WINDOW + full_mask = q_mask[:, None] & causal_mask & sliding_mask & page_mask[None, :] + else: + full_mask = q_mask[:, None] & causal_mask & page_mask[None, :] + qk = tl.where(full_mask, qk, float("-inf")) + + m_ij = tl.max(qk, axis=1) + m_i_new = tl.maximum(m_i, m_ij) + if SLIDING_WINDOW > 0: + alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) + p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) + else: + alpha = tl.exp(m_i - m_i_new) + p = tl.exp(qk - m_i_new[:, None]) + acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_i_new + + l_i = tl.where(l_i == 0.0, 1.0, l_i) + o = acc / l_i[:, None] + o_store_offsets = ( + (q_start + q_offsets[:, None]) * o_stride_token + + head_id * o_stride_head + + dhead_offsets[None, :] + ) + tl.store(o_ptr + o_store_offsets, o, mask=q_load_mask) + + +@triton.autotune( + configs=[ + triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=2), + triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=4), + triton.Config({"Q_BLOCK": 64}, num_stages=4, num_warps=4), + triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=4), + triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=8), + triton.Config({"Q_BLOCK": 128}, num_stages=3, num_warps=8), + ], + key=["HEAD_DIM", "PAGE_SIZE", "PAGE_BLOCK"], +) +@triton.jit +def _paged_context_masked_kernel( + # Inputs + q_ptr, + kv_cache_ptr, + custom_mask_ptr, + # Metadata + qo_indptr_ptr, + kv_indptr_ptr, + kv_indices_ptr, + seq_len_with_cache_ptr, + # Output + o_ptr, + # Strides + q_stride_token: tl.constexpr, + q_stride_head: tl.constexpr, + o_stride_token: tl.constexpr, + o_stride_head: tl.constexpr, + cache_stride_block: tl.constexpr, + cache_stride_kv: tl.constexpr, + cache_stride_head: tl.constexpr, + cache_stride_token: tl.constexpr, + mask_stride_batch: tl.constexpr, + mask_stride_head: tl.constexpr, + mask_stride_q: tl.constexpr, + mask_stride_k: tl.constexpr, + # Autotuned + Q_BLOCK: tl.constexpr, + # Constants + SM_SCALE: tl.constexpr, + N_HEADS: tl.constexpr, + N_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_BLOCK: tl.constexpr, + SLIDING_WINDOW: tl.constexpr = 0, +): + """Context/prefill attention with paged KV cache and a backend-provided bool allow-mask.""" + batch_id = tl.program_id(axis=0) + head_id = tl.program_id(axis=1) + q_block_id = tl.program_id(axis=2) + + HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS + kv_head_id = head_id // HEAD_RATIO + + q_start = tl.load(qo_indptr_ptr + batch_id) + q_end = tl.load(qo_indptr_ptr + batch_id + 1) + q_len = q_end - q_start + + kv_page_start = tl.load(kv_indptr_ptr + batch_id) + kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) + num_kv_pages = kv_page_end - kv_page_start + total_kv_len = tl.load(seq_len_with_cache_ptr + batch_id) + + q_block_start = q_block_id * Q_BLOCK + q_offsets = q_block_start + tl.arange(0, Q_BLOCK) + q_mask = q_offsets < q_len + if tl.sum(q_mask.to(tl.int32)) == 0: + return + + dhead_offsets = tl.arange(0, HEAD_DIM) + q_load_offsets = ( + (q_start + q_offsets[:, None]) * q_stride_token + + head_id * q_stride_head + + dhead_offsets[None, :] + ) + q_load_mask = q_mask[:, None] + q = tl.load(q_ptr + q_load_offsets, mask=q_load_mask, other=0.0) + + acc = tl.zeros([Q_BLOCK, HEAD_DIM], dtype=tl.float32) + m_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - float("inf") + l_i = tl.zeros([Q_BLOCK], dtype=tl.float32) + + page_offsets = tl.arange(0, PAGE_BLOCK) + kv_head_offset = kv_head_id * cache_stride_head + local_kv = page_offsets[:, None] * cache_stride_token + dhead_offsets[None, :] + mask_batch_offset = batch_id * mask_stride_batch + # The mask is broadcast over heads (head dim == 1), so the head offset is always 0. + mask_head_offset = 0 * mask_stride_head + + for page_idx in range(num_kv_pages): + kv_base_pos = page_idx * PAGE_SIZE + physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) + valid_tokens = tl.minimum(PAGE_SIZE, total_kv_len - kv_base_pos) + page_mask = page_offsets < valid_tokens + + page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset + page_mask_2d = page_mask[:, None] + k = tl.load(kv_cache_ptr + page_base + local_kv, mask=page_mask_2d, other=0.0) + v = tl.load( + kv_cache_ptr + page_base + local_kv + cache_stride_kv, + mask=page_mask_2d, + other=0.0, + ) + + qk = tl.dot(q, tl.trans(k)) * SM_SCALE + kv_positions = kv_base_pos + page_offsets[None, :] + mask_offsets = ( + mask_batch_offset + + mask_head_offset + + q_offsets[:, None] * mask_stride_q + + kv_positions * mask_stride_k + ) + valid_mask = q_mask[:, None] & page_mask[None, :] + custom_mask = tl.load(custom_mask_ptr + mask_offsets, mask=valid_mask, other=0) + if SLIDING_WINDOW > 0: + cache_len = total_kv_len - q_len + query_positions = q_offsets[:, None] + cache_len + sliding_mask = (query_positions >= kv_positions) & ( + (query_positions - kv_positions) < SLIDING_WINDOW + ) + full_mask = valid_mask & sliding_mask & (custom_mask != 0) + else: + full_mask = valid_mask & (custom_mask != 0) + qk = tl.where(full_mask, qk, float("-inf")) + + m_ij = tl.max(qk, axis=1) + m_i_new = tl.maximum(m_i, m_ij) + if SLIDING_WINDOW > 0: + alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) + p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) + else: + alpha = tl.exp(m_i - m_i_new) + p = tl.exp(qk - m_i_new[:, None]) + acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_i_new + + l_i = tl.where(l_i == 0.0, 1.0, l_i) + o = acc / l_i[:, None] + o_store_offsets = ( + (q_start + q_offsets[:, None]) * o_stride_token + + head_id * o_stride_head + + dhead_offsets[None, :] + ) + tl.store(o_ptr + o_store_offsets, o, mask=q_load_mask) + + +@triton.jit +def _fast_gather_sdpa_kernel( + kv_cache_ptr, + kv_indices_ptr, + kv_indptr_ptr, + seq_len_with_cache_ptr, + out_k_ptr, + out_v_ptr, + # Strides + cache_stride_block: tl.constexpr, + cache_stride_kv: tl.constexpr, + cache_stride_head: tl.constexpr, + cache_stride_token: tl.constexpr, + out_stride_seq: tl.constexpr, + out_stride_head: tl.constexpr, + out_stride_token: tl.constexpr, + # Constants + MAX_PAGES: tl.constexpr, + N_KV_HEADS: tl.constexpr, + PAGE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, +): + """Gather scattered pages into separate K, V buffers in SDPA layout. + + Grid: (total_pages, N_KV_HEADS) + Each program copies one page for one KV head into contiguous K and V + outputs shaped [num_seq, n_kv_heads, max_kv_len, head_dim]. + No precomputed mapping needed — seq_id and local_page computed from global index. + """ + page_global_idx = tl.program_id(0) + kv_head_id = tl.program_id(1) + + # Compute seq_id and local_page from global page index + seq_id = page_global_idx // MAX_PAGES + local_page = page_global_idx % MAX_PAGES + + seq_page_start = tl.load(kv_indptr_ptr + seq_id) + seq_page_end = tl.load(kv_indptr_ptr + seq_id + 1) + seq_pages = seq_page_end - seq_page_start + page_is_valid = local_page < seq_pages + + physical_page = tl.load( + kv_indices_ptr + seq_page_start + local_page, mask=page_is_valid, other=0 ) + token_offsets = tl.arange(0, PAGE_SIZE) + head_offsets = tl.arange(0, HEAD_DIM) + + # Source: kv_cache[physical_page, 0/1, kv_head_id, :, :] + src_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_id * cache_stride_head + src_offsets = token_offsets[:, None] * cache_stride_token + head_offsets[None, :] + + # Destination: out_k/v[seq_id, kv_head_id, local_page*PAGE_SIZE + :, :] + local_token_start = local_page * PAGE_SIZE + seq_len = tl.load(seq_len_with_cache_ptr + seq_id) + valid_tokens = tl.minimum(PAGE_SIZE, seq_len - local_token_start) + valid_tokens = tl.maximum(0, valid_tokens) + token_is_valid = token_offsets < valid_tokens + load_mask = page_is_valid & token_is_valid[:, None] + + k_data = tl.load(kv_cache_ptr + src_base + src_offsets, mask=load_mask, other=0.0) + v_data = tl.load( + kv_cache_ptr + src_base + cache_stride_kv + src_offsets, mask=load_mask, other=0.0 + ) -def _prefill_attention( - q: torch.Tensor, # [num_prefill_tokens, num_heads, qk_head_dim] - k: torch.Tensor, # [num_prefill_tokens, num_kv_heads, qk_head_dim] - v: torch.Tensor, # [num_prefill_tokens, num_kv_heads, v_head_dim] - k_cache: torch.Tensor, - v_cache: torch.Tensor, - input_pos: torch.Tensor, # [num_prefill] - slot_idx: torch.Tensor, # [num_prefill] - seq_len: torch.Tensor, # [num_prefill] - seq_start: torch.Tensor, # [num_prefill] - scale: float, - out: torch.Tensor, # [num_prefill_tokens, num_heads, v_head_dim] - sinks: Optional[torch.Tensor] = None, + dst_base = ( + seq_id * out_stride_seq + + kv_head_id * out_stride_head + + (local_token_start + token_offsets[:, None]) * out_stride_token + + head_offsets[None, :] + ) + + tl.store(out_k_ptr + dst_base, k_data) + tl.store(out_v_ptr + dst_base, v_data) + + +def triton_context( + q: torch.Tensor, + kv_cache: torch.Tensor, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_last_page_len: torch.Tensor, + seq_len_with_cache: torch.Tensor, + sm_scale: float, sliding_window: Optional[int] = None, -) -> None: - """Handle prefill phase - context attention with variable sequence lengths.""" - # NOTE: num_prefill_tokens == sum(seq_len) - num_prefill_tokens, n_heads, q_d_head = q.shape - max_cache_seq_len, n_kv_heads = k_cache.shape[1:3] - v_d_head = v.shape[-1] - num_prefill = len(input_pos) - SEQ_BLOCK = 32 - - update_kv_cache[(num_prefill, n_kv_heads, (max(seq_len) + SEQ_BLOCK - 1) // SEQ_BLOCK)]( - k, - v, - seq_len, - seq_start, - k_cache, - v_cache, - input_pos, - slot_idx, - max_cache_seq_len, - n_kv_heads, - q_d_head, - v_d_head, - 32, - GENERATE_ONLY=False, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Context/prefill attention with paged KV cache.""" + total_tokens, n_heads, head_dim = q.shape + _, _, n_kv_heads, page_size, _ = kv_cache.shape + num_seq = qo_indptr.shape[0] - 1 + page_block = _get_page_block(page_size) + + output = out if out is not None else torch.empty_like(q) + + if num_seq == 0 or total_tokens == 0: + return output + + q_lens = qo_indptr[1:] - qo_indptr[:-1] + + # Compute max_q_len without GPU sync for single-sequence batches (most common + # in serving). For multi-sequence batches, we must use .item() because + # total_tokens // num_seq gives the average, not the max — variable-length + # sequences can produce wrong results (under-launched Q blocks or wrong SDPA reshape). + if num_seq == 1: + max_q_len = total_tokens + else: + max_q_len = int(q_lens.max().item()) + + # Adaptive dispatch: gather + cuDNN SDPA for seq>=512 (outperforms paged kernel), + # paged Triton kernel for shorter sequences where gather overhead dominates. + # Normalize sliding_window for kernel constexpr: None/non-positive → 0 + sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 + + # Force SDPA for large head_dim: the Triton paged kernel's tl.dot produces + # misaligned shared memory accesses on Blackwell when HEAD_DIM > 256. + large_head_dim = head_dim > 256 + # SDPA reshape requires all sequences to have the same q_len (since q is + # packed as [total_tokens, ...] and we reshape to [num_seq, max_q_len, ...]). + # Check without GPU sync: sum(q_len_i) == num_seq * max_q_len iff all equal. + all_same_q_len = total_tokens == num_seq * max_q_len + use_sdpa_candidate = ( + (max_q_len >= 512 or large_head_dim) + and (num_seq <= 64 or large_head_dim) + and all_same_q_len + and sw == 0 # SDPA doesn't support sliding window natively + ) + kv_lens = seq_len_with_cache[:num_seq] + cache_lens = kv_lens - q_lens + + max_kv_len = int(kv_lens.max().item()) if use_sdpa_candidate else 0 + max_pages = (max_kv_len + page_size - 1) // page_size + total_expected_pages = num_seq * max_pages + use_sdpa = use_sdpa_candidate and max_pages > 0 + + if use_sdpa: + # Fast Triton gather: scattered pages → separate K, V in SDPA layout + # Single alloc for both K and V, single kernel to fill + padded_kv_len = max_pages * page_size + kv_buf = torch.empty( + 2, + num_seq, + n_kv_heads, + padded_kv_len, + head_dim, + dtype=kv_cache.dtype, + device=kv_cache.device, + ) + k_buf = kv_buf[0] + v_buf = kv_buf[1] + _fast_gather_sdpa_kernel[(total_expected_pages, n_kv_heads)]( + kv_cache, + kv_indices, + kv_indptr, + seq_len_with_cache, + k_buf, + v_buf, + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + kv_cache.stride(3), + k_buf.stride(0), + k_buf.stride(1), + k_buf.stride(2), + MAX_PAGES=max_pages, + N_KV_HEADS=n_kv_heads, + PAGE_SIZE=page_size, + HEAD_DIM=head_dim, + ) + k_sdpa = k_buf[:, :, :max_kv_len, :] + v_sdpa = v_buf[:, :, :max_kv_len, :] + + # Cast k/v to query dtype if kv cache uses a different dtype (e.g., fp8) + if kv_cache.dtype != q.dtype: + k_sdpa = k_sdpa.to(q.dtype) + v_sdpa = v_sdpa.to(q.dtype) + + q_positions = torch.arange(max_q_len, device=q.device, dtype=cache_lens.dtype) + kv_positions = torch.arange(max_kv_len, device=q.device, dtype=kv_lens.dtype) + attn_mask = kv_positions.view(1, 1, 1, max_kv_len) < kv_lens.view(num_seq, 1, 1, 1) + causal_mask = kv_positions.view(1, 1, 1, max_kv_len) <= ( + cache_lens.view(num_seq, 1, 1, 1) + q_positions.view(1, 1, max_q_len, 1) + ) + attn_mask = attn_mask & causal_mask + + # SDPA with GQA + o_sdpa = torch.nn.functional.scaled_dot_product_attention( + q.view(num_seq, max_q_len, n_heads, head_dim).transpose(1, 2), + k_sdpa, + v_sdpa, + scale=sm_scale, + attn_mask=attn_mask, + is_causal=False, + enable_gqa=True, + ) + output.view(num_seq, max_q_len, n_heads, head_dim).copy_(o_sdpa.permute(0, 2, 1, 3)) + else: + # Use paged kernel (better for small workloads) + def grid_paged(meta): + q_block = meta["Q_BLOCK"] + num_q_blocks = (max_q_len + q_block - 1) // q_block + return (num_seq, n_heads, num_q_blocks) + + _paged_context_kernel[grid_paged]( + q, + kv_cache, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + seq_len_with_cache, + output, + q.stride(0), + q.stride(1), + output.stride(0), + output.stride(1), + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + kv_cache.stride(3), + SM_SCALE=sm_scale, + N_HEADS=n_heads, + N_KV_HEADS=n_kv_heads, + HEAD_DIM=head_dim, + PAGE_SIZE=page_size, + PAGE_BLOCK=page_block, + SLIDING_WINDOW=sw, + ) + + return output + + +def triton_context_with_custom_mask( + q: torch.Tensor, + kv_cache: torch.Tensor, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + seq_len_with_cache: torch.Tensor, + custom_attn_mask: torch.Tensor, + sm_scale: float, + sliding_window: Optional[int] = None, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Context/prefill attention with paged KV cache and a backend-provided bool allow-mask. + + The mask must be broadcastable over heads, i.e. shape ``[B, 1, S_q, S_k]``. + Per-head masks (``N > 1`` in the head dimension) are **not** supported. + """ + output = out if out is not None else torch.empty_like(q) + num_seq = qo_indptr.shape[0] - 1 + total_tokens, n_heads, head_dim = q.shape + _, _, n_kv_heads, page_size, _ = kv_cache.shape + page_block = _get_page_block(page_size) + + if num_seq == 0 or total_tokens == 0: + return output + + assert custom_attn_mask.shape[1] == 1, ( + f"Per-head masks are not supported; expected mask head dim == 1, " + f"got {custom_attn_mask.shape[1]}. Mask shape: {custom_attn_mask.shape}" ) - grid = (num_prefill, n_heads, (max(seq_len) + SEQ_BLOCK - 1) // SEQ_BLOCK) - has_sinks = sinks is not None + if num_seq == 1: + max_q_len = total_tokens + else: + q_lens = qo_indptr[1:] - qo_indptr[:-1] + max_q_len = int(q_lens.max().item()) - context_attention_kv_flattened[grid]( + if not custom_attn_mask.is_contiguous() or custom_attn_mask.dtype != torch.uint8: + custom_attn_mask = custom_attn_mask.contiguous().to(dtype=torch.uint8) + + sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 + + def grid_masked(meta): + q_block = meta["Q_BLOCK"] + num_q_blocks = (max_q_len + q_block - 1) // q_block + return (num_seq, n_heads, num_q_blocks) + + _paged_context_masked_kernel[grid_masked]( q, - seq_len, - seq_start, - k_cache, - v_cache, - input_pos, - slot_idx, - out, - scale, - n_heads, - n_kv_heads, - q_d_head, - v_d_head, - SEQ_BLOCK, - max_cache_seq_len, - sliding_window if sliding_window is not None else -1, - has_sinks, - sinks, + kv_cache, + custom_attn_mask, + qo_indptr, + kv_indptr, + kv_indices, + seq_len_with_cache, + output, + q.stride(0), + q.stride(1), + output.stride(0), + output.stride(1), + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + kv_cache.stride(3), + custom_attn_mask.stride(0), + custom_attn_mask.stride(1), + custom_attn_mask.stride(2), + custom_attn_mask.stride(3), + SM_SCALE=sm_scale, + N_HEADS=n_heads, + N_KV_HEADS=n_kv_heads, + HEAD_DIM=head_dim, + PAGE_SIZE=page_size, + PAGE_BLOCK=page_block, + SLIDING_WINDOW=sw, ) + return output -@torch.library.custom_op( - "auto_deploy::triton_attention_flattened_mha_with_cache", - mutates_args=("k_cache", "v_cache"), -) -def flattened_mha_with_cache( + +@torch.library.custom_op("auto_deploy::triton_prepare_metadata", mutates_args=()) +def prepare_triton_metadata( + position_ids: torch.Tensor, + batch_info_host: torch.Tensor, + cu_seqlen: torch.Tensor, + seq_len_with_cache: torch.Tensor, +) -> List[torch.Tensor]: + """Prepare metadata for Triton attention.""" + from ..attention_interface import BatchInfo + + batch_info = BatchInfo(batch_info_host) + num_prefill, num_prefill_tokens, num_decode = batch_info.get_absorbed_info() + num_seq = num_prefill + num_decode + num_tokens = num_prefill_tokens + num_decode + + qo_indptr = cu_seqlen[: num_seq + 1] + + batch_indices, positions = flashinfer.get_batch_indices_positions( + qo_indptr, seq_len_with_cache[:num_seq], num_tokens + ) + + return batch_indices, positions + + +@prepare_triton_metadata.register_fake +def prepare_triton_metadata_fake( + position_ids: torch.Tensor, + batch_info_host: torch.Tensor, + cu_seqlen: torch.Tensor, + seq_len_with_cache: torch.Tensor, +): + num_tokens = position_ids.shape[0] * position_ids.shape[1] + return ( + torch.empty(num_tokens, dtype=torch.int32, device=position_ids.device), + torch.empty(num_tokens, dtype=torch.int32, device=position_ids.device), + ) + + +@torch.library.custom_op("auto_deploy::triton_mha_with_cache", mutates_args=("kv_cache",)) +def triton_mha_with_cache( # Q, K, V q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, # STANDARD METADATA batch_info_host: torch.Tensor, - seq_len: torch.Tensor, - input_pos: torch.Tensor, - slot_idx: torch.Tensor, - cu_seqlen: torch.Tensor, + cu_seqlen_host: torch.Tensor, + cu_num_pages: torch.Tensor, + cu_num_pages_host: torch.Tensor, + cache_loc: torch.Tensor, + last_page_len: torch.Tensor, + last_page_len_host: torch.Tensor, + seq_len_with_cache_host: torch.Tensor, # EXTRA METADATA - # - # CACHES - k_cache: torch.Tensor, - v_cache: torch.Tensor, - # BUFFERS - # + triton_batch_indices: torch.Tensor, + triton_positions: torch.Tensor, + # CACHES - combined KV cache + kv_cache: torch.Tensor, # CONSTANTS - scale: Optional[float], - sinks: Optional[torch.Tensor] = None, + scale: Optional[float] = None, sliding_window: Optional[int] = None, + read_cache_only: bool = False, + # OPTIONAL INPUTS + custom_attn_mask: Optional[torch.Tensor] = None, + # OPTIONAL PRE-ALLOCATED OUTPUT out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Flattened MHA with cache that takes q, k, v in BSND layout. + """Triton attention with mixed batch support.""" + head_dim = kv_cache.shape[-1] + q_shape_og = q.shape + b, s = q_shape_og[:2] + + q = q.reshape(b * s, -1, head_dim).contiguous() + k = k.reshape(b * s, -1, head_dim).contiguous() + v = v.reshape(b * s, -1, head_dim).contiguous() + + from ..attention_interface import BatchInfo - NOTE: this op can also handle seq_len==0, which might be useful for CUDAGRAPH. - """ - # Extract batch info from batch_info_host batch_info = BatchInfo(batch_info_host) num_prefill, num_prefill_tokens, num_decode = batch_info.get_absorbed_info() num_seq = num_prefill + num_decode num_total_tokens = num_prefill_tokens + num_decode - # Get cache and head dimensions - num_kv_heads, qk_head_dim = k_cache.shape[-2:] - v_head_dim = v_cache.shape[-1] - b, s = q.shape[:2] + sm_scale = _get_sm_scale(head_dim, scale) - # Determine num_heads from input shape - num_heads = q.shape[2] // qk_head_dim if q.ndim == 3 else q.shape[2] - - # Define output shape (preserve original input format) - output_shape = (b, s, num_heads * v_head_dim) if q.ndim == 3 else (b, s, num_heads, v_head_dim) - - # Flatten Q, K, V to [total_tokens, heads, head_dim] - bs = b * s - q_flat = q.contiguous().view(bs, num_heads, qk_head_dim) - k_flat = k.contiguous().view(bs, num_kv_heads, qk_head_dim) - v_flat = v.contiguous().view(bs, num_kv_heads, v_head_dim) - - # Compute scale if not provided - scale = 1.0 / math.sqrt(qk_head_dim) if scale is None else scale + if not read_cache_only: + update_paged_kv_cache( + k[:num_total_tokens], + v[:num_total_tokens], + triton_batch_indices[:num_total_tokens], + triton_positions[:num_total_tokens], + kv_cache, + cache_loc, + cu_num_pages[: num_seq + 1], + ) - # Preallocate output tensor - y = q_flat.new_empty(bs, num_heads, v_head_dim) + if out is not None: + y = out.view(-1, q.shape[1], head_dim) + else: + y = torch.empty_like(q) - # PREFILL: process context tokens with variable sequence lengths + # Process prefill tokens if any if num_prefill > 0: - _prefill_attention( - q_flat[:num_prefill_tokens], - k_flat[:num_prefill_tokens], - v_flat[:num_prefill_tokens], - k_cache, - v_cache, - input_pos[:num_prefill], - slot_idx[:num_prefill], - seq_len[:num_prefill], - cu_seqlen[:num_prefill], - scale, - y[:num_prefill_tokens], - sinks, - sliding_window, - ) + cu_seqlen = cu_seqlen_host[: num_prefill + 1].to(q.device, non_blocking=True) + seq_len_with_cache = seq_len_with_cache_host[:num_prefill].to(q.device, non_blocking=True) + has_custom_attn_mask = custom_attn_mask is not None and custom_attn_mask.numel() > 0 + if not has_custom_attn_mask: + triton_context( + q[:num_prefill_tokens], + kv_cache, + cu_seqlen, + cu_num_pages[: num_prefill + 1], + cache_loc, + last_page_len[:num_prefill], + seq_len_with_cache, + sm_scale, + sliding_window=sliding_window, + out=y[:num_prefill_tokens], + ) + else: + triton_context_with_custom_mask( + q[:num_prefill_tokens], + kv_cache, + cu_seqlen, + cu_num_pages[: num_prefill + 1], + cache_loc, + seq_len_with_cache, + custom_attn_mask[:num_prefill], + sm_scale, + sliding_window=sliding_window, + out=y[:num_prefill_tokens], + ) - # DECODE: process single-token generation + # Process decode tokens if any if num_decode > 0: - _decode_attention( - q_flat[num_prefill_tokens:num_total_tokens], - k_flat[num_prefill_tokens:num_total_tokens], - v_flat[num_prefill_tokens:num_total_tokens], - k_cache, - v_cache, - slot_idx[num_prefill:num_seq], - input_pos[num_prefill:num_seq], - scale, - y[num_prefill_tokens:num_total_tokens], - sinks, - sliding_window, + triton_decode( + q[num_prefill_tokens:num_total_tokens], + kv_cache, + cache_loc, + cu_num_pages[num_prefill : num_seq + 1], + last_page_len[num_prefill:num_seq], + sm_scale, + sliding_window=sliding_window, + out=y[num_prefill_tokens:num_total_tokens], ) if out is not None: - out_flat = out.view(bs, num_heads, v_head_dim) - out_flat[:num_total_tokens].copy_(y[:num_total_tokens]) + # Zero stale data in padding region for CUDA graph replay stability + bs = b * s if num_total_tokens < bs: - out_flat[num_total_tokens:].zero_() + y[num_total_tokens:].zero_() + # Return a 0-element dummy to satisfy PyTorch's no-alias constraint. + # The caller (DynamicOpWrapper._coalesce_output) picks ``out`` over + # this dummy, so the pre-allocated buffer is used downstream. return out.new_empty(0) - # Zero padding positions so downstream ops don't see garbage (piecewise CG) - if num_total_tokens < bs: - y[num_total_tokens:].zero_() + return y.view(q_shape_og) - return y.view(*output_shape) - -@flattened_mha_with_cache.register_fake -def flattened_mha_fake( - # Q, K, V +@triton_mha_with_cache.register_fake +def triton_mha_with_cache_fake( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - # STANDARD METADATA batch_info_host: torch.Tensor, - seq_len: torch.Tensor, - input_pos: torch.Tensor, - slot_idx: torch.Tensor, - cu_seqlen: torch.Tensor, - # EXTRA METADATA - # - # CACHES - k_cache: torch.Tensor, - v_cache: torch.Tensor, - # BUFFERS - # - # CONSTANTS - scale: Optional[float], - sinks: Optional[torch.Tensor] = None, + cu_seqlen_host: torch.Tensor, + cu_num_pages: torch.Tensor, + cu_num_pages_host: torch.Tensor, + cache_loc: torch.Tensor, + last_page_len: torch.Tensor, + last_page_len_host: torch.Tensor, + seq_len_with_cache_host: torch.Tensor, + triton_batch_indices: torch.Tensor, + triton_positions: torch.Tensor, + kv_cache: torch.Tensor, + scale: Optional[float] = None, sliding_window: Optional[int] = None, + read_cache_only: bool = False, + custom_attn_mask: Optional[torch.Tensor] = None, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: if out is not None: return out.new_empty(0) - return q.new_empty(*q.shape[:-1], v.shape[-1]).contiguous() + return torch.empty_like(q.contiguous()) @AttentionRegistry.register("triton") class TritonAttention(AttentionDescriptor): + """Descriptor for Triton Attention backend. + + Optimized with GQA head batching, FlashDecoding, and page-aligned iteration. + """ + @classmethod def get_attention_layout(cls) -> AttentionLayout: - """Get the attention layout expected by the source op and the cached attention op.""" return "bsnd" @classmethod def get_num_qkv_args(cls) -> int: - """Get the number of qkv arguments expected by the source op.""" return 3 @classmethod @@ -355,73 +1512,81 @@ def get_source_attention_op(cls) -> OpOverloadPacket: @classmethod def get_cached_attention_op(cls) -> MHACallable: - return torch.ops.auto_deploy.triton_attention_flattened_mha_with_cache.default + return torch.ops.auto_deploy.triton_mha_with_cache.default + + @classmethod + def supports_shared_kv(cls) -> bool: + return True @classmethod def get_standard_metadata_args(cls) -> List[str]: - return ["batch_info_host", "seq_len", "input_pos", "slot_idx", "cu_seqlen"] + return [ + "batch_info_host", + "cu_seqlen_host", + "cu_num_pages", + "cu_num_pages_host", + "cache_loc", + "last_page_len", + "last_page_len_host", + "seq_len_with_cache_host", + ] + + @classmethod + def get_prepare_extra_metadata_info( + cls, any_source_attn_node: Node + ) -> Tuple[Optional[PrepareMetadataCallable], int, List[Constant]]: + return ( + torch.ops.auto_deploy.triton_prepare_metadata.default, + 2, + [], + ) @classmethod def get_cache_initializers( cls, source_attn_node: Node, cache_config: KvCacheConfig ) -> ResourceHandlerDict: - # source op is [bsnd] layout already k_fake: FakeTensor = source_attn_node.args[1].meta["val"] - v_fake: FakeTensor = source_attn_node.args[2].meta["val"] num_kv_heads = k_fake.shape[2] - k_head_dim = k_fake.shape[3] - v_head_dim = v_fake.shape[3] + head_dim = k_fake.shape[3] + (sw,) = extract_op_args(source_attn_node, "sliding_window") + sliding_window = sw if isinstance(sw, int) and sw > 0 else 0 return { - "k_cache": UnpagedResourceHandler( + "kv_cache": KVPagedResourceHandler( num_kv_heads, - k_head_dim, + head_dim, dtype=cls.resolve_cache_dtype(cache_config.dtype, k_fake.dtype), - ), - "v_cache": UnpagedResourceHandler( - num_kv_heads, - v_head_dim, - dtype=cls.resolve_cache_dtype(cache_config.dtype, v_fake.dtype), - ), + kv_factor=2, + kv_layout=KV_LAYOUT, + sliding_window=sliding_window, + ) } @classmethod def get_constants(cls, source_attn_node: Node) -> List[Constant]: - # Sanity check: layout == "bsnd" - # extract_op_args handles kwargs and positional arguments consistently. - layout = extract_op_args(source_attn_node, "layout")[0] + layout, scale, _attn_mask, dropout_p, _is_causal = extract_op_args( + source_attn_node, "layout", "scale", "attn_mask", "dropout_p", "is_causal" + ) + if layout != "bsnd": raise RuntimeError( f"Expected torch_attention layout='bsnd' but got {layout!r} " f"for node: {source_attn_node.format_node()}" ) - # retrieve head_dim from k_fake - attn_mask, dropout_p, is_causal = extract_op_args( - source_attn_node, "attn_mask", "dropout_p", "is_causal" - ) - if attn_mask is not None or dropout_p != 0.0 or not is_causal: + if dropout_p != 0.0: ad_logger.debug( - "Unsupported attention arguments for " - f"{source_attn_node=}: {attn_mask=}, {dropout_p=}, {is_causal=}" + f"Unsupported attention arguments for {source_attn_node=}: {dropout_p=}" ) - # Get scale from args or kwargs - if len(source_attn_node.args) > 6: - scale = source_attn_node.args[6] - else: - scale = source_attn_node.kwargs.get("scale", None) - - # do a sanity check on the scale if it is not None, we only support the default scale - # of 1/sqrt(head_dim) and so we should do an approximate check for that one if not (isinstance(scale, float) or scale is None): - ad_logger.warning(f"Provided {scale=} is not a float. Using default scale instead.") + ad_logger.warning(f"Provided {scale=}, is not a float. Using default scale instead.") scale = None - # Get sinks and sliding_window from args or kwargs - sinks = extract_op_args(source_attn_node, "sinks")[0] + sliding_window = extract_op_args(source_attn_node, "sliding_window")[0] + return [ - scale, # softmax scale - sinks, + scale, sliding_window, + cls.get_shared_kv_source_layer_idx(source_attn_node) is not None, ] diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention_with_kv_cache.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention_with_kv_cache.py deleted file mode 100644 index 15372e7bab4a..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_attention_with_kv_cache.py +++ /dev/null @@ -1,882 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Multi-head attention kernel that can operate with kv-caches.""" - -import triton -from triton import language as tl - - -@triton.jit -def update_kv_cache( - k_ptr, # [B*S, N, D] - v_ptr, # [B*S, N, D] - seq_len_ptr, # [b] # length of each sequence in a batch - seq_start_indices_ptr, # [b] # start indices of a sequence in flattened q/k/v. - k_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - v_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - input_pos_ptr, # Specifies the sequence index in the caches at which to write the provided kv - slot_idx_ptr, # Specifies the slot index for each of the input sequences - MAX_SEQ_LENGTH: tl.constexpr, - N_KV_HEADS: tl.constexpr, - Q_D_HEAD: tl.constexpr, - V_D_HEAD: tl.constexpr, - SEQ_BLOCK: tl.constexpr, - GENERATE_ONLY: tl.constexpr, -): - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - - # Each program is responsible for a block of tokens in a single batch. - if GENERATE_ONLY: - seq_start_index = batch_id - seq_len: tl.constexpr = 1 - else: - seq_start_index = tl.load(seq_start_indices_ptr + batch_id) - seq_len = tl.load(seq_len_ptr + batch_id) - - # cache is [bsnd] - # slot_idx_ptr stores the slot index for the sequences provided to the kernel. - slot_idx = tl.load(slot_idx_ptr + batch_id) - - kv_position = tl.load(input_pos_ptr + batch_id) - - K_D_HEAD: tl.constexpr = Q_D_HEAD - k_cache_batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LENGTH * K_D_HEAD - v_cache_batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LENGTH * V_D_HEAD - - k_dhead_offsets = tl.arange(0, triton.next_power_of_2(K_D_HEAD)) - k_dhead_mask = k_dhead_offsets < K_D_HEAD - - v_dhead_offsets = tl.arange(0, triton.next_power_of_2(V_D_HEAD)) - v_dhead_mask = v_dhead_offsets < V_D_HEAD - - seq_offsets = seq_block_id * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - seq_mask = seq_offsets < seq_len - - k_load_mask = seq_mask[:, None] * k_dhead_mask[None, :] - v_load_mask = seq_mask[:, None] * v_dhead_mask[None, :] - - k_batch_offset = seq_start_index * N_KV_HEADS * K_D_HEAD - v_batch_offset = seq_start_index * N_KV_HEADS * V_D_HEAD - # Write back to kv-caches - ks = tl.load( - k_ptr - + k_batch_offset - + seq_offsets[:, None] * N_KV_HEADS * K_D_HEAD - + head_id * K_D_HEAD - + k_dhead_offsets[None, :], - mask=k_load_mask, - ) - vs = tl.load( - v_ptr - + v_batch_offset - + seq_offsets[:, None] * N_KV_HEADS * V_D_HEAD - + head_id * V_D_HEAD - + v_dhead_offsets[None, :], - mask=v_load_mask, - ) - - kv_writeback_seq_offsets = seq_offsets + kv_position - - k_cache_offset = ( - k_cache_batch_offset - + kv_writeback_seq_offsets[:, None] * K_D_HEAD * N_KV_HEADS - + head_id * K_D_HEAD - + k_dhead_offsets[None, :] - ) - - v_cache_offset = ( - v_cache_batch_offset - + kv_writeback_seq_offsets[:, None] * V_D_HEAD * N_KV_HEADS - + head_id * V_D_HEAD - + v_dhead_offsets[None, :] - ) - tl.store(k_cache_ptr + k_cache_offset, ks, k_load_mask) - tl.store(v_cache_ptr + v_cache_offset, vs, v_load_mask) - - -@triton.jit -def gqa_attention_kv_stage1( - q_ptr, # [Batch, 1, N_HEADS, D_HEAD] - k_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - v_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - slot_idx_ptr, # [Batch] # Specifies the slot index for each of the generate tokens. - input_pos_ptr, # [Batch] - output_values_ptr, # [Batch, N_HEADS, num_blocks, D_HEAD] - output_logsumexp_ptr, # [Batch, N_HEADS, num_blocks] - num_blocks, - SCALE: tl.constexpr, - MAX_SEQ_LEN: tl.constexpr, # Maximum supported sequence length - N_HEADS: tl.constexpr, # Number of heads - N_KV_HEADS: tl.constexpr, # Number of KV heads. - Q_D_HEAD: tl.constexpr, # Dimension of each query head. - V_D_HEAD: tl.constexpr, # Dimension of each key/value head - SEQ_BLOCK_SIZE: tl.constexpr, # Block size used for tiling the sequence dim. - HEAD_BLOCK_SIZE: tl.constexpr, # pad to 16 if HEAD_RATIO is < 16 to invoke tensor cores. - SLIDING_WINDOW: tl.constexpr, -): - """Attention kernel to be used for generate-only batches. - - Specialized for GQA. - - Assumes that kv caches have been updated. - - Supports non-power-of-2 D_HEAD - - Uses flash decoding. - KV-cache layout is assumed to be [Batch, Seq, Head, Dim] - 1. Fetch the K-cache from 0 to input_pos - 2. Fetch the V-cache from 0 to input_pos - 3. A = Q*K^T [1,D_HEAD] * [1,seq_len,D_HEAD] -> [1, seq_len] - 4. S = softmax(A) - 5. O = S*V [1, seq_len] * [1, seq_len, D_HEAD] -> [1, D_HEAD] - """ - # Assume KV-cache layout: [Batch, Seq, Head, Dim] - # A program is responsible for 1 batch, 1 head and a block of sequences. - batch_id = tl.program_id(axis=0) - kv_head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - - kv_position = tl.load(input_pos_ptr + batch_id) - slot_idx = tl.load(slot_idx_ptr + batch_id) - K_D_HEAD: tl.constexpr = Q_D_HEAD - batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LEN - - # Offsets for the block of sequences this program processes. - seq_start_pos = seq_block_id * SEQ_BLOCK_SIZE - - # The number of Q heads that map to each KV head. - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS # This needs to be a power-of-2 - - # Apply sliding window constraints - if SLIDING_WINDOW > 0: - # For sliding window, limit the sequence range - sliding_start = tl.maximum(0, kv_position - SLIDING_WINDOW + 1) - if seq_start_pos + SEQ_BLOCK_SIZE <= sliding_start or seq_start_pos > kv_position: - return - seq_offsets = seq_start_pos + tl.arange(0, SEQ_BLOCK_SIZE) - seq_mask = (seq_offsets <= kv_position) & (seq_offsets >= sliding_start) - else: - if seq_start_pos > kv_position: - return - seq_offsets = seq_start_pos + tl.arange(0, SEQ_BLOCK_SIZE) - seq_mask = seq_offsets <= kv_position - - # Need to pad the head dim to 16 if HEAD_RATIO is < 16 so that tensor cores can be invoked - # - head_offsets = kv_head_id * HEAD_RATIO + tl.arange(0, HEAD_BLOCK_SIZE) - head_mask = head_offsets < (kv_head_id * HEAD_RATIO + HEAD_RATIO) - # Assuming D_HEAD is a power of 2 - q_dhead_offsets = tl.arange(0, triton.next_power_of_2(Q_D_HEAD)) - q_dhead_mask = q_dhead_offsets < Q_D_HEAD - - v_dhead_offsets = tl.arange(0, triton.next_power_of_2(V_D_HEAD)) - v_dhead_mask = v_dhead_offsets < V_D_HEAD - - # Program loads the entire Q for the head assigned to it. - # [NUM_HEADS, Q_D_HEAD] - q_batch_offset = batch_id * N_HEADS * Q_D_HEAD - q_head_offsets = head_offsets * Q_D_HEAD - - # Q layout : BSND - q = tl.load( - q_ptr + q_batch_offset + q_head_offsets[:, None] + q_dhead_offsets[None, :], - mask=head_mask[:, None] * q_dhead_mask[None, :], - other=0.0, - ) - - # [BSND] - k_block_offsets = ( - batch_offset * K_D_HEAD - + seq_offsets[:, None] * K_D_HEAD * N_KV_HEADS - + kv_head_id * K_D_HEAD - + q_dhead_offsets[None, :] - ) - k_mask = seq_mask[:, None] * q_dhead_mask[None, :] # K and Q share the same head dim - k = tl.load(k_cache_ptr + k_block_offsets, mask=k_mask, other=0.0) - - v_block_offsets = ( - batch_offset * V_D_HEAD - + seq_offsets[:, None] * V_D_HEAD * N_KV_HEADS - + kv_head_id * V_D_HEAD - + v_dhead_offsets[None, :] - ) - v_mask = seq_mask[:, None] * v_dhead_mask[None, :] - - # [seq_block, V_D_HEAD] - v = tl.load(v_cache_ptr + v_block_offsets, mask=v_mask, other=0.0) - - # Note: check the output precision of the sum. - # compute q*K^T - # [NUM_HEADS, Q_D_HEAD] * [seq_block, Q_D_HEAD], sum along axis 1 - attn = tl.dot(q, k.trans()) # [N, seq_block] - attn = attn.to(tl.float32) - attn *= SCALE - # Set to -inf attn values where mask is not set. This forces exp(attn) to 0. - attn = tl.where(head_mask[:, None] * seq_mask[None, :], attn, float("-inf")) - # compute max_attn only when invalid attn values are masked out. - max_attn = tl.max(attn, axis=1) # [N, 1] - - exp_attn = tl.exp(attn - max_attn[:, None]) - sumexp = tl.sum(exp_attn, axis=1) # [N, 1] - - # [NUM_HEADS, seq_len] * [seq_len, V_D_HEAD], sum along axis 0 - output = tl.dot(exp_attn.to(v.dtype), v) - - output = output / sumexp[:, None] # [N, D_HEAD] - - # We store the log-sum-exp after removing the max. - logsumexp = tl.log(sumexp) + max_attn - # when seq_mask is all false, max_attn will be -inf and sumexp is zero - - tl.store( - output_values_ptr - + batch_id * N_HEADS * V_D_HEAD * num_blocks - + head_offsets[:, None] * V_D_HEAD * num_blocks - + seq_block_id * V_D_HEAD - + v_dhead_offsets[None, :], - output, - mask=head_mask[:, None] * v_dhead_mask[None, :], - ) - tl.store( - output_logsumexp_ptr - + batch_id * N_HEADS * num_blocks - + head_offsets * num_blocks - + seq_block_id, - logsumexp, - mask=head_mask, - ) - - -@triton.jit -def attention_kv_stage1( - q_ptr, # [Batch, 1, N_HEADS, D_HEAD] - k_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - v_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - slot_idx_ptr, # [Batch] # Specifies the slot index for each of the generate tokens. - input_pos_ptr, # [Batch] - output_values_ptr, # [Batch, N_HEADS, num_blocks, D_HEAD] - output_logsumexp_ptr, # [Batch, N_HEADS, num_blocks] - num_blocks, - MAX_SEQ_LEN: tl.constexpr, # Maximum supported sequence length - N_HEADS: tl.constexpr, # Number of heads - N_KV_HEADS: tl.constexpr, # Number of KV heads. - D_HEAD: tl.constexpr, # Dimension of each head. - SEQ_BLOCK_SIZE: tl.constexpr, # Block size used for tiling the sequence dim. -): - """Attention kernel to be used for generate-only batches. - - Assumes that kv caches have been updated. - - Uses flash decoding. - KV-cache layout is assumed to be [Batch,Seq, Head, Dim] - 1. Fetch the K-cache from 0 to input_pos - 2. Fetch the V-cache from 0 to input_pos - 3. A = Q*K^T [1,D_HEAD] * [1,seq_len,D_HEAD] -> [1, seq_len] - 4. S = softmax(A) - 5. O = S*V [1, seq_len] * [1, seq_len, D_HEAD] -> [1, D_HEAD] - """ - # Assume KV-cache layout: [Batch, Seq, Head, Dim] - # A program is responsible for 1 batch, 1 head and a block of sequences. - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - epsilon: tl.constexpr = 1e-38 # float32 smallest positive number - - kv_position = tl.load(input_pos_ptr + batch_id) - slot_idx = tl.load(slot_idx_ptr + batch_id) - slot_batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LEN * D_HEAD - # Offsets for the block of sequences this program processes. - seq_start_pos = seq_block_id * SEQ_BLOCK_SIZE - - if seq_start_pos > kv_position: - return - seq_offsets = seq_start_pos + tl.arange(0, SEQ_BLOCK_SIZE) - seq_mask = seq_offsets <= kv_position - # Assuming D_HEAD is a power of 2 - dhead_offsets = tl.arange(0, triton.next_power_of_2(D_HEAD)) - dhead_mask = dhead_offsets < D_HEAD - - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS - kv_head_offset = (head_id // HEAD_RATIO) * D_HEAD - - sm_scale: tl.constexpr = 1.0 / (D_HEAD**0.5) - - # Program loads the entire Q for the head assigned to it. - # [D_HEAD] - q_batch_offset = batch_id * N_HEADS * D_HEAD - q_head_offset = head_id * D_HEAD - q = tl.load(q_ptr + q_batch_offset + q_head_offset + dhead_offsets, mask=dhead_mask) - - kv_block_offsets = ( - slot_batch_offset - + seq_offsets[:, None] * D_HEAD * N_KV_HEADS - + kv_head_offset - + dhead_offsets[None, :] - ) # [BSND] - kv_mask = seq_mask[:, None] * dhead_mask[None, :] - - # [seq_block, D_HEAD] - k = tl.load(k_cache_ptr + kv_block_offsets, mask=kv_mask, other=0.0) - v = tl.load(v_cache_ptr + kv_block_offsets, mask=kv_mask, other=0.0) - - # Note: check the output precision of the sum. - # compute q*K^T - # [D_HEAD] * [seq_block, D_HEAD], sum along axis 1 - attn = tl.sum(q[None, :].to(tl.float32) * k.to(tl.float32), axis=1) # [seq_block] - - attn *= sm_scale - max_attn = tl.max(attn) - # Set to -inf attn values where mask is not set. This forces exp(attn) to 0. - attn = tl.where(seq_mask, attn, float("-inf")) - exp_attn = tl.exp(attn - max_attn) - exp_attn = tl.where(exp_attn == 0, epsilon, exp_attn) - sumexp = tl.sum(exp_attn, axis=0) # scalar. - - # [seq_len] * [seq_len, D_HEAD], sum along axis 0 - output = tl.sum(exp_attn[:, None] * v, axis=0) # [D_HEAD] - - output = output / sumexp - - # We store the log-sum-exp after removing the max. - logsumexp = tl.log(sumexp) + max_attn - # when seq_mask is all false, max_attn will be -inf and sumexp is zero - - tl.store( - output_values_ptr - + batch_id * N_HEADS * D_HEAD * num_blocks - + head_id * D_HEAD * num_blocks - + seq_block_id * D_HEAD - + dhead_offsets, - output, - mask=dhead_mask, - ) - tl.store( - output_logsumexp_ptr - + batch_id * N_HEADS * num_blocks - + head_id * num_blocks - + seq_block_id, - logsumexp, - ) - - -@triton.jit -def attention_kv_stage2( - values_ptr, # [Batch, N_HEADS, num_blocks, D_HEAD] - logsumexp_ptr, # [Batch, N_HEADS, num_blocks] - output_ptr, # [Batch, N_HEADS, D_HEAD] - input_pos_ptr, - NUM_BLOCKS: tl.constexpr, - N_HEADS: tl.constexpr, - D_HEAD: tl.constexpr, - SEQ_BLOCK_SIZE: tl.constexpr, # Nearest power of 2 for num_blocks - HAS_SINKS: tl.constexpr, - sinks_ptr, -): - # There are batch * N_HEADS programs - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - - dhead_offsets = tl.arange(0, triton.next_power_of_2(D_HEAD)) - dhead_mask = dhead_offsets < D_HEAD - - kv_position = tl.load(input_pos_ptr + batch_id) - block_id = kv_position // SEQ_BLOCK_SIZE + 1 - - NUM_BLOCKS_POW2: tl.constexpr = triton.next_power_of_2(NUM_BLOCKS) - block_offsets = tl.arange(0, NUM_BLOCKS_POW2) - - block_mask = block_offsets < block_id - logsumexp = tl.load( - logsumexp_ptr + batch_id * N_HEADS * NUM_BLOCKS + head_id * NUM_BLOCKS + block_offsets, - mask=block_mask, - other=float("-inf"), - ) - max_logsumexp = tl.max(logsumexp) - sumexp = tl.exp(logsumexp - max_logsumexp) # [NUM_BLOCKS_POW2] - - aggregate_sumexp = tl.sum(sumexp, axis=0) - # Add sinks contribution to the softmax denominator - if HAS_SINKS: - sinks_val = tl.load(sinks_ptr + batch_id * N_HEADS + head_id) - sinks_exp = tl.exp(sinks_val - max_logsumexp) - aggregate_sumexp += sinks_exp - - values_offsets = block_offsets[:, None] * D_HEAD + dhead_offsets[None, :] - values_mask = block_mask[:, None] * dhead_mask[None, :] - - values = tl.load( - values_ptr - + batch_id * N_HEADS * D_HEAD * NUM_BLOCKS - + head_id * D_HEAD * NUM_BLOCKS - + values_offsets, - mask=values_mask, - other=0.0, - ) # [BLOCK_SIZE, D_HEAD] - values *= sumexp[:, None] - values /= aggregate_sumexp - - output = tl.sum(values, axis=0) # [DHEAD] - - tl.store( - output_ptr + batch_id * N_HEADS * D_HEAD + head_id * D_HEAD + dhead_offsets, - output, - mask=dhead_mask, - ) - - -@triton.jit -def context_attention_kv( - q_ptr, # [bsnd] - k_ptr, # [bsnd] - v_ptr, # [bsnd] - k_cache_ptr, # [bsnd] - v_cache_ptr, # [bsnd] - seq_len, - o_ptr, - SCALE: tl.constexpr, - N_HEADS: tl.constexpr, # Number of heads - N_KV_HEADS: tl.constexpr, # Number of KV heads. - Q_D_HEAD: tl.constexpr, # Dimension of each query head. - V_D_HEAD: tl.constexpr, # Dimension of each value head. - SEQ_BLOCK: tl.constexpr, - MAX_SEQ_LENGTH: tl.constexpr, -): - """Kernel for context phase. - - Assuming: - 1. Self-attention [seqlen(Q) == seqlen(K)] - 2. Causal attention - 3. QKV layout: [bsnd] - """ - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS - K_D_HEAD: tl.constexpr = Q_D_HEAD - - q_dhead_offsets = tl.arange(0, triton.next_power_of_2(Q_D_HEAD)) - q_dhead_mask = q_dhead_offsets < Q_D_HEAD - - v_dhead_offsets = tl.arange(0, triton.next_power_of_2(V_D_HEAD)) - v_dhead_mask = v_dhead_offsets < V_D_HEAD - - seq_offsets = seq_block_id * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - seq_mask = seq_offsets < seq_len - - q_load_mask = seq_mask[:, None] * q_dhead_mask[None, :] - - q_batch_offset = batch_id * seq_len * N_HEADS - kv_batch_offset = batch_id * seq_len * N_KV_HEADS - - k_head_offset = (head_id // HEAD_RATIO) * K_D_HEAD - v_head_offset = (head_id // HEAD_RATIO) * V_D_HEAD - - # Q will stay in SRAM - q = tl.load( - q_ptr - + q_batch_offset * Q_D_HEAD - + seq_offsets[:, None] * N_HEADS * Q_D_HEAD - + head_id * Q_D_HEAD - + q_dhead_offsets[None, :], - mask=q_load_mask, - ) - acc = tl.zeros([SEQ_BLOCK, triton.next_power_of_2(V_D_HEAD)], dtype=tl.float32) - lse_i = tl.zeros([SEQ_BLOCK], dtype=tl.float32) - float("inf") - m_i = tl.zeros([SEQ_BLOCK], dtype=tl.float32) - float("inf") - - for s in range(0, seq_block_id + 1, 1): - kv_seq_offsets = s * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - kv_seq_mask = kv_seq_offsets < seq_len - k_load_mask = kv_seq_mask[:, None] * q_dhead_mask[None, :] - - k = tl.load( - k_ptr - + kv_batch_offset * K_D_HEAD - + kv_seq_offsets[:, None] * N_KV_HEADS * K_D_HEAD - + k_head_offset - + q_dhead_offsets[None, :], - mask=k_load_mask, - ) - qk = tl.zeros([SEQ_BLOCK, SEQ_BLOCK], dtype=tl.float32) - qk += tl.dot(q, k.trans()) - # causal mask - qk = tl.where(seq_offsets[:, None] >= kv_seq_offsets[None, :], qk, float("-inf")) - qk *= SCALE - # rowmax - m_ij = tl.maximum(tl.max(qk, 1), lse_i) - p = tl.exp(qk - m_ij[:, None]) # [S,S] - v = tl.load( - v_ptr - + kv_batch_offset * V_D_HEAD - + kv_seq_offsets[:, None] * N_KV_HEADS * V_D_HEAD - + v_head_offset - + v_dhead_offsets[None, :], - mask=kv_seq_mask[:, None] * v_dhead_mask[None, :], - ) - - l_ij = tl.sum(p, 1) - acc_scale = tl.exp(m_i - m_ij) - acc = acc * acc_scale[:, None] - p = p.to(v.dtype) - acc += tl.dot(p, v) - m_i = m_ij - l_i_new = tl.exp(lse_i - m_ij) + l_ij - lse_i = m_ij + tl.log(l_i_new) - - o_scale = tl.exp(m_i - lse_i) - - acc = acc * o_scale[:, None] - - tl.store( - o_ptr - + batch_id * seq_len * N_HEADS * V_D_HEAD - + seq_offsets[:, None] * N_HEADS * V_D_HEAD - + head_id * V_D_HEAD - + v_dhead_offsets[None, :], - acc, - mask=seq_mask[:, None] * v_dhead_mask[None, :], - ) - - # Write back to kv-caches - - ks = tl.load( - k_ptr - + kv_batch_offset * K_D_HEAD - + seq_offsets[:, None] * N_KV_HEADS * K_D_HEAD - + k_head_offset - + q_dhead_offsets[None, :], - mask=seq_mask[:, None] * q_dhead_mask[None, :], - ) - vs = tl.load( - v_ptr - + kv_batch_offset * V_D_HEAD - + seq_offsets[:, None] * N_KV_HEADS * V_D_HEAD - + v_head_offset - + v_dhead_offsets[None, :], - mask=seq_mask[:, None] * v_dhead_mask[None, :], - ) - # cache is [bsnd] - k_cache_offset = ( - batch_id * N_KV_HEADS * MAX_SEQ_LENGTH * K_D_HEAD - + seq_offsets[:, None] * K_D_HEAD * N_KV_HEADS - + k_head_offset - + q_dhead_offsets[None, :] - ) - - v_cache_offset = ( - batch_id * N_KV_HEADS * MAX_SEQ_LENGTH * V_D_HEAD - + seq_offsets[:, None] * V_D_HEAD * N_KV_HEADS - + v_head_offset - + v_dhead_offsets[None, :] - ) - tl.store(k_cache_ptr + k_cache_offset, ks, seq_mask[:, None] * q_dhead_mask[None, :]) - tl.store(v_cache_ptr + v_cache_offset, vs, seq_mask[:, None] * v_dhead_mask[None, :]) - - -@triton.jit -def context_attention_kv_flattened( - q_ptr, # [b*s,nd] - seq_len_ptr, # [b] # length of each sequence in a batch - seq_start_indices_ptr, # [b] # start indices of a sequence in flattened q/k/v. - k_cache_ptr, # [bsnd] - v_cache_ptr, # [bsnd] - input_pos_ptr, # [b] # specifies the location in the sequence where kv must be written back. - slot_idx_ptr, # [b] # slot index of the sequence in the cache. - o_ptr, - SCALE: tl.constexpr, - N_HEADS: tl.constexpr, # Number of heads - N_KV_HEADS: tl.constexpr, # Number of KV heads. - Q_D_HEAD: tl.constexpr, # Dimension of each query head. - V_D_HEAD: tl.constexpr, # Dimension of each value head. - SEQ_BLOCK: tl.constexpr, - MAX_SEQ_LENGTH: tl.constexpr, - SLIDING_WINDOW: tl.constexpr, # Sliding window size, -1 means no sliding window - HAS_SINKS: tl.constexpr, - sinks_ptr, -): - """Kernel for context phase. - - Assumes that kv caches have been updated. - Assuming QKV layout: [b*s,n,d] - """ - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - - # Each program is responsible for a block of tokens in a single batch. - seq_start_index = tl.load(seq_start_indices_ptr + batch_id) - seq_len = tl.load(seq_len_ptr + batch_id) - K_D_HEAD: tl.constexpr = Q_D_HEAD - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS - - # cache is [bsnd] - # slot_idx_ptr stores the slot index for the sequences provided to the kernel. - slot_idx = tl.load(slot_idx_ptr + batch_id) - - cache_batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LENGTH - cache_head_offset = head_id // HEAD_RATIO - - q_dhead_offsets = tl.arange(0, triton.next_power_of_2(Q_D_HEAD)) - q_dhead_mask = q_dhead_offsets < Q_D_HEAD - - v_dhead_offsets = tl.arange(0, triton.next_power_of_2(V_D_HEAD)) - v_dhead_mask = v_dhead_offsets < V_D_HEAD - - seq_offsets = seq_block_id * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - seq_mask = seq_offsets < seq_len - - # Q will stay in SRAM - q = tl.load( - q_ptr - + seq_start_index * N_HEADS * Q_D_HEAD - + seq_offsets[:, None] * N_HEADS * Q_D_HEAD - + head_id * Q_D_HEAD - + q_dhead_offsets[None, :], - mask=seq_mask[:, None] * q_dhead_mask[None, :], - ) - - acc = tl.zeros([SEQ_BLOCK, triton.next_power_of_2(V_D_HEAD)], dtype=tl.float32) - lse_i = tl.zeros([SEQ_BLOCK], dtype=tl.float32) - float("inf") - m_i = tl.zeros([SEQ_BLOCK], dtype=tl.float32) - float("inf") - - # Loop over the entire KV-history - # input_pos_ptr stores the location at which kv must be written back for the given batch. - kv_position = tl.load(input_pos_ptr + batch_id) - num_blocks = (kv_position + seq_len + SEQ_BLOCK - 1) // SEQ_BLOCK - start = 0 - if SLIDING_WINDOW > 0: - # Use the LAST query in this block for more conservative start calculation - last_q_pos = ( - (seq_block_id + 1) * SEQ_BLOCK - 1 + kv_position - ) # Last query's absolute position - earliest_kv_pos = max(0, last_q_pos - SLIDING_WINDOW + 1) - start = max(0, earliest_kv_pos // SEQ_BLOCK) - for s in range(start, num_blocks + 1): - kv_seq_offsets = s * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - kv_seq_mask = kv_seq_offsets < (kv_position + seq_len) - - k = tl.load( - k_cache_ptr - + cache_batch_offset * K_D_HEAD - + kv_seq_offsets[:, None] * K_D_HEAD * N_KV_HEADS - + cache_head_offset * K_D_HEAD - + q_dhead_offsets[None, :], - mask=kv_seq_mask[:, None] * q_dhead_mask[None, :], - ) - qk = tl.zeros([SEQ_BLOCK, SEQ_BLOCK], dtype=tl.float32) - qk += tl.dot(q, k.trans()) - # Apply causal mask - causal_mask = (seq_offsets[:, None] + kv_position) >= kv_seq_offsets[None, :] - # Apply sliding window mask if enabled - if SLIDING_WINDOW > 0: - sliding_window_mask = kv_seq_offsets[None, :] >= ( - seq_offsets[:, None] + kv_position - SLIDING_WINDOW + 1 - ) - combined_mask = sliding_window_mask & causal_mask - else: - combined_mask = causal_mask - qk = tl.where(combined_mask, qk, float("-inf")) - qk *= SCALE - # rowmax - m_ij = tl.maximum(tl.max(qk, 1), lse_i) - p = tl.exp(qk - m_ij[:, None]) - v = tl.load( - v_cache_ptr - + cache_batch_offset * V_D_HEAD - + kv_seq_offsets[:, None] * V_D_HEAD * N_KV_HEADS - + cache_head_offset * V_D_HEAD - + v_dhead_offsets[None, :], - mask=kv_seq_mask[:, None] * v_dhead_mask[None, :], - ) - - l_ij = tl.sum(p, 1) - acc_scale = tl.exp(m_i - m_ij) - acc = acc * acc_scale[:, None] - p = p.to(v.dtype) - acc += tl.dot(p, v) - m_i = m_ij - l_i_new = tl.exp(lse_i - m_ij) + l_ij - lse_i = m_ij + tl.log(l_i_new) - - # Add sinks contribution to the final softmax calculation - if HAS_SINKS: - sinks_val = tl.load(sinks_ptr + batch_id * N_HEADS + head_id) - m_sinks = tl.maximum(m_i, sinks_val) - acc_scale = tl.exp(m_i - m_sinks) - acc = acc * acc_scale[:, None] - l_sinks = tl.exp(lse_i - m_sinks) + tl.exp(sinks_val - m_sinks) - lse_i = m_sinks + tl.log(l_sinks) - m_i = m_sinks - - o_scale = tl.exp(m_i - lse_i) - - acc = acc * o_scale[:, None] - - tl.store( - o_ptr - + seq_start_index * N_HEADS * V_D_HEAD - + seq_offsets[:, None] * N_HEADS * V_D_HEAD - + head_id * V_D_HEAD - + v_dhead_offsets[None, :], - acc, - mask=seq_mask[:, None] * v_dhead_mask[None, :], - ) - - -@triton.jit -def update_kv_cache_rope_fusion( - q_ptr, # [B*S, N, D] - k_ptr, # [B*S, N, D] - v_ptr, # [B*S, N, D] - seq_len_ptr, # [b] # length of each sequence in a batch - seq_start_indices_ptr, # [b] # start indices of a sequence in flattened q/k/v. - q_rope_ptr, # [B*S, N, D], roped q result - k_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - v_cache_ptr, # [MAX_BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD] - input_pos_ptr, # Specifies the sequence index in the caches at which to write the provided kv - slot_idx_ptr, # Specifies the slot index for each of the input sequences - f_ptr, # [MAX_SEQ_LEN, D_HEAD//2, 2] # frequencies for rope embadding. - MAX_SEQ_LENGTH: tl.constexpr, - N_HEADS: tl.constexpr, - N_KV_HEADS: tl.constexpr, - D_HEAD: tl.constexpr, - SEQ_BLOCK: tl.constexpr, - HEAD_BLOCK_SIZE: tl.constexpr, # pad to 16 if HEAD_RATIO is < 16 to invoke tensor cores. - GENERATE_ONLY: tl.constexpr, -): - """Fuse q and k rope with update_kv_cache kernel. - - The input is interleaved as [2, D//2] in D_HEAD dim. - Update q_rope with the post-rope-embadding q values. - Update k_cache with the post-rope-embadding k values. - For rope computation, q and k need to load and store in tensors pair of 2 * [D//2]. - Update v_cache with v. - """ - batch_id = tl.program_id(axis=0) - kv_head_id = tl.program_id(axis=1) - seq_block_id = tl.program_id(axis=2) - - # Each program is responsible for a block of tokens in a single batch. - if GENERATE_ONLY: - seq_start_index = batch_id - seq_len: tl.constexpr = 1 - else: - seq_start_index = tl.load(seq_start_indices_ptr + batch_id) - seq_len = tl.load(seq_len_ptr + batch_id) - - # cache is [bsnd] - # slot_idx_ptr stores the slot index for the sequences provided to the kernel. - slot_idx = tl.load(slot_idx_ptr + batch_id) - - kv_position = tl.load(input_pos_ptr + batch_id) - - cache_batch_offset = slot_idx * N_KV_HEADS * MAX_SEQ_LENGTH * D_HEAD - cache_head_offset = kv_head_id * D_HEAD - - # Assuming D_HEAD is a power of 2 - dhead_offsets = tl.arange(0, D_HEAD) - dhead_mask = dhead_offsets < D_HEAD - - seq_offsets = seq_block_id * SEQ_BLOCK + tl.arange(0, SEQ_BLOCK) - seq_mask = seq_offsets < seq_len - - load_mask = seq_mask[:, None] * dhead_mask[None, :] - - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS # This needs to be a power-of-2 - q_head_offsets = kv_head_id * HEAD_RATIO + tl.arange(0, HEAD_BLOCK_SIZE) - q_head_mask = q_head_offsets < (kv_head_id * HEAD_RATIO + HEAD_RATIO) - - q_batch_offset = seq_start_index * N_HEADS * D_HEAD - - kv_batch_offset = seq_start_index * N_KV_HEADS * D_HEAD - kv_head_offset = cache_head_offset - - D2: tl.constexpr = D_HEAD // 2 - # input is interleaved as [2, D//2] in dim [D_HEAD]. - d2_offsets = tl.arange(0, D2) - dhead_offsets1 = d2_offsets - dhead_offsets2 = d2_offsets + D2 - d2_mask = dhead_offsets2 < D_HEAD - d2_load_mask = seq_mask[:, None] * d2_mask[None, :] - - # offsets of [bsn] - q_offsets_base = ( - q_batch_offset - + seq_offsets[:, None, None] * N_HEADS * D_HEAD - + q_head_offsets[None, :, None] * D_HEAD - ) - q_offsets1 = q_offsets_base + dhead_offsets1[None, None, :] - q_offsets2 = q_offsets_base + dhead_offsets2[None, None, :] - q_mask = d2_load_mask[:, None, :] * q_head_mask[None, :, None] - - q1 = tl.load(q_ptr + q_offsets1, mask=q_mask).to(tl.float32) - q2 = tl.load(q_ptr + q_offsets2, mask=q_mask).to(tl.float32) - - k_offsets_base = kv_batch_offset + seq_offsets[:, None] * N_KV_HEADS * D_HEAD + kv_head_offset - k_offsets1 = k_offsets_base + dhead_offsets1[None, :] - k_offsets2 = k_offsets_base + dhead_offsets2[None, :] - - k1 = tl.load(k_ptr + k_offsets1, mask=d2_load_mask).to(tl.float32) - k2 = tl.load(k_ptr + k_offsets2, mask=d2_load_mask).to(tl.float32) - - # ----------------------------------- - # torch version sin/cos - # cos and sin values are interleaved in frequencies tensor. - f_offsets = seq_offsets[:, None] * D2 + d2_offsets[None, :] - cos_ref = tl.load(f_ptr + kv_position * D_HEAD + f_offsets * 2, mask=d2_load_mask).to( - dtype=tl.float32 - ) - sin_ref = tl.load(f_ptr + kv_position * D_HEAD + f_offsets * 2 + 1, mask=d2_load_mask).to( - dtype=tl.float32 - ) - - qs1 = cos_ref[:, None, :] * q1 - sin_ref[:, None, :] * q2 - qs2 = sin_ref[:, None, :] * q1 + cos_ref[:, None, :] * q2 - - tl.store(q_rope_ptr + q_offsets1, qs1, mask=q_mask) - tl.store(q_rope_ptr + q_offsets2, qs2, mask=q_mask) - - ks1 = cos_ref * k1 - sin_ref * k2 - ks2 = sin_ref * k1 + cos_ref * k2 - - # Write back to kv-caches - vs = tl.load( - v_ptr - + kv_batch_offset - + seq_offsets[:, None] * N_KV_HEADS * D_HEAD - + kv_head_offset - + dhead_offsets[None, :], - mask=load_mask, - ) - - kv_writeback_seq_offsets = seq_offsets + kv_position - - cache_offset_base = ( - cache_batch_offset - + kv_writeback_seq_offsets[:, None] * D_HEAD * N_KV_HEADS - + cache_head_offset - ) - - k_cache_offset1 = cache_offset_base + dhead_offsets1[None, :] - k_cache_offset2 = cache_offset_base + dhead_offsets2[None, :] - tl.store(k_cache_ptr + k_cache_offset1, ks1, mask=d2_load_mask) - tl.store(k_cache_ptr + k_cache_offset2, ks2, mask=d2_load_mask) - - v_cache_offset = cache_offset_base + dhead_offsets[None, :] - tl.store(v_cache_ptr + v_cache_offset, vs, load_mask) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py deleted file mode 100644 index 192a2e659d44..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/attention/triton_paged_attention.py +++ /dev/null @@ -1,1578 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Triton Paged Attention - -This module provides a Triton-based paged attention implementation with: -- Combined KV cache with HND layout: [num_blocks, 2, num_kv_heads, page_size, head_dim] -- Context/prefill kernel with causal masking -""" - -import math -from typing import List, Literal, Optional, Tuple - -import flashinfer -import torch -import triton -import triton.language as tl -from torch._ops import OpOverloadPacket -from torch._subclasses import FakeTensor -from torch.fx import Node - -from ..._compat import KvCacheConfig -from ...utils.logger import ad_logger -from ...utils.node_utils import extract_op_args -from ..attention_interface import ( - AttentionDescriptor, - AttentionLayout, - AttentionRegistry, - Constant, - KVPagedResourceHandler, - MHACallable, - PrepareMetadataCallable, - ResourceHandlerDict, -) - -KV_LAYOUT: Literal["HND", "NHD"] = "HND" - -# Cache SM count to avoid repeated get_device_properties calls -_NUM_SMS: Optional[int] = None - - -def _get_num_sms() -> int: - """Get the number of SMs on the current GPU (cached).""" - global _NUM_SMS - if _NUM_SMS is None: - _NUM_SMS = torch.cuda.get_device_properties(0).multi_processor_count - return _NUM_SMS - - -def _get_sm_scale(head_dim: int, scale: Optional[float]) -> float: - """Get softmax scale, computing default if not provided.""" - return scale if scale is not None else 1.0 / math.sqrt(head_dim) - - -@triton.jit -def _update_paged_kv_cache_kernel( - # Input K, V - k_ptr, - v_ptr, - # Metadata - batch_indices_ptr, - positions_ptr, - # KV cache - kv_cache_ptr, - # Page table - kv_indices_ptr, - kv_indptr_ptr, - # Constants - NUM_TOKENS: tl.constexpr, - N_KV_HEADS: tl.constexpr, - HEAD_DIM: tl.constexpr, - PAGE_SIZE: tl.constexpr, - # Strides for kv_cache: [num_blocks, 2, num_kv_heads, page_size, head_dim] - cache_stride_block: tl.constexpr, - cache_stride_kv: tl.constexpr, - cache_stride_head: tl.constexpr, - cache_stride_token: tl.constexpr, -): - """Update combined KV cache with new tokens.""" - token_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - - if token_id >= NUM_TOKENS: - return - - batch_idx = tl.load(batch_indices_ptr + token_id) - position = tl.load(positions_ptr + token_id) - - page_idx_in_seq = position // PAGE_SIZE - offset_in_page = position % PAGE_SIZE - - page_start = tl.load(kv_indptr_ptr + batch_idx) - physical_page = tl.load(kv_indices_ptr + page_start + page_idx_in_seq) - - head_offsets = tl.arange(0, HEAD_DIM) - kv_offset = token_id * N_KV_HEADS * HEAD_DIM + head_id * HEAD_DIM + head_offsets - - k = tl.load(k_ptr + kv_offset) - v = tl.load(v_ptr + kv_offset) - - # Compute cache offset (use int64 to avoid overflow when physical_page * stride > 2^31) - cache_base = ( - physical_page.to(tl.int64) * cache_stride_block - + head_id * cache_stride_head - + offset_in_page.to(tl.int64) * cache_stride_token - + head_offsets - ) - - tl.store(kv_cache_ptr + cache_base, k) - tl.store(kv_cache_ptr + cache_base + cache_stride_kv, v) - - -def update_paged_kv_cache( - k: torch.Tensor, - v: torch.Tensor, - batch_indices: torch.Tensor, - positions: torch.Tensor, - kv_cache: torch.Tensor, - kv_indices: torch.Tensor, - kv_indptr: torch.Tensor, -) -> None: - """Update the combined paged KV cache with new K, V tensors.""" - num_tokens, n_kv_heads, head_dim = k.shape - page_size = kv_cache.shape[3] - - if num_tokens == 0: - return - - grid = (num_tokens, n_kv_heads) - _update_paged_kv_cache_kernel[grid]( - k, - v, - batch_indices, - positions, - kv_cache, - kv_indices, - kv_indptr, - NUM_TOKENS=num_tokens, - N_KV_HEADS=n_kv_heads, - HEAD_DIM=head_dim, - PAGE_SIZE=page_size, - cache_stride_block=kv_cache.stride(0), - cache_stride_kv=kv_cache.stride(1), - cache_stride_head=kv_cache.stride(2), - cache_stride_token=kv_cache.stride(3), - ) - - -# ============================================================================= -# FLASH DECODE - HELPERS -# ============================================================================= - - -def _get_num_splits(max_seq_len: int, batch_size: int, n_kv_heads: int, page_size: int) -> int: - """Compute optimal number of KV splits for FlashDecoding. - - With GQA batching, the grid is (batch, n_kv_heads, num_splits). - We want enough blocks to saturate the GPU. - """ - if max_seq_len <= 0: - return 1 - - num_sms = _get_num_sms() - existing_parallelism = batch_size * n_kv_heads - - # Already enough parallelism - if existing_parallelism >= num_sms * 2: - return 1 - - # Target ~4 waves of thread blocks - target_blocks = num_sms * 4 - num_splits = max(1, (target_blocks + existing_parallelism - 1) // existing_parallelism) - - # Cap splits so each block has at least 2 pages of work. - # With fewer pages, the per-block overhead (Q load, accumulator init, - # partial_o/lse store, plus stage2 reduction cost) dominates the useful - # compute (page-loop iterations). 2 pages is a conservative lower bound - # to keep the overhead-to-work ratio acceptable. - max_pages = max_seq_len // page_size - max_splits = max(1, max_pages // 2) - num_splits = min(num_splits, max_splits) - - # Round to next power of 2 for Triton compile caching - if num_splits > 1: - num_splits = 2 ** math.ceil(math.log2(num_splits)) - - return min(num_splits, 128) - - -@triton.autotune( - configs=[ - triton.Config({}, num_warps=2, num_stages=2), - triton.Config({}, num_warps=2, num_stages=3), - triton.Config({}, num_warps=4, num_stages=2), - triton.Config({}, num_warps=4, num_stages=3), - triton.Config({}, num_warps=8, num_stages=2), - triton.Config({}, num_warps=8, num_stages=3), - ], - key=["HEAD_DIM", "PAGE_SIZE", "HEAD_RATIO_PADDED", "SLIDING_WINDOW"], -) -@triton.jit -def _flash_decode_stage1_kernel( - # Query input - q_ptr, - # KV cache (combined) - kv_cache_ptr, - # Page table - kv_indices_ptr, - kv_indptr_ptr, - kv_last_page_len_ptr, - # Intermediate outputs - partial_o_ptr, - partial_lse_ptr, - # Q strides: [batch, n_heads, head_dim] - q_stride_batch: tl.constexpr, - q_stride_head: tl.constexpr, - # Partial output strides: [batch, n_heads, num_splits, head_dim] - po_stride_batch: tl.constexpr, - po_stride_head: tl.constexpr, - po_stride_split: tl.constexpr, - # Partial LSE strides: [batch, n_heads, num_splits] - plse_stride_batch: tl.constexpr, - plse_stride_head: tl.constexpr, - plse_stride_split: tl.constexpr, - # Cache strides: [num_blocks, 2, n_kv_heads, page_size, head_dim] - cache_stride_block: tl.constexpr, - cache_stride_kv: tl.constexpr, - cache_stride_head: tl.constexpr, - cache_stride_token: tl.constexpr, - # Constants - SM_SCALE: tl.constexpr, - N_HEADS: tl.constexpr, - N_KV_HEADS: tl.constexpr, - HEAD_DIM: tl.constexpr, - PAGE_SIZE: tl.constexpr, - HEAD_RATIO: tl.constexpr, - HEAD_RATIO_PADDED: tl.constexpr, - NUM_SPLITS: tl.constexpr, - SLIDING_WINDOW: tl.constexpr = 0, -): - """ - Key optimizations: - - Loads KV once for HEAD_RATIO Q heads - - Iterates by page for contiguous memory access - - Splits KV sequence across blocks for GPU utilization - """ - batch_id = tl.program_id(axis=0) - kv_head_id = tl.program_id(axis=1) - split_id = tl.program_id(axis=2) - - # Get sequence info from page table - kv_page_start = tl.load(kv_indptr_ptr + batch_id) - kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) - num_pages = kv_page_end - kv_page_start - last_page_len = tl.load(kv_last_page_len_ptr + batch_id) - - # Sliding window: restrict attention to pages within the window. - # Compute the total sequence length and the first valid KV position. - seq_len = (num_pages - 1) * PAGE_SIZE + last_page_len - if SLIDING_WINDOW > 0: - first_valid_pos = tl.maximum(0, seq_len - SLIDING_WINDOW) - first_window_page = first_valid_pos // PAGE_SIZE - else: - first_valid_pos = 0 - first_window_page = 0 - - # Only split over pages within the window - window_pages = num_pages - first_window_page - pages_per_split = (window_pages + NUM_SPLITS - 1) // NUM_SPLITS - page_split_start = first_window_page + split_id * pages_per_split - page_split_end = tl.minimum(page_split_start + pages_per_split, num_pages) - - dhead_offsets = tl.arange(0, HEAD_DIM) - # Use padded range for Triton power-of-2 requirement; mask out-of-bounds heads - head_local = tl.arange(0, HEAD_RATIO_PADDED) - head_ids = kv_head_id * HEAD_RATIO + head_local - head_mask = head_local < HEAD_RATIO - - # Handle inactive splits (beyond the sequence's pages) - if page_split_start >= num_pages: - # Store zeros + -inf LSE for valid HEAD_RATIO Q heads only - po_offsets = ( - batch_id * po_stride_batch - + head_ids[:, None] * po_stride_head - + split_id * po_stride_split - + dhead_offsets[None, :] - ) - tl.store( - partial_o_ptr + po_offsets, - tl.zeros([HEAD_RATIO_PADDED, HEAD_DIM], dtype=tl.float32), - mask=head_mask[:, None], - ) - plse_offsets = ( - batch_id * plse_stride_batch - + head_ids * plse_stride_head - + split_id * plse_stride_split - ) - tl.store( - partial_lse_ptr + plse_offsets, - tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) + float("-inf"), - mask=head_mask, - ) - return - - # Load Q for HEAD_RATIO heads sharing this KV head: [HEAD_RATIO_PADDED, HEAD_DIM] - # Padded rows get zeros, producing zero attention scores (harmless, never stored) - q_offsets = ( - batch_id * q_stride_batch + head_ids[:, None] * q_stride_head + dhead_offsets[None, :] - ) - q_all = tl.load(q_ptr + q_offsets, mask=head_mask[:, None], other=0.0) - - acc = tl.zeros([HEAD_RATIO_PADDED, HEAD_DIM], dtype=tl.float32) - m_i = tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) + float("-inf") - l_i = tl.zeros([HEAD_RATIO_PADDED], dtype=tl.float32) - - num_pages_this_split = page_split_end - page_split_start - for local_page_idx in range(num_pages_this_split): - page_idx = page_split_start + local_page_idx - physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) - - # Determine valid tokens in this page - is_last_page_of_seq = page_idx == (num_pages - 1) - valid_tokens = tl.where(is_last_page_of_seq, last_page_len, PAGE_SIZE) - - page_offsets = tl.arange(0, PAGE_SIZE) - page_mask = page_offsets < valid_tokens - - # Compute cache offset (use int64 to avoid overflow when physical_page * stride > 2^31) - cache_base = ( - physical_page.to(tl.int64) * cache_stride_block - + kv_head_id * cache_stride_head - + page_offsets[:, None] * cache_stride_token - + dhead_offsets[None, :] - ) - page_mask_2d = page_mask[:, None] - - k = tl.load(kv_cache_ptr + cache_base, mask=page_mask_2d, other=0.0).to( - q_all.dtype - ) # [PAGE_SIZE, HEAD_DIM]; cast from fp8 if kv cache is fp8 - v = tl.load( - kv_cache_ptr + cache_base + cache_stride_kv, - mask=page_mask_2d, - other=0.0, - ).to(k.dtype) # [PAGE_SIZE, HEAD_DIM]; cast from fp8 if kv cache is fp8 - - # [HEAD_RATIO_PADDED, HEAD_DIM] @ [HEAD_DIM, PAGE_SIZE] -> [HEAD_RATIO_PADDED, PAGE_SIZE] - attn = tl.dot(q_all, tl.trans(k)) * SM_SCALE - - # Combine validity mask with sliding window mask - if SLIDING_WINDOW > 0: - global_pos = page_idx * PAGE_SIZE + page_offsets - window_mask = global_pos >= first_valid_pos - attn = tl.where(page_mask[None, :] & window_mask[None, :], attn, float("-inf")) - else: - attn = tl.where(page_mask[None, :], attn, float("-inf")) - - # Online softmax update (vectorized over HEAD_RATIO_PADDED) - m_ij = tl.max(attn, axis=1) # [HEAD_RATIO_PADDED] - m_i_new = tl.maximum(m_i, m_ij) - alpha = tl.exp(m_i - m_i_new) - p = tl.exp(attn - m_i_new[:, None]) # [HEAD_RATIO_PADDED, PAGE_SIZE] - - # [HEAD_RATIO_PADDED, PAGE_SIZE] @ [PAGE_SIZE, HEAD_DIM] -> [HEAD_RATIO_PADDED, HEAD_DIM] - acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) - l_i = l_i * alpha + tl.sum(p, axis=1) - m_i = m_i_new - - # Finalize: normalize and compute LSE - l_i_safe = tl.where(l_i == 0.0, 1.0, l_i) - partial_o_val = acc / l_i_safe[:, None] # [HEAD_RATIO_PADDED, HEAD_DIM] - lse_val = m_i + tl.log(l_i_safe) # [HEAD_RATIO_PADDED] - - # Store results for valid HEAD_RATIO Q heads only (masked 2D store) - po_offsets = ( - batch_id * po_stride_batch - + head_ids[:, None] * po_stride_head - + split_id * po_stride_split - + dhead_offsets[None, :] - ) - tl.store(partial_o_ptr + po_offsets, partial_o_val, mask=head_mask[:, None]) - - plse_offsets = ( - batch_id * plse_stride_batch + head_ids * plse_stride_head + split_id * plse_stride_split - ) - tl.store(partial_lse_ptr + plse_offsets, lse_val, mask=head_mask) - - -@triton.jit -def _flash_decode_stage2_kernel( - # Partial results - partial_o_ptr, - partial_lse_ptr, - # Final output - o_ptr, - # Partial output strides: [batch, n_heads, num_splits, head_dim] - po_stride_batch: tl.constexpr, - po_stride_head: tl.constexpr, - po_stride_split: tl.constexpr, - # Partial LSE strides: [batch, n_heads, num_splits] - plse_stride_batch: tl.constexpr, - plse_stride_head: tl.constexpr, - plse_stride_split: tl.constexpr, - # Output strides: [batch, n_heads, head_dim] - o_stride_batch: tl.constexpr, - o_stride_head: tl.constexpr, - # Constants - HEAD_DIM: tl.constexpr, - NUM_SPLITS: tl.constexpr, -): - """ - Each program combines results from all splits for one (batch, head) pair. - """ - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - - dhead_offsets = tl.arange(0, HEAD_DIM) - - # Find global maximum LSE across splits for numerical stability - global_max_lse = float("-inf") - for split_id in range(NUM_SPLITS): - plse_offset = ( - batch_id * plse_stride_batch + head_id * plse_stride_head + split_id * plse_stride_split - ) - lse = tl.load(partial_lse_ptr + plse_offset) - global_max_lse = tl.maximum(global_max_lse, lse) - - # Guard: if all splits had -inf LSE (empty sequence), output zeros - o_offset = batch_id * o_stride_batch + head_id * o_stride_head + dhead_offsets - if global_max_lse == float("-inf"): - tl.store(o_ptr + o_offset, tl.zeros([HEAD_DIM], dtype=tl.float32)) - return - - # Weighted combination: weight_i = exp(lse_i - global_max) - acc = tl.zeros([HEAD_DIM], dtype=tl.float32) - total_weight = 0.0 - - for split_id in range(NUM_SPLITS): - plse_offset = ( - batch_id * plse_stride_batch + head_id * plse_stride_head + split_id * plse_stride_split - ) - lse = tl.load(partial_lse_ptr + plse_offset) - weight = tl.exp(lse - global_max_lse) - - po_base = batch_id * po_stride_batch + head_id * po_stride_head + split_id * po_stride_split - partial_o = tl.load(partial_o_ptr + po_base + dhead_offsets) - - acc += weight * partial_o - total_weight += weight - - # Normalize and store - total_weight = tl.where(total_weight == 0.0, 1.0, total_weight) - o = acc / total_weight - tl.store(o_ptr + o_offset, o) - - -def triton_paged_decode( - q: torch.Tensor, - kv_cache: torch.Tensor, - kv_indices: torch.Tensor, - kv_indptr: torch.Tensor, - kv_last_page_len: torch.Tensor, - sm_scale: float, - sliding_window: Optional[int] = None, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Optimized paged decode with GQA batching + FlashDecoding + page-aligned iteration. - - Args: - q: Query tensor [batch_size, n_heads, head_dim] - kv_cache: Combined cache [num_blocks, 2, n_kv_heads, page_size, head_dim] - kv_indices: Physical page indices (flattened) - kv_indptr: Cumulative page counts [batch_size + 1] - kv_last_page_len: Valid tokens in last page [batch_size] - sm_scale: Softmax scale factor - sliding_window: If set, only attend to the last sliding_window tokens - out: Optional output tensor [batch_size, n_heads, head_dim] - - Returns: - Output tensor [batch_size, n_heads, head_dim] - """ - batch_size, n_heads, head_dim = q.shape - _, _, n_kv_heads, page_size, _ = kv_cache.shape - head_ratio = n_heads // n_kv_heads - head_ratio_padded = max(1, 2 ** math.ceil(math.log2(head_ratio))) if head_ratio > 1 else 1 - - max_pages = kv_indices.shape[0] - max_seq_len = max_pages * page_size - # Normalize sliding_window: None/non-positive → 0 (full attention) - sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 - - output = out if out is not None else torch.empty_like(q) - - if batch_size == 0: - return output - - # Use effective sequence length (capped by sliding window) for split-K heuristic - effective_seq_len = min(max_seq_len, sw) if sw > 0 else max_seq_len - num_splits = _get_num_splits(effective_seq_len, batch_size, n_kv_heads, page_size) - - # Allocate intermediate buffers for split-K - partial_o = torch.empty( - batch_size, - n_heads, - num_splits, - head_dim, - dtype=torch.float32, - device=q.device, - ) - partial_lse = torch.empty( - batch_size, - n_heads, - num_splits, - dtype=torch.float32, - device=q.device, - ) - - # Stage 1: GQA-batched parallel KV processing - _flash_decode_stage1_kernel[(batch_size, n_kv_heads, num_splits)]( - q, - kv_cache, - kv_indices, - kv_indptr, - kv_last_page_len, - partial_o, - partial_lse, - # Q strides - q.stride(0), - q.stride(1), - # Partial output strides - partial_o.stride(0), - partial_o.stride(1), - partial_o.stride(2), - # Partial LSE strides - partial_lse.stride(0), - partial_lse.stride(1), - partial_lse.stride(2), - # Cache strides - kv_cache.stride(0), - kv_cache.stride(1), - kv_cache.stride(2), - kv_cache.stride(3), - # Constants - SM_SCALE=sm_scale, - N_HEADS=n_heads, - N_KV_HEADS=n_kv_heads, - HEAD_DIM=head_dim, - PAGE_SIZE=page_size, - HEAD_RATIO=head_ratio, - HEAD_RATIO_PADDED=head_ratio_padded, - NUM_SPLITS=num_splits, - SLIDING_WINDOW=sw, - ) - - # Stage 2: Combine partial results - _flash_decode_stage2_kernel[(batch_size, n_heads)]( - partial_o, - partial_lse, - output, - # Partial output strides - partial_o.stride(0), - partial_o.stride(1), - partial_o.stride(2), - # Partial LSE strides - partial_lse.stride(0), - partial_lse.stride(1), - partial_lse.stride(2), - # Output strides - output.stride(0), - output.stride(1), - # Constants - HEAD_DIM=head_dim, - NUM_SPLITS=num_splits, - ) - - return output - - -# ============================================================================= -# TRITON KERNELS - CONTEXT/PREFILL (page-aligned, causal skip, autotuned) -# ============================================================================= -@triton.autotune( - configs=[ - triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=2), - triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=4), - triton.Config({"Q_BLOCK": 64}, num_stages=4, num_warps=4), - triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=4), - triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=8), - triton.Config({"Q_BLOCK": 128}, num_stages=3, num_warps=8), - ], - key=["HEAD_DIM", "PAGE_SIZE"], -) -@triton.jit -def _paged_context_kernel( - # Inputs - q_ptr, - kv_cache_ptr, - # Metadata - qo_indptr_ptr, - kv_indptr_ptr, - kv_indices_ptr, - kv_last_page_len_ptr, - seq_len_with_cache_ptr, - # Output - o_ptr, - # Strides - q_stride_token: tl.constexpr, - q_stride_head: tl.constexpr, - o_stride_token: tl.constexpr, - o_stride_head: tl.constexpr, - cache_stride_block: tl.constexpr, - cache_stride_kv: tl.constexpr, - cache_stride_head: tl.constexpr, - cache_stride_token: tl.constexpr, - # Autotuned - Q_BLOCK: tl.constexpr, - # Constants - SM_SCALE: tl.constexpr, - N_HEADS: tl.constexpr, - N_KV_HEADS: tl.constexpr, - HEAD_DIM: tl.constexpr, - PAGE_SIZE: tl.constexpr, - SLIDING_WINDOW: tl.constexpr = 0, -): - """Context/prefill attention with paged KV cache, causal skip, and page-aligned iteration. - - Grid: (num_seq, n_heads, num_q_blocks) - - Optimizations: - - Page-aligned iteration: 1 scalar page table load per page, no div/mod, - contiguous KV memory access within each page. - - Causal skip: pages entirely beyond the last Q position are skipped, - saving ~50% of KV loads on average for causal attention. - - Autotuned Q_BLOCK, num_stages, num_warps for best tile/pipeline config. - """ - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - q_block_id = tl.program_id(axis=2) - - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS - kv_head_id = head_id // HEAD_RATIO - - q_start = tl.load(qo_indptr_ptr + batch_id) - q_end = tl.load(qo_indptr_ptr + batch_id + 1) - q_len = q_end - q_start - - kv_page_start = tl.load(kv_indptr_ptr + batch_id) - kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) - num_kv_pages = kv_page_end - kv_page_start - total_kv_len = tl.load(seq_len_with_cache_ptr + batch_id) - - cache_len = total_kv_len - q_len - - q_block_start = q_block_id * Q_BLOCK - q_offsets = q_block_start + tl.arange(0, Q_BLOCK) - q_mask = q_offsets < q_len - - if tl.sum(q_mask.to(tl.int32)) == 0: - return - - dhead_offsets = tl.arange(0, HEAD_DIM) - q_load_offsets = ( - (q_start + q_offsets[:, None]) * q_stride_token - + head_id * q_stride_head - + dhead_offsets[None, :] - ) - q_load_mask = q_mask[:, None] - q = tl.load(q_ptr + q_load_offsets, mask=q_load_mask, other=0.0) - - acc = tl.zeros([Q_BLOCK, HEAD_DIM], dtype=tl.float32) - m_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - float("inf") - l_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - - page_offsets = tl.arange(0, PAGE_SIZE) - - # Two-phase page loop: - # Phase 1 (full pages): pages entirely before the first Q position need no causal mask. - # First Q position in KV coords = q_block_start + cache_len. - # A page ending at (page_idx+1)*PAGE_SIZE - 1 is fully attended if that's <= first Q pos. - # Phase 2 (boundary pages): remaining pages up to last Q position need causal masking. - first_q_kv_pos = q_block_start + cache_len - max_q_pos = q_block_start + Q_BLOCK - 1 + cache_len - - # Number of full pages (all tokens in these pages are attended by all Q tokens) - num_full_pages = first_q_kv_pos // PAGE_SIZE - - # Sliding window: compute the first page within the window for Phase 1 pruning. - # Each query at position q_pos attends to KV in [q_pos - W + 1, q_pos]. - # The most restrictive query is the first one (q_block_start), so: - # first_valid_pos = max(0, first_q_kv_pos - SLIDING_WINDOW + 1) - if SLIDING_WINDOW > 0: - first_valid_pos = tl.maximum(0, first_q_kv_pos - SLIDING_WINDOW + 1) - first_window_page = first_valid_pos // PAGE_SIZE - else: - first_window_page = 0 - - # Check if this is a full Q block (no q_mask needed) - is_full_q_block = (q_block_start + Q_BLOCK) <= q_len - - # Phase 1: Full pages — no causal mask, no validity mask - # Process one page at a time with a clean inner loop - kv_head_offset = kv_head_id * cache_stride_head - local_kv = page_offsets[:, None] * cache_stride_token + dhead_offsets[None, :] - - for page_idx in range(first_window_page, num_full_pages): - physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) - - # Use int64 to avoid overflow when physical_page * stride > 2^31 - page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset - - # When sliding window is active, the first window page may partially - # overlap the window boundary, requiring per-token masking. - # Use masked loads (like Phase 2) instead of block_ptr loads. - if SLIDING_WINDOW > 0: - k = tl.load( - kv_cache_ptr + page_base + local_kv, - mask=tl.full([PAGE_SIZE, HEAD_DIM], 1, tl.int1), - other=0.0, - ).to(q.dtype) # cast from fp8 if kv cache is fp8 - v = tl.load( - kv_cache_ptr + page_base + local_kv + cache_stride_kv, - mask=tl.full([PAGE_SIZE, HEAD_DIM], 1, tl.int1), - other=0.0, - ).to(q.dtype) # cast from fp8 if kv cache is fp8 - - qk = tl.dot(q, tl.trans(k)) * SM_SCALE - - # Per-query sliding window mask: each query position q_pos - # can attend to KV in [q_pos - W + 1, q_pos]. - kv_positions = page_idx * PAGE_SIZE + page_offsets[None, :] - q_kv_pos = q_offsets[:, None] + cache_len - sw_mask = (q_kv_pos - kv_positions) < SLIDING_WINDOW - full_mask_p1 = q_mask[:, None] & sw_mask - qk = tl.where(full_mask_p1, qk, float("-inf")) - else: - k = tl.load( - kv_cache_ptr + page_base + local_kv, - mask=tl.full([PAGE_SIZE, HEAD_DIM], 1, tl.int1), - other=0.0, - ).to(q.dtype) # cast from fp8 if kv cache is fp8 - v = tl.load( - kv_cache_ptr + page_base + local_kv + cache_stride_kv, - mask=tl.full([PAGE_SIZE, HEAD_DIM], 1, tl.int1), - other=0.0, - ).to(q.dtype) # cast from fp8 if kv cache is fp8 - - qk = tl.dot(q, tl.trans(k)) * SM_SCALE - - if not is_full_q_block: - qk = tl.where(q_mask[:, None], qk, float("-inf")) - - m_ij = tl.max(qk, axis=1) - m_i_new = tl.maximum(m_i, m_ij) - if SLIDING_WINDOW > 0: - # Guard against NaN when m_i == m_i_new == -inf (no valid tokens seen - # yet for a query whose window doesn't overlap this page at all). - alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) - p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) - else: - alpha = tl.exp(m_i - m_i_new) - p = tl.exp(qk - m_i_new[:, None]) - acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) - l_i = l_i * alpha + tl.sum(p, axis=1) - m_i = m_i_new - - # Phase 2: Boundary pages — need causal mask and validity mask - # Pre-compute q_positions outside loop (invariant across pages) - q_positions_2d = q_offsets[:, None] + cache_len - - for page_idx in range(num_full_pages, num_kv_pages): - kv_base_pos = page_idx * PAGE_SIZE - - # Causal skip: if entire page is beyond last Q position, skip it. - if kv_base_pos <= max_q_pos: - physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) - valid_tokens = tl.minimum(PAGE_SIZE, total_kv_len - kv_base_pos) - page_mask = page_offsets < valid_tokens - - # Use int64 to avoid overflow when physical_page * stride > 2^31 - page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset - page_mask_2d = page_mask[:, None] - k = tl.load(kv_cache_ptr + page_base + local_kv, mask=page_mask_2d, other=0.0).to( - q.dtype - ) # cast from fp8 if kv cache is fp8 - v = tl.load( - kv_cache_ptr + page_base + local_kv + cache_stride_kv, - mask=page_mask_2d, - other=0.0, - ).to(q.dtype) # cast from fp8 if kv cache is fp8 - - qk = tl.dot(q, tl.trans(k)) * SM_SCALE - kv_positions = kv_base_pos + page_offsets[None, :] - causal_mask = q_positions_2d >= kv_positions - if SLIDING_WINDOW > 0: - sliding_mask = (q_positions_2d - kv_positions) < SLIDING_WINDOW - full_mask = q_mask[:, None] & causal_mask & sliding_mask & page_mask[None, :] - else: - full_mask = q_mask[:, None] & causal_mask & page_mask[None, :] - qk = tl.where(full_mask, qk, float("-inf")) - - m_ij = tl.max(qk, axis=1) - m_i_new = tl.maximum(m_i, m_ij) - if SLIDING_WINDOW > 0: - alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) - p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) - else: - alpha = tl.exp(m_i - m_i_new) - p = tl.exp(qk - m_i_new[:, None]) - acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) - l_i = l_i * alpha + tl.sum(p, axis=1) - m_i = m_i_new - - l_i = tl.where(l_i == 0.0, 1.0, l_i) - o = acc / l_i[:, None] - o_store_offsets = ( - (q_start + q_offsets[:, None]) * o_stride_token - + head_id * o_stride_head - + dhead_offsets[None, :] - ) - tl.store(o_ptr + o_store_offsets, o, mask=q_load_mask) - - -@triton.autotune( - configs=[ - triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=2), - triton.Config({"Q_BLOCK": 64}, num_stages=2, num_warps=4), - triton.Config({"Q_BLOCK": 64}, num_stages=4, num_warps=4), - triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=4), - triton.Config({"Q_BLOCK": 128}, num_stages=2, num_warps=8), - triton.Config({"Q_BLOCK": 128}, num_stages=3, num_warps=8), - ], - key=["HEAD_DIM", "PAGE_SIZE"], -) -@triton.jit -def _paged_context_masked_kernel( - # Inputs - q_ptr, - kv_cache_ptr, - custom_mask_ptr, - # Metadata - qo_indptr_ptr, - kv_indptr_ptr, - kv_indices_ptr, - seq_len_with_cache_ptr, - # Output - o_ptr, - # Strides - q_stride_token: tl.constexpr, - q_stride_head: tl.constexpr, - o_stride_token: tl.constexpr, - o_stride_head: tl.constexpr, - cache_stride_block: tl.constexpr, - cache_stride_kv: tl.constexpr, - cache_stride_head: tl.constexpr, - cache_stride_token: tl.constexpr, - mask_stride_batch: tl.constexpr, - mask_stride_head: tl.constexpr, - mask_stride_q: tl.constexpr, - mask_stride_k: tl.constexpr, - # Autotuned - Q_BLOCK: tl.constexpr, - # Constants - SM_SCALE: tl.constexpr, - N_HEADS: tl.constexpr, - N_KV_HEADS: tl.constexpr, - HEAD_DIM: tl.constexpr, - PAGE_SIZE: tl.constexpr, - SLIDING_WINDOW: tl.constexpr = 0, -): - """Context/prefill attention with paged KV cache and a backend-provided bool allow-mask.""" - batch_id = tl.program_id(axis=0) - head_id = tl.program_id(axis=1) - q_block_id = tl.program_id(axis=2) - - HEAD_RATIO: tl.constexpr = N_HEADS // N_KV_HEADS - kv_head_id = head_id // HEAD_RATIO - - q_start = tl.load(qo_indptr_ptr + batch_id) - q_end = tl.load(qo_indptr_ptr + batch_id + 1) - q_len = q_end - q_start - - kv_page_start = tl.load(kv_indptr_ptr + batch_id) - kv_page_end = tl.load(kv_indptr_ptr + batch_id + 1) - num_kv_pages = kv_page_end - kv_page_start - total_kv_len = tl.load(seq_len_with_cache_ptr + batch_id) - - q_block_start = q_block_id * Q_BLOCK - q_offsets = q_block_start + tl.arange(0, Q_BLOCK) - q_mask = q_offsets < q_len - if tl.sum(q_mask.to(tl.int32)) == 0: - return - - dhead_offsets = tl.arange(0, HEAD_DIM) - q_load_offsets = ( - (q_start + q_offsets[:, None]) * q_stride_token - + head_id * q_stride_head - + dhead_offsets[None, :] - ) - q_load_mask = q_mask[:, None] - q = tl.load(q_ptr + q_load_offsets, mask=q_load_mask, other=0.0) - - acc = tl.zeros([Q_BLOCK, HEAD_DIM], dtype=tl.float32) - m_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - float("inf") - l_i = tl.zeros([Q_BLOCK], dtype=tl.float32) - - page_offsets = tl.arange(0, PAGE_SIZE) - kv_head_offset = kv_head_id * cache_stride_head - local_kv = page_offsets[:, None] * cache_stride_token + dhead_offsets[None, :] - mask_batch_offset = batch_id * mask_stride_batch - # The mask is broadcast over heads (head dim == 1), so the head offset is always 0. - mask_head_offset = 0 * mask_stride_head - - for page_idx in range(num_kv_pages): - kv_base_pos = page_idx * PAGE_SIZE - physical_page = tl.load(kv_indices_ptr + kv_page_start + page_idx) - valid_tokens = tl.minimum(PAGE_SIZE, total_kv_len - kv_base_pos) - page_mask = page_offsets < valid_tokens - - page_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_offset - page_mask_2d = page_mask[:, None] - k = tl.load(kv_cache_ptr + page_base + local_kv, mask=page_mask_2d, other=0.0) - v = tl.load( - kv_cache_ptr + page_base + local_kv + cache_stride_kv, - mask=page_mask_2d, - other=0.0, - ) - - qk = tl.dot(q, tl.trans(k)) * SM_SCALE - kv_positions = kv_base_pos + page_offsets[None, :] - mask_offsets = ( - mask_batch_offset - + mask_head_offset - + q_offsets[:, None] * mask_stride_q - + kv_positions * mask_stride_k - ) - valid_mask = q_mask[:, None] & page_mask[None, :] - custom_mask = tl.load(custom_mask_ptr + mask_offsets, mask=valid_mask, other=0) - if SLIDING_WINDOW > 0: - cache_len = total_kv_len - q_len - query_positions = q_offsets[:, None] + cache_len - sliding_mask = (query_positions >= kv_positions) & ( - (query_positions - kv_positions) < SLIDING_WINDOW - ) - full_mask = valid_mask & sliding_mask & (custom_mask != 0) - else: - full_mask = valid_mask & (custom_mask != 0) - qk = tl.where(full_mask, qk, float("-inf")) - - m_ij = tl.max(qk, axis=1) - m_i_new = tl.maximum(m_i, m_ij) - if SLIDING_WINDOW > 0: - alpha = tl.where(m_i > float("-inf"), tl.exp(m_i - m_i_new), 0.0) - p = tl.where(m_i_new[:, None] > float("-inf"), tl.exp(qk - m_i_new[:, None]), 0.0) - else: - alpha = tl.exp(m_i - m_i_new) - p = tl.exp(qk - m_i_new[:, None]) - acc = tl.dot(p.to(v.dtype), v, acc=acc * alpha[:, None]) - l_i = l_i * alpha + tl.sum(p, axis=1) - m_i = m_i_new - - l_i = tl.where(l_i == 0.0, 1.0, l_i) - o = acc / l_i[:, None] - o_store_offsets = ( - (q_start + q_offsets[:, None]) * o_stride_token - + head_id * o_stride_head - + dhead_offsets[None, :] - ) - tl.store(o_ptr + o_store_offsets, o, mask=q_load_mask) - - -@triton.jit -def _fast_gather_sdpa_kernel( - kv_cache_ptr, - kv_indices_ptr, - kv_indptr_ptr, - seq_len_with_cache_ptr, - out_k_ptr, - out_v_ptr, - # Strides - cache_stride_block: tl.constexpr, - cache_stride_kv: tl.constexpr, - cache_stride_head: tl.constexpr, - cache_stride_token: tl.constexpr, - out_stride_seq: tl.constexpr, - out_stride_head: tl.constexpr, - out_stride_token: tl.constexpr, - # Constants - MAX_PAGES: tl.constexpr, - N_KV_HEADS: tl.constexpr, - PAGE_SIZE: tl.constexpr, - HEAD_DIM: tl.constexpr, -): - """Gather scattered pages into separate K, V buffers in SDPA layout. - - Grid: (total_pages, N_KV_HEADS) - Each program copies one page for one KV head into contiguous K and V - outputs shaped [num_seq, n_kv_heads, max_kv_len, head_dim]. - No precomputed mapping needed — seq_id and local_page computed from global index. - """ - page_global_idx = tl.program_id(0) - kv_head_id = tl.program_id(1) - - # Compute seq_id and local_page from global page index - seq_id = page_global_idx // MAX_PAGES - local_page = page_global_idx % MAX_PAGES - - seq_page_start = tl.load(kv_indptr_ptr + seq_id) - seq_page_end = tl.load(kv_indptr_ptr + seq_id + 1) - seq_pages = seq_page_end - seq_page_start - page_is_valid = local_page < seq_pages - - physical_page = tl.load( - kv_indices_ptr + seq_page_start + local_page, mask=page_is_valid, other=0 - ) - - token_offsets = tl.arange(0, PAGE_SIZE) - head_offsets = tl.arange(0, HEAD_DIM) - - # Source: kv_cache[physical_page, 0/1, kv_head_id, :, :] - src_base = physical_page.to(tl.int64) * cache_stride_block + kv_head_id * cache_stride_head - src_offsets = token_offsets[:, None] * cache_stride_token + head_offsets[None, :] - - # Destination: out_k/v[seq_id, kv_head_id, local_page*PAGE_SIZE + :, :] - local_token_start = local_page * PAGE_SIZE - seq_len = tl.load(seq_len_with_cache_ptr + seq_id) - valid_tokens = tl.minimum(PAGE_SIZE, seq_len - local_token_start) - valid_tokens = tl.maximum(0, valid_tokens) - token_is_valid = token_offsets < valid_tokens - load_mask = page_is_valid & token_is_valid[:, None] - - k_data = tl.load(kv_cache_ptr + src_base + src_offsets, mask=load_mask, other=0.0) - v_data = tl.load( - kv_cache_ptr + src_base + cache_stride_kv + src_offsets, mask=load_mask, other=0.0 - ) - - dst_base = ( - seq_id * out_stride_seq - + kv_head_id * out_stride_head - + (local_token_start + token_offsets[:, None]) * out_stride_token - + head_offsets[None, :] - ) - - tl.store(out_k_ptr + dst_base, k_data) - tl.store(out_v_ptr + dst_base, v_data) - - -def triton_paged_context( - q: torch.Tensor, - kv_cache: torch.Tensor, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - kv_last_page_len: torch.Tensor, - seq_len_with_cache: torch.Tensor, - sm_scale: float, - sliding_window: Optional[int] = None, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Context/prefill attention with paged KV cache.""" - total_tokens, n_heads, head_dim = q.shape - _, _, n_kv_heads, page_size, _ = kv_cache.shape - num_seq = qo_indptr.shape[0] - 1 - - output = out if out is not None else torch.empty_like(q) - - if num_seq == 0 or total_tokens == 0: - return output - - q_lens = qo_indptr[1:] - qo_indptr[:-1] - - # Compute max_q_len without GPU sync for single-sequence batches (most common - # in serving). For multi-sequence batches, we must use .item() because - # total_tokens // num_seq gives the average, not the max — variable-length - # sequences can produce wrong results (under-launched Q blocks or wrong SDPA reshape). - if num_seq == 1: - max_q_len = total_tokens - else: - max_q_len = int(q_lens.max().item()) - - # Adaptive dispatch: gather + cuDNN SDPA for seq>=512 (outperforms paged kernel), - # paged Triton kernel for shorter sequences where gather overhead dominates. - # Normalize sliding_window for kernel constexpr: None/non-positive → 0 - sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 - - # Force SDPA for large head_dim: the Triton paged kernel's tl.dot produces - # misaligned shared memory accesses on Blackwell when HEAD_DIM > 256. - large_head_dim = head_dim > 256 - # SDPA reshape requires all sequences to have the same q_len (since q is - # packed as [total_tokens, ...] and we reshape to [num_seq, max_q_len, ...]). - # Check without GPU sync: sum(q_len_i) == num_seq * max_q_len iff all equal. - all_same_q_len = total_tokens == num_seq * max_q_len - use_sdpa_candidate = ( - (max_q_len >= 512 or large_head_dim) - and (num_seq <= 64 or large_head_dim) - and all_same_q_len - and sw == 0 # SDPA doesn't support sliding window natively - ) - kv_lens = seq_len_with_cache[:num_seq] - cache_lens = kv_lens - q_lens - - max_kv_len = int(kv_lens.max().item()) if use_sdpa_candidate else 0 - max_pages = (max_kv_len + page_size - 1) // page_size - total_expected_pages = num_seq * max_pages - use_sdpa = use_sdpa_candidate and max_pages > 0 - - if use_sdpa: - # Fast Triton gather: scattered pages → separate K, V in SDPA layout - # Single alloc for both K and V, single kernel to fill - padded_kv_len = max_pages * page_size - kv_buf = torch.empty( - 2, - num_seq, - n_kv_heads, - padded_kv_len, - head_dim, - dtype=kv_cache.dtype, - device=kv_cache.device, - ) - k_buf = kv_buf[0] - v_buf = kv_buf[1] - _fast_gather_sdpa_kernel[(total_expected_pages, n_kv_heads)]( - kv_cache, - kv_indices, - kv_indptr, - seq_len_with_cache, - k_buf, - v_buf, - kv_cache.stride(0), - kv_cache.stride(1), - kv_cache.stride(2), - kv_cache.stride(3), - k_buf.stride(0), - k_buf.stride(1), - k_buf.stride(2), - MAX_PAGES=max_pages, - N_KV_HEADS=n_kv_heads, - PAGE_SIZE=page_size, - HEAD_DIM=head_dim, - ) - k_sdpa = k_buf[:, :, :max_kv_len, :] - v_sdpa = v_buf[:, :, :max_kv_len, :] - - # Cast k/v to query dtype if kv cache uses a different dtype (e.g., fp8) - if kv_cache.dtype != q.dtype: - k_sdpa = k_sdpa.to(q.dtype) - v_sdpa = v_sdpa.to(q.dtype) - - q_positions = torch.arange(max_q_len, device=q.device, dtype=cache_lens.dtype) - kv_positions = torch.arange(max_kv_len, device=q.device, dtype=kv_lens.dtype) - attn_mask = kv_positions.view(1, 1, 1, max_kv_len) < kv_lens.view(num_seq, 1, 1, 1) - causal_mask = kv_positions.view(1, 1, 1, max_kv_len) <= ( - cache_lens.view(num_seq, 1, 1, 1) + q_positions.view(1, 1, max_q_len, 1) - ) - attn_mask = attn_mask & causal_mask - - # SDPA with GQA - o_sdpa = torch.nn.functional.scaled_dot_product_attention( - q.view(num_seq, max_q_len, n_heads, head_dim).transpose(1, 2), - k_sdpa, - v_sdpa, - scale=sm_scale, - attn_mask=attn_mask, - is_causal=False, - enable_gqa=True, - ) - output.view(num_seq, max_q_len, n_heads, head_dim).copy_(o_sdpa.permute(0, 2, 1, 3)) - else: - # Use paged kernel (better for small workloads) - def grid_paged(meta): - q_block = meta["Q_BLOCK"] - num_q_blocks = (max_q_len + q_block - 1) // q_block - return (num_seq, n_heads, num_q_blocks) - - _paged_context_kernel[grid_paged]( - q, - kv_cache, - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_len, - seq_len_with_cache, - output, - q.stride(0), - q.stride(1), - output.stride(0), - output.stride(1), - kv_cache.stride(0), - kv_cache.stride(1), - kv_cache.stride(2), - kv_cache.stride(3), - SM_SCALE=sm_scale, - N_HEADS=n_heads, - N_KV_HEADS=n_kv_heads, - HEAD_DIM=head_dim, - PAGE_SIZE=page_size, - SLIDING_WINDOW=sw, - ) - - return output - - -def triton_paged_context_with_custom_mask( - q: torch.Tensor, - kv_cache: torch.Tensor, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - seq_len_with_cache: torch.Tensor, - custom_attn_mask: torch.Tensor, - sm_scale: float, - sliding_window: Optional[int] = None, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Context/prefill attention with paged KV cache and a backend-provided bool allow-mask. - - The mask must be broadcastable over heads, i.e. shape ``[B, 1, S_q, S_k]``. - Per-head masks (``N > 1`` in the head dimension) are **not** supported. - """ - output = out if out is not None else torch.empty_like(q) - num_seq = qo_indptr.shape[0] - 1 - total_tokens, n_heads, head_dim = q.shape - _, _, n_kv_heads, page_size, _ = kv_cache.shape - - if num_seq == 0 or total_tokens == 0: - return output - - assert custom_attn_mask.shape[1] == 1, ( - f"Per-head masks are not supported; expected mask head dim == 1, " - f"got {custom_attn_mask.shape[1]}. Mask shape: {custom_attn_mask.shape}" - ) - - if num_seq == 1: - max_q_len = total_tokens - else: - q_lens = qo_indptr[1:] - qo_indptr[:-1] - max_q_len = int(q_lens.max().item()) - - if not custom_attn_mask.is_contiguous() or custom_attn_mask.dtype != torch.uint8: - custom_attn_mask = custom_attn_mask.contiguous().to(dtype=torch.uint8) - - sw = sliding_window if isinstance(sliding_window, int) and sliding_window > 0 else 0 - - def grid_masked(meta): - q_block = meta["Q_BLOCK"] - num_q_blocks = (max_q_len + q_block - 1) // q_block - return (num_seq, n_heads, num_q_blocks) - - _paged_context_masked_kernel[grid_masked]( - q, - kv_cache, - custom_attn_mask, - qo_indptr, - kv_indptr, - kv_indices, - seq_len_with_cache, - output, - q.stride(0), - q.stride(1), - output.stride(0), - output.stride(1), - kv_cache.stride(0), - kv_cache.stride(1), - kv_cache.stride(2), - kv_cache.stride(3), - custom_attn_mask.stride(0), - custom_attn_mask.stride(1), - custom_attn_mask.stride(2), - custom_attn_mask.stride(3), - SM_SCALE=sm_scale, - N_HEADS=n_heads, - N_KV_HEADS=n_kv_heads, - HEAD_DIM=head_dim, - PAGE_SIZE=page_size, - SLIDING_WINDOW=sw, - ) - - return output - - -@torch.library.custom_op("auto_deploy::triton_paged_prepare_metadata", mutates_args=()) -def prepare_triton_paged_metadata( - position_ids: torch.Tensor, - batch_info_host: torch.Tensor, - cu_seqlen: torch.Tensor, - seq_len_with_cache: torch.Tensor, -) -> List[torch.Tensor]: - """Prepare metadata for Triton paged attention.""" - from ..attention_interface import BatchInfo - - batch_info = BatchInfo(batch_info_host) - num_prefill, num_prefill_tokens, num_decode = batch_info.get_absorbed_info() - num_seq = num_prefill + num_decode - num_tokens = num_prefill_tokens + num_decode - - qo_indptr = cu_seqlen[: num_seq + 1] - - batch_indices, positions = flashinfer.get_batch_indices_positions( - qo_indptr, seq_len_with_cache[:num_seq], num_tokens - ) - - return batch_indices, positions - - -@prepare_triton_paged_metadata.register_fake -def prepare_triton_paged_metadata_fake( - position_ids: torch.Tensor, - batch_info_host: torch.Tensor, - cu_seqlen: torch.Tensor, - seq_len_with_cache: torch.Tensor, -): - num_tokens = position_ids.shape[0] * position_ids.shape[1] - return ( - torch.empty(num_tokens, dtype=torch.int32, device=position_ids.device), - torch.empty(num_tokens, dtype=torch.int32, device=position_ids.device), - ) - - -@torch.library.custom_op("auto_deploy::triton_paged_mha_with_cache", mutates_args=("kv_cache",)) -def triton_paged_mha_with_cache( - # Q, K, V - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - # STANDARD METADATA - batch_info_host: torch.Tensor, - cu_seqlen_host: torch.Tensor, - cu_num_pages: torch.Tensor, - cu_num_pages_host: torch.Tensor, - cache_loc: torch.Tensor, - last_page_len: torch.Tensor, - last_page_len_host: torch.Tensor, - seq_len_with_cache_host: torch.Tensor, - # EXTRA METADATA - triton_batch_indices: torch.Tensor, - triton_positions: torch.Tensor, - # CACHES - combined KV cache - kv_cache: torch.Tensor, - # CONSTANTS - scale: Optional[float] = None, - sliding_window: Optional[int] = None, - # OPTIONAL INPUTS - read_cache_only: bool = False, - custom_attn_mask: Optional[torch.Tensor] = None, - # OPTIONAL PRE-ALLOCATED OUTPUT - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Triton paged attention with mixed batch support.""" - head_dim = kv_cache.shape[-1] - q_shape_og = q.shape - b, s = q_shape_og[:2] - - q = q.reshape(b * s, -1, head_dim).contiguous() - k = k.reshape(b * s, -1, head_dim).contiguous() - v = v.reshape(b * s, -1, head_dim).contiguous() - - from ..attention_interface import BatchInfo - - batch_info = BatchInfo(batch_info_host) - num_prefill, num_prefill_tokens, num_decode = batch_info.get_absorbed_info() - num_seq = num_prefill + num_decode - num_total_tokens = num_prefill_tokens + num_decode - - sm_scale = _get_sm_scale(head_dim, scale) - - if not read_cache_only: - update_paged_kv_cache( - k[:num_total_tokens], - v[:num_total_tokens], - triton_batch_indices[:num_total_tokens], - triton_positions[:num_total_tokens], - kv_cache, - cache_loc, - cu_num_pages[: num_seq + 1], - ) - - if out is not None: - y = out.view(-1, q.shape[1], head_dim) - else: - y = torch.empty_like(q) - - # Process prefill tokens if any - if num_prefill > 0: - cu_seqlen = cu_seqlen_host[: num_prefill + 1].to(q.device, non_blocking=True) - seq_len_with_cache = seq_len_with_cache_host[:num_prefill].to(q.device, non_blocking=True) - has_custom_attn_mask = custom_attn_mask is not None and custom_attn_mask.numel() > 0 - if has_custom_attn_mask: - triton_paged_context_with_custom_mask( - q[:num_prefill_tokens], - kv_cache, - cu_seqlen, - cu_num_pages[: num_prefill + 1], - cache_loc, - seq_len_with_cache, - custom_attn_mask[:num_prefill], - sm_scale, - sliding_window=sliding_window, - out=y[:num_prefill_tokens], - ) - else: - triton_paged_context( - q[:num_prefill_tokens], - kv_cache, - cu_seqlen, - cu_num_pages[: num_prefill + 1], - cache_loc, - last_page_len[:num_prefill], - seq_len_with_cache, - sm_scale, - sliding_window=sliding_window, - out=y[:num_prefill_tokens], - ) - - # Process decode tokens if any - if num_decode > 0: - triton_paged_decode( - q[num_prefill_tokens:num_total_tokens], - kv_cache, - cache_loc, - cu_num_pages[num_prefill : num_seq + 1], - last_page_len[num_prefill:num_seq], - sm_scale, - sliding_window=sliding_window, - out=y[num_prefill_tokens:num_total_tokens], - ) - - if out is not None: - # Zero stale data in padding region for CUDA graph replay stability - bs = b * s - if num_total_tokens < bs: - y[num_total_tokens:].zero_() - # Return a 0-element dummy to satisfy PyTorch's no-alias constraint. - # The caller (DynamicOpWrapper._coalesce_output) picks ``out`` over - # this dummy, so the pre-allocated buffer is used downstream. - return out.new_empty(0) - - return y.view(q_shape_og) - - -@triton_paged_mha_with_cache.register_fake -def triton_paged_mha_with_cache_fake( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - batch_info_host: torch.Tensor, - cu_seqlen_host: torch.Tensor, - cu_num_pages: torch.Tensor, - cu_num_pages_host: torch.Tensor, - cache_loc: torch.Tensor, - last_page_len: torch.Tensor, - last_page_len_host: torch.Tensor, - seq_len_with_cache_host: torch.Tensor, - triton_batch_indices: torch.Tensor, - triton_positions: torch.Tensor, - kv_cache: torch.Tensor, - scale: Optional[float] = None, - sliding_window: Optional[int] = None, - read_cache_only: bool = False, - custom_attn_mask: Optional[torch.Tensor] = None, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - if out is not None: - return out.new_empty(0) - return torch.empty_like(q.contiguous()) - - -@AttentionRegistry.register("triton_paged") -class TritonPagedAttention(AttentionDescriptor): - """Descriptor for Triton Paged Attention backend. - - Optimized with GQA head batching, FlashDecoding, and page-aligned iteration. - """ - - @classmethod - def get_attention_layout(cls) -> AttentionLayout: - return "bsnd" - - @classmethod - def get_num_qkv_args(cls) -> int: - return 3 - - @classmethod - def get_source_attention_op(cls) -> OpOverloadPacket: - return torch.ops.auto_deploy.torch_attention - - @classmethod - def get_cached_attention_op(cls) -> MHACallable: - return torch.ops.auto_deploy.triton_paged_mha_with_cache.default - - @classmethod - def supports_shared_kv(cls) -> bool: - return True - - @classmethod - def get_standard_metadata_args(cls) -> List[str]: - return [ - "batch_info_host", - "cu_seqlen_host", - "cu_num_pages", - "cu_num_pages_host", - "cache_loc", - "last_page_len", - "last_page_len_host", - "seq_len_with_cache_host", - ] - - @classmethod - def get_prepare_extra_metadata_info( - cls, any_source_attn_node: Node - ) -> Tuple[Optional[PrepareMetadataCallable], int, List[Constant]]: - return ( - torch.ops.auto_deploy.triton_paged_prepare_metadata.default, - 2, - [], - ) - - @classmethod - def get_cache_initializers( - cls, source_attn_node: Node, cache_config: KvCacheConfig - ) -> ResourceHandlerDict: - """Build the per-layer KV handler used by the kvcache transform.""" - k_fake: FakeTensor = source_attn_node.args[1].meta["val"] - num_kv_heads = k_fake.shape[2] - head_dim = k_fake.shape[3] - # ``sliding_window`` is propagated into the handler so layers - # with different windows land in separate pools. - (sw,) = extract_op_args(source_attn_node, "sliding_window") - sliding_window = sw if isinstance(sw, int) and sw > 0 else 0 - - return { - "kv_cache": KVPagedResourceHandler( - num_kv_heads, - head_dim, - dtype=cls.resolve_cache_dtype(cache_config.dtype, k_fake.dtype), - kv_factor=2, - kv_layout=KV_LAYOUT, - sliding_window=sliding_window, - ) - } - - @classmethod - def get_constants(cls, source_attn_node: Node) -> List[Constant]: - layout, scale, _attn_mask, dropout_p, _is_causal = extract_op_args( - source_attn_node, "layout", "scale", "attn_mask", "dropout_p", "is_causal" - ) - - if layout != "bsnd": - raise RuntimeError( - f"Expected torch_attention layout='bsnd' but got {layout!r} " - f"for node: {source_attn_node.format_node()}" - ) - - if dropout_p != 0.0: - ad_logger.debug( - f"Unsupported attention arguments for {source_attn_node=}: {dropout_p=}" - ) - - if not (isinstance(scale, float) or scale is None): - ad_logger.warning(f"Provided {scale=}, is not a float. Using default scale instead.") - scale = None - - sliding_window = extract_op_args(source_attn_node, "sliding_window")[0] - - return [ - scale, - sliding_window, - cls.get_shared_kv_source_layer_idx(source_attn_node) is not None, - ] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py index 934d951d374e..5bbfc9e30ecf 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_gemma4.py @@ -250,7 +250,7 @@ def _gemma4_prepare_multimodal_mask_fake( torch.ops.auto_deploy.gemma4_multimodal_mask, "torch", _GEMMA4_MASK_SPEC ) SemanticMaskRegistry.register( - torch.ops.auto_deploy.gemma4_multimodal_mask, "triton_paged", _GEMMA4_MASK_SPEC + torch.ops.auto_deploy.gemma4_multimodal_mask, "triton", _GEMMA4_MASK_SPEC ) diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py index 79afb6f6ace4..f131d2d6292e 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py @@ -620,14 +620,6 @@ def _apply( class InsertCachedAttention(_InsertCachedOperator): """A transform to insert cached attention into the graph module.""" - def _apply(self, *args, **kwargs): - if self.config.backend == "triton": - self._log_warning( - "'triton' backend only supports GREEDY sampling (top_k=1). " - "Please set top_k=1 in the sampling parameters to ensure cohesive output." - ) - return super()._apply(*args, **kwargs) - @TransformRegistry.register("insert_cached_mla_attention") class InsertCachedMLAAttention(_InsertCachedOperator): diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index 4490dbf76afa..5c3d26f2e79b 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -222,7 +222,7 @@ class TestLlama3_1_8B(LlmapiAccuracyTestHarness): "max_seq_len": 2048, "compile_backend": "torch-simple", }, - "triton_paged": { + "triton": { "max_batch_size": 128, "max_seq_len": 8192, "compile_backend": "torch-cudagraph", @@ -294,7 +294,7 @@ def get_default_sampling_params(self): # For batch_size=32: 8.25 GiB KV + ~15 GiB weights ~= 23.3 GiB. # If batch size is increased, this parameterization must be gated at a higher memory threshold. "torch", - "triton_paged", + "triton", ], ) def test_auto_dtype(self, world_size, enable_chunked_prefill, attn_backend): diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 785e65d22078..e6a923f5d0e9 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -108,7 +108,7 @@ accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_auto_dtype[trtllm-True] accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_nvfp4[False] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_attention_dp[4] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[torch-True-1] -accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1] +accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-4] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-True-1] diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 1d1ebe9e2ae2..e82c4ee7b109 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -330,7 +330,7 @@ l0_b200: # SM100+-gated). HW-agnostic tests (compile, models, shim, utils, plus most # custom_ops, smoke, transformations files) and pure-FP8 tests are covered # on Hopper (l0_h100.yml) and not duplicated here. - - unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention.py::TestSDPADispatch + - unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py::TestSDPADispatch - unittest/auto_deploy/singlegpu/custom_ops/moe/test_ad_moe_op.py - unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py - unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py @@ -357,7 +357,7 @@ l0_b200: stage: post_merge backend: autodeploy tests: - - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1] + - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1] - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[torch-True-1] - accuracy/test_llm_api_autodeploy.py::TestGLM4Flash::test_nvfp4[False] - accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index acf3f8e19f1c..c61a9f1ed392 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -506,7 +506,7 @@ l0_h100: - accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it - accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True] - accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-trtllm] - - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton_paged-False-1] + - accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1] - examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph] - examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple] - examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_attention_op.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_attention_op.py deleted file mode 100644 index 90193f6459c0..000000000000 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_attention_op.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pytest -import torch -from torch_attention_reference import TorchAttentionReference - -import tensorrt_llm._torch.auto_deploy # noqa: F401 -from tensorrt_llm._torch.auto_deploy.custom_ops.attention_interface import BatchInfo - - -@pytest.mark.parametrize("num_generate_ratio", [0.0, 0.5, 1.0]) -@pytest.mark.parametrize("max_seq_len", [0, 1, 16]) -@pytest.mark.parametrize("group_size", [1, 4]) -@pytest.mark.parametrize("n_heads", [8]) -@pytest.mark.parametrize("batch_size", [1, 16]) -@pytest.mark.parametrize("dtype", ["float16", "float32"]) -@pytest.mark.parametrize("device", ["cuda"]) -def test_flat_gqa_op( - device, dtype, batch_size, n_heads, group_size, max_seq_len, num_generate_ratio -): - n_heads = n_heads - n_kv_heads = n_heads // group_size - D_HEAD = 16 - dtype = getattr(torch, dtype) - int_kwargs = {"device": device, "dtype": torch.int32} - dtype_kwargs = {"device": device, "dtype": dtype} - - # setup caches with 2*batch_size, 2*max_seq_len since we also randomize input_pos - cache_max_seq_len = 2 * (max_seq_len + 1) - cache_max_batch_size = 2 * batch_size - cache_size = (cache_max_batch_size, cache_max_seq_len, n_kv_heads, D_HEAD) - cache_loc = torch.randperm(cache_max_batch_size, **int_kwargs)[:batch_size] - - k_cache = torch.randn(cache_size, **dtype_kwargs) - v_cache = torch.randn(cache_size, **dtype_kwargs) - - # randomize num_context vs num_generate; NOTE: we can use context kernel for generate as well - num_generate = torch.tensor(num_generate_ratio * batch_size, **int_kwargs) - num_context = batch_size - num_generate - - # construct random input_positions - input_positions = torch.randint(0, max_seq_len + 1, (batch_size,), **int_kwargs) - - # construct seq_len, seq_start; - seq_len = torch.cat( - [ - torch.randint(0, max_seq_len + 1, (num_context,), **int_kwargs), # context - torch.zeros(num_generate, **int_kwargs) + (max_seq_len > 0), # generate - ] - ) - seq_start = seq_len.cumsum(0) - seq_len - - # get fake input - q = torch.randn(1, seq_len.sum(), n_heads * D_HEAD, **dtype_kwargs) - k = torch.randn(1, seq_len.sum(), n_kv_heads * D_HEAD, **dtype_kwargs) - v = torch.randn(1, seq_len.sum(), n_kv_heads * D_HEAD, **dtype_kwargs) - - num_prefill_tokens = seq_len[:num_context].sum() - nc = int(num_context.item()) if isinstance(num_context, torch.Tensor) else num_context - ng = int(num_generate.item()) if isinstance(num_generate, torch.Tensor) else num_generate - npt = ( - int(num_prefill_tokens.item()) - if isinstance(num_prefill_tokens, torch.Tensor) - else num_prefill_tokens - ) - _bi = BatchInfo() - _bi.update([nc, npt, 0, 0, ng, ng]) - batch_info_host = _bi.serialize() - - # run op - output = torch.ops.auto_deploy.triton_attention_flattened_mha_with_cache( - # Q, K, V - q, - k, - v, - # STANDARD METADATA - batch_info_host, - seq_len, - input_positions, - cache_loc, - seq_start, # cu_seqlen - # CACHES - k_cache, - v_cache, - # CONSTANTS - scale=None, - ) - - # Use torch backend as clean reference - ref_flat = TorchAttentionReference.flattened_mha_with_cache( - q, k, v, batch_info_host, seq_len, input_positions, cache_loc, seq_start, k_cache, v_cache - ) - - assert torch.allclose( - ref_flat.cpu().to(torch.float32), - output.cpu().to(torch.float32), - atol=1e-2, - rtol=1e-2, - ) diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py similarity index 90% rename from tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention.py rename to tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py index 580c1ca8a004..047983c3d8f2 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for Triton Paged Attention. +"""Unit tests for Triton Attention. -Tests the Triton paged attention kernels and compares against FlashInfer for correctness. +Tests the Triton attention kernels and compares against FlashInfer for correctness. """ import math @@ -31,10 +31,10 @@ @torch.inference_mode() -def test_gemma4_prepare_multimodal_mask_can_drive_triton_paged_custom_mask(): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - prepare_triton_paged_metadata, - triton_paged_mha_with_cache, +def test_gemma4_prepare_multimodal_mask_can_drive_triton_custom_mask(): + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + prepare_triton_metadata, + triton_mha_with_cache, ) n_heads, n_kv_heads, head_dim, page_size = 8, 2, 128, 16 @@ -61,7 +61,7 @@ def test_gemma4_prepare_multimodal_mask_can_drive_triton_paged_custom_mask(): ) position_ids = torch.arange(seq_len, device="cuda") - batch_indices, positions = prepare_triton_paged_metadata( + batch_indices, positions = prepare_triton_metadata( position_ids, batch_info_host, cu_seqlen_host.to("cuda", non_blocking=True), @@ -77,7 +77,7 @@ def test_gemma4_prepare_multimodal_mask_can_drive_triton_paged_custom_mask(): torch.tensor([0, 2], dtype=torch.int32), ).to("cuda") - output = triton_paged_mha_with_cache( + output = triton_mha_with_cache( q, k, v, @@ -145,7 +145,7 @@ def create_page_table( return kv_indices, kv_indptr, kv_last_page_len -class TestTritonPagedDecodeKernel: +class TestTritonDecodeKernel: """Tests for the FlashDecoding paged decode kernel (stage1 + stage2).""" @pytest.mark.parametrize("batch_size", [1, 4, 8]) @@ -156,8 +156,8 @@ def test_decode_kernel_vs_pytorch_reference( self, batch_size: int, n_heads: int, n_kv_heads: int, head_dim: int, seq_len: int ): """Test decode kernel against PyTorch SDPA reference.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_decode, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_decode, update_paged_kv_cache, ) @@ -217,7 +217,7 @@ def test_decode_kernel_vs_pytorch_reference( sm_scale = 1.0 / math.sqrt(head_dim) # Run Triton kernel - output_triton = triton_paged_decode( + output_triton = triton_decode( q, kv_cache, kv_indices, kv_indptr, kv_last_page_len, sm_scale ) @@ -243,8 +243,54 @@ def test_decode_kernel_vs_pytorch_reference( # Compare torch.testing.assert_close(output_triton.float(), output_ref.float(), rtol=1e-2, atol=1e-2) + def test_decode_kernel_supports_small_page_size(self): + """Regression test for page sizes below Triton's minimum dot K dimension.""" + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_decode, + update_paged_kv_cache, + ) + + batch_size, seq_len = 1, 8 + n_heads, n_kv_heads, head_dim = 4, 2, 32 + page_size = 4 + num_pages_per_seq = (seq_len + page_size - 1) // page_size + num_blocks = num_pages_per_seq + 2 -class TestTritonPagedContextKernel: + q = torch.randn(batch_size, n_heads, head_dim, dtype=torch.float16, device="cuda") + k = torch.randn( + batch_size, seq_len, n_kv_heads, head_dim, dtype=torch.float16, device="cuda" + ) + v = torch.randn( + batch_size, seq_len, n_kv_heads, head_dim, dtype=torch.float16, device="cuda" + ) + k_flat = k.reshape(batch_size * seq_len, n_kv_heads, head_dim) + v_flat = v.reshape(batch_size * seq_len, n_kv_heads, head_dim) + batch_indices = torch.zeros(batch_size * seq_len, dtype=torch.int32, device="cuda") + positions = torch.arange(seq_len, device="cuda", dtype=torch.int32) + kv_indptr = torch.tensor([0, num_pages_per_seq], dtype=torch.int32, device="cuda") + kv_indices = torch.arange(num_pages_per_seq, dtype=torch.int32, device="cuda") + kv_last_page_len = torch.tensor([page_size], dtype=torch.int32, device="cuda") + kv_cache = create_paged_kv_cache(num_blocks, page_size, n_kv_heads, head_dim) + + update_paged_kv_cache( + k_flat, v_flat, batch_indices, positions, kv_cache, kv_indices, kv_indptr + ) + sm_scale = 1.0 / math.sqrt(head_dim) + output_triton = triton_decode( + q, kv_cache, kv_indices, kv_indptr, kv_last_page_len, sm_scale + ) + + q_ref = q.unsqueeze(2) + k_ref = k.transpose(1, 2).repeat_interleave(n_heads // n_kv_heads, dim=1) + v_ref = v.transpose(1, 2).repeat_interleave(n_heads // n_kv_heads, dim=1) + output_ref = torch.nn.functional.scaled_dot_product_attention( + q_ref, k_ref, v_ref, scale=sm_scale, is_causal=False + ).squeeze(2) + + torch.testing.assert_close(output_triton.float(), output_ref.float(), rtol=1e-2, atol=1e-2) + + +class TestTritonContextKernel: """Tests for the context/prefill kernel.""" @pytest.mark.parametrize("batch_size", [1, 2]) @@ -255,8 +301,8 @@ def test_context_kernel_vs_pytorch_reference( self, batch_size: int, n_heads: int, n_kv_heads: int, head_dim: int, seq_len: int ): """Test context kernel against PyTorch SDPA reference.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -308,7 +354,7 @@ def test_context_kernel_vs_pytorch_reference( sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, @@ -338,9 +384,60 @@ def test_context_kernel_vs_pytorch_reference( torch.testing.assert_close(output.float(), output_ref.float(), rtol=1e-2, atol=1e-2) + def test_context_kernel_supports_small_page_size(self): + """Regression test for page sizes below Triton's minimum dot K dimension.""" + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, + update_paged_kv_cache, + ) + + batch_size, seq_len = 1, 8 + n_heads, n_kv_heads, head_dim = 4, 2, 32 + page_size = 4 + num_pages_per_seq = (seq_len + page_size - 1) // page_size + num_blocks = num_pages_per_seq + 2 + total_tokens = batch_size * seq_len + + q = torch.randn(total_tokens, n_heads, head_dim, dtype=torch.float16, device="cuda") + k = torch.randn(total_tokens, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + v = torch.randn(total_tokens, n_kv_heads, head_dim, dtype=torch.float16, device="cuda") + qo_indptr = torch.tensor([0, seq_len], dtype=torch.int32, device="cuda") + kv_indptr = torch.tensor([0, num_pages_per_seq], dtype=torch.int32, device="cuda") + kv_indices = torch.arange(num_pages_per_seq, dtype=torch.int32, device="cuda") + kv_last_page_len = torch.tensor([page_size], dtype=torch.int32, device="cuda") + seq_len_with_cache = torch.tensor([seq_len], dtype=torch.int32, device="cuda") + batch_indices = torch.zeros(total_tokens, dtype=torch.int32, device="cuda") + positions = torch.arange(seq_len, device="cuda", dtype=torch.int32) + kv_cache = create_paged_kv_cache(num_blocks, page_size, n_kv_heads, head_dim) + + update_paged_kv_cache(k, v, batch_indices, positions, kv_cache, kv_indices, kv_indptr) + sm_scale = 1.0 / math.sqrt(head_dim) + output = triton_context( + q, + kv_cache, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + seq_len_with_cache, + sm_scale, + ) + + q_ref = q.view(batch_size, seq_len, n_heads, head_dim).transpose(1, 2) + k_ref = k.view(batch_size, seq_len, n_kv_heads, head_dim).transpose(1, 2) + v_ref = v.view(batch_size, seq_len, n_kv_heads, head_dim).transpose(1, 2) + k_ref = k_ref.repeat_interleave(n_heads // n_kv_heads, dim=1) + v_ref = v_ref.repeat_interleave(n_heads // n_kv_heads, dim=1) + output_ref = torch.nn.functional.scaled_dot_product_attention( + q_ref, k_ref, v_ref, scale=sm_scale, is_causal=True + ) + output_ref = output_ref.transpose(1, 2).reshape(total_tokens, n_heads, head_dim) + + torch.testing.assert_close(output.float(), output_ref.float(), rtol=1e-2, atol=1e-2) + def test_context_with_custom_bool_mask_matches_torch_attention(self): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context_with_custom_mask, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context_with_custom_mask, update_paged_kv_cache, ) @@ -387,7 +484,7 @@ def test_context_with_custom_bool_mask_matches_torch_attention(self): (pos.unsqueeze(0) <= pos.unsqueeze(1)).unsqueeze(0) | media_mask ).unsqueeze(1) - output = triton_paged_context_with_custom_mask( + output = triton_context_with_custom_mask( q, kv_cache, qo_indptr, @@ -411,8 +508,8 @@ def test_context_with_custom_bool_mask_matches_torch_attention(self): torch.testing.assert_close(output.float(), output_ref.float(), rtol=1e-2, atol=1e-2) def test_context_with_custom_bool_mask_and_sliding_window_with_cache_prefix(self): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context_with_custom_mask, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context_with_custom_mask, update_paged_kv_cache, ) @@ -480,7 +577,7 @@ def test_context_with_custom_bool_mask_and_sliding_window_with_cache_prefix(self ).unsqueeze(1) custom_attn_mask = full_mask[:, :, -query_len:, :] - output = triton_paged_context_with_custom_mask( + output = triton_context_with_custom_mask( q, kv_cache, qo_indptr, @@ -515,8 +612,8 @@ def test_context_with_custom_bool_mask_and_sliding_window_with_cache_prefix(self torch.testing.assert_close(output.float(), output_ref.float(), rtol=1e-2, atol=1e-2) def test_context_with_custom_bool_mask_and_sliding_window_matches_torch_attention(self): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context_with_custom_mask, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context_with_custom_mask, update_paged_kv_cache, ) @@ -568,7 +665,7 @@ def test_context_with_custom_bool_mask_and_sliding_window_matches_torch_attentio (pos.unsqueeze(0) <= pos.unsqueeze(1)).unsqueeze(0) | media_mask ).unsqueeze(1) - output = triton_paged_context_with_custom_mask( + output = triton_context_with_custom_mask( q, kv_cache, qo_indptr, @@ -599,7 +696,7 @@ class TestCacheUpdate: def test_cache_update_writes_correct_values(self): """Test that cache update writes K, V to correct locations.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( update_paged_kv_cache, ) @@ -646,8 +743,8 @@ def test_cache_update_writes_correct_values(self): torch.testing.assert_close(actual_v, expected_v) -class TestTritonPagedMHAIntegration: - """Integration tests for triton_paged_mha_with_cache and prepare_triton_paged_metadata. +class TestTritonMHAIntegration: + """Integration tests for triton_mha_with_cache and prepare_triton_metadata. These test the full integration layer including BatchInfo parsing, metadata preparation, KV cache update, and mixed prefill/decode dispatch. @@ -680,14 +777,14 @@ def _make_batch_info( return bi def test_batch_info_12_element_format(self): - """Test that triton_paged_mha_with_cache handles 12-element batch_info_host. + """Test that triton_mha_with_cache handles 12-element batch_info_host. Regression test: batch_info_host changed from 3-element to 12-element tensor. The old code did `num_prefill, num_prefill_tokens, num_decode = batch_info_host.tolist()` which would crash with ValueError: too many values to unpack. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_mha_with_cache, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_mha_with_cache, ) n_heads, n_kv_heads, head_dim, page_size = 32, 8, 128, 16 @@ -717,12 +814,12 @@ def test_batch_info_12_element_format(self): ) # Prepare metadata (this also uses batch_info_host) - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - prepare_triton_paged_metadata, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + prepare_triton_metadata, ) position_ids = torch.arange(seq_len, device="cuda") - batch_indices, positions = prepare_triton_paged_metadata( + batch_indices, positions = prepare_triton_metadata( position_ids, batch_info_host, cu_seqlen_host.to("cuda", non_blocking=True), @@ -730,7 +827,7 @@ def test_batch_info_12_element_format(self): ) # Run the full MHA with cache (should not crash) - output = triton_paged_mha_with_cache( + output = triton_mha_with_cache( q, k, v, @@ -755,8 +852,8 @@ def test_batch_info_12_element_format(self): def test_context_prefill_honors_out_buffer(self): """The generic context prefill path must write through out= exactly.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_mha_with_cache, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_mha_with_cache, ) torch.manual_seed(0) @@ -786,7 +883,7 @@ def test_context_prefill_honors_out_buffer(self): kv_cache = torch.zeros( num_blocks, 2, n_kv_heads, page_size, head_dim, dtype=dtype, device="cuda" ) - expected_from_op = triton_paged_mha_with_cache( + expected_from_op = triton_mha_with_cache( q, k, v, @@ -807,7 +904,7 @@ def test_context_prefill_honors_out_buffer(self): out = torch.full_like(q, float("nan")) kv_cache_with_out = torch.zeros_like(kv_cache) - returned = triton_paged_mha_with_cache( + returned = triton_mha_with_cache( q, k, v, @@ -873,9 +970,9 @@ def test_accepts_positional_constants_with_keyword_custom_mask(self): The KV-cache transform passes scale/sliding_window positionally and custom_attn_mask by keyword. This should bind correctly. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - prepare_triton_paged_metadata, - triton_paged_mha_with_cache, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + prepare_triton_metadata, + triton_mha_with_cache, ) n_heads, n_kv_heads, head_dim, page_size = 8, 2, 128, 16 @@ -902,7 +999,7 @@ def test_accepts_positional_constants_with_keyword_custom_mask(self): ) position_ids = torch.arange(seq_len, device="cuda") - batch_indices, positions = prepare_triton_paged_metadata( + batch_indices, positions = prepare_triton_metadata( position_ids, batch_info_host, cu_seqlen_host.to("cuda", non_blocking=True), @@ -910,7 +1007,7 @@ def test_accepts_positional_constants_with_keyword_custom_mask(self): ) custom_attn_mask = torch.ones(1, 1, seq_len, seq_len, dtype=torch.bool, device="cuda") - output = triton_paged_mha_with_cache( + output = triton_mha_with_cache( q, k, v, @@ -935,9 +1032,9 @@ def test_accepts_positional_constants_with_keyword_custom_mask(self): assert not torch.isinf(output).any(), "Output contains Inf" def test_prepare_metadata_with_12_element_batch_info(self): - """Test prepare_triton_paged_metadata with 12-element batch_info_host.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - prepare_triton_paged_metadata, + """Test prepare_triton_metadata with 12-element batch_info_host.""" + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + prepare_triton_metadata, ) batch_info_host = self._make_batch_info(num_prefill=1, num_prefill_tokens=7, num_decode=0) @@ -946,7 +1043,7 @@ def test_prepare_metadata_with_12_element_batch_info(self): seq_len_with_cache = torch.tensor([7], dtype=torch.int32, device="cuda") # Should not raise ValueError - batch_indices, positions = prepare_triton_paged_metadata( + batch_indices, positions = prepare_triton_metadata( position_ids, batch_info_host, cu_seqlen, seq_len_with_cache ) @@ -1011,8 +1108,8 @@ def test_decode_sliding_window( sliding_window: int, ): """Test decode with sliding window against reference (seq_len > window).""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_decode, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_decode, update_paged_kv_cache, ) @@ -1065,7 +1162,7 @@ def test_decode_sliding_window( sm_scale = 1.0 / math.sqrt(head_dim) - output_triton = triton_paged_decode( + output_triton = triton_decode( q, kv_cache, kv_indices, @@ -1105,8 +1202,8 @@ def test_context_sliding_window( sliding_window: int, ): """Test prefill with sliding window against manual reference (seq_len > window).""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1155,7 +1252,7 @@ def test_context_sliding_window( sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, @@ -1183,8 +1280,8 @@ def test_context_sliding_window( def test_no_sliding_window_unchanged(self): """Verify that sliding_window=None produces the same output as before.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_decode, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_decode, update_paged_kv_cache, ) @@ -1237,7 +1334,7 @@ def test_no_sliding_window_unchanged(self): sm_scale = 1.0 / math.sqrt(head_dim) - out_none = triton_paged_decode( + out_none = triton_decode( q, kv_cache, kv_indices, @@ -1246,7 +1343,7 @@ def test_no_sliding_window_unchanged(self): sm_scale, sliding_window=None, ) - out_zero = triton_paged_decode( + out_zero = triton_decode( q, kv_cache, kv_indices, @@ -1272,8 +1369,8 @@ def test_decode_vs_flashinfer( """Compare decode output against FlashInfer.""" import flashinfer - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_decode, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_decode, update_paged_kv_cache, ) @@ -1327,7 +1424,7 @@ def test_decode_vs_flashinfer( update_paged_kv_cache( k_flat, v_flat, batch_indices, positions, kv_cache_triton, kv_indices, kv_indptr ) - output_triton = triton_paged_decode( + output_triton = triton_decode( q, kv_cache_triton, kv_indices, kv_indptr, kv_last_page_len, sm_scale ) @@ -1384,8 +1481,8 @@ def test_prefill_vs_flashinfer( """Compare prefill output against FlashInfer.""" import flashinfer - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1437,7 +1534,7 @@ def test_prefill_vs_flashinfer( update_paged_kv_cache( k, v, batch_indices, positions, kv_cache_triton, kv_indices, kv_indptr ) - output_triton = triton_paged_context( + output_triton = triton_context( q, kv_cache_triton, qo_indptr, @@ -1486,7 +1583,7 @@ def test_prefill_vs_flashinfer( class TestSDPADispatch: - """Tests for the adaptive SDPA dispatch path in triton_paged_context. + """Tests for the adaptive SDPA dispatch path in triton_context. Covers: large head_dim forcing SDPA, variable-length sequence fallback to paged kernel, FP8 KV cache dtype casting, and oversized kv_indices buffers. @@ -1501,12 +1598,12 @@ def _run_context_and_reference( page_size: int = 16, kv_cache_dtype: torch.dtype | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Run triton_paged_context and a PyTorch SDPA reference, return both outputs. + """Run triton_context and a PyTorch SDPA reference, return both outputs. Supports variable-length sequences within a batch. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1567,7 +1664,7 @@ def _run_context_and_reference( sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, @@ -1681,8 +1778,8 @@ def test_large_head_dim_context_respects_cache_prefix_and_page_tail(self): This protects the large-head-dim SDPA path for a cache-prefix context. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1725,7 +1822,7 @@ def test_large_head_dim_context_respects_cache_prefix_and_page_tail(self): def run_with_tail(k_sentinel: float, v_sentinel: float) -> torch.Tensor: kv_cache[0, 0, :, kv_len:page_size, :].fill_(k_sentinel) kv_cache[0, 1, :, kv_len:page_size, :].fill_(v_sentinel) - return triton_paged_context( + return triton_context( q, kv_cache, qo_indptr, @@ -1766,8 +1863,8 @@ def test_sdpa_dispatch_respects_cache_prefix_and_page_tail(self): """SDPA dispatch must mask page-tail padding and use absolute query positions.""" from unittest.mock import patch - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1821,7 +1918,7 @@ def run_with_tail(k_sentinel: float, v_sentinel: float) -> torch.Tensor: kv_cache[last_page, 0, :, last_page_tail_start:page_size, :].fill_(k_sentinel) kv_cache[last_page, 1, :, last_page_tail_start:page_size, :].fill_(v_sentinel) with patch.object(torch.nn.functional, "scaled_dot_product_attention", tracking_sdpa): - return triton_paged_context( + return triton_context( q, kv_cache, qo_indptr, @@ -1865,8 +1962,8 @@ def test_oversized_kv_indices_buffer(self): Tests the pages_uniform fallback via kv_indptr when kv_indices is pre-allocated. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -1921,7 +2018,7 @@ def test_oversized_kv_indices_buffer(self): update_paged_kv_cache(k, v, batch_indices, positions, kv_cache, kv_indices, kv_indptr) sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, @@ -1953,8 +2050,8 @@ def test_oversized_kv_indices_buffer(self): @pytest.mark.parametrize("seq_len", [512, 1024]) def test_fp8_kv_cache_dtype_casting(self, batch_size: int, seq_len: int): """FP8 KV cache values are correctly cast to query dtype in both paths.""" - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, update_paged_kv_cache, ) @@ -2003,7 +2100,7 @@ def test_fp8_kv_cache_dtype_casting(self, batch_size: int, seq_len: int): sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache_fp8, qo_indptr, diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_swa.py similarity index 97% rename from tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py rename to tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_swa.py index 2da593035874..a6b67dc463c5 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_paged_attention_swa.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_swa.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SWA correctness tests for Triton paged attention. +"""SWA correctness tests for Triton attention. Two regimes that the existing kernel tests don't exercise: @@ -100,7 +100,7 @@ def _write_kv_to_cache( tensor listing physical page ids in token-order. Assumes page_table covers exactly enough pages for k.shape[0] tokens. """ - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( update_paged_kv_cache, ) @@ -111,7 +111,7 @@ def _write_kv_to_cache( update_paged_kv_cache(k, v, batch_indices, positions, kv_cache, page_table, kv_indptr) -class TestTritonPagedLongPrefillSWA: +class TestTritonLongPrefillSWA: """Prefill longer than the sliding window — no eviction yet. Verifies the kernel correctly applies the SW mask within a single long @@ -133,8 +133,8 @@ def test_long_prefill_sw_matches_sdpa( n_kv_heads: int, head_dim: int, ): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, ) torch.manual_seed(0) @@ -165,7 +165,7 @@ def test_long_prefill_sw_matches_sdpa( seq_len_with_cache = torch.tensor([prefill_len], dtype=torch.int32, device="cuda") sm_scale = 1.0 / math.sqrt(head_dim) - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, @@ -188,7 +188,7 @@ def test_long_prefill_sw_matches_sdpa( torch.testing.assert_close(output.float(), ref.float(), rtol=1e-2, atol=1e-2) -class TestTritonPagedSWAFrontEviction: +class TestTritonSWAFrontEviction: """Front-eviction SWA — the shim hands the kernel a window-local view. The setup mirrors what `ad_executor.py` does after Phase 2: it slices @@ -209,8 +209,8 @@ def test_front_evicted_context_matches_sdpa_on_live_window( page_size: int, with_sliding_window: bool, ): - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - triton_paged_context, + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( + triton_context, ) # Fixed simple geometry for clarity. Window W=256 → 16 pages at PAGE_SIZE=16, @@ -285,7 +285,7 @@ def test_front_evicted_context_matches_sdpa_on_live_window( new_v_padded[:new_chunk] = new_v # We can't easily write a partial-page; only fill if new_chunk is multiple of page_size. # For partial pages, fall back to per-token write. - from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( + from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import ( update_paged_kv_cache, ) @@ -323,7 +323,7 @@ def test_front_evicted_context_matches_sdpa_on_live_window( sm_scale = 1.0 / math.sqrt(head_dim) sw_arg = sliding_window if with_sliding_window else 0 - output = triton_paged_context( + output = triton_context( q, kv_cache, qo_indptr, diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_with_kv_cache.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_with_kv_cache.py deleted file mode 100644 index 00a67908bb8d..000000000000 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention_with_kv_cache.py +++ /dev/null @@ -1,724 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math -import random - -import pytest -import torch -import triton -from _custom_op_utils import torch_rope_reference -from _model_test_utils import repeat_kv - -from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention_with_kv_cache import ( - attention_kv_stage1, - attention_kv_stage2, - context_attention_kv, - context_attention_kv_flattened, - gqa_attention_kv_stage1, - update_kv_cache, - update_kv_cache_rope_fusion, -) - - -def torch_reference_stage2(values, logsumexp, sinks=None): - max_logsumexp = torch.max(logsumexp, axis=-1, keepdim=True)[0] # [b, n_heads, 1] - sumexp = torch.exp(logsumexp - max_logsumexp) # [b, n_heads, num_blocks] - aggregate_sumexp = torch.sum(sumexp, axis=-1) # [b, n_heads] - # Add sinks contribution to the softmax denominator - if sinks is not None: - sinks_exp = torch.exp(sinks - max_logsumexp.squeeze(-1)) # [b, n_heads] - aggregate_sumexp += sinks_exp - output = values * sumexp[:, :, :, None] # [b, n_heads, num_blocks, d_head] - output = output / aggregate_sumexp[:, :, None, None] - output = torch.sum(output, axis=2) - return output - - -@pytest.mark.parametrize("k_d_head", [16, 96]) -@pytest.mark.parametrize("v_d_head", [16, 96]) -@pytest.mark.parametrize( - "seq_lens", - [ - [16, 8, 9, 21], # context only sequences - [1, 1, 1, 1, 1, 1], # decode only sequences - [5, 10, 4, 1, 1, 1], # context + decode sequences - ], -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) -def test_update_kv_cache(k_d_head, v_d_head, seq_lens, dtype): - DEVICE = "cuda" - DTYPE = dtype - N_KV_HEADS = 8 - K_D_HEAD = k_d_head - V_D_HEAD = v_d_head - MAX_SEQ_LEN = 64 - MAX_BATCH_SIZE = 16 - SEQ_LENS = seq_lens - SLOT_IDX = list(range(0, len(SEQ_LENS))) - random.shuffle(SLOT_IDX) - INPUT_POS = [ - random.randint(0, 16) for _ in range(len(SEQ_LENS)) - ] # The starting position for in the cache for each of the sequences. - k = [] - v = [] - for i, s in enumerate(SEQ_LENS): - k.append(torch.ones(1, s, N_KV_HEADS, K_D_HEAD, dtype=DTYPE, device=DEVICE)) - v.append(torch.ones(1, s, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE)) - - (k_f, v_f) = tuple(map(lambda x: torch.cat(x, 1), (k, v))) - k_f = k_f.contiguous() - v_f = v_f.contiguous() - k_cache = torch.zeros( - MAX_BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, K_D_HEAD, dtype=DTYPE, device=DEVICE - ) - v_cache = torch.zeros( - MAX_BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE - ) - - GENERATE_ONLY = all(s == 1 for s in SEQ_LENS) - SEQ_BLOCK = 1 if GENERATE_ONLY else 32 - grid = (len(SEQ_LENS), N_KV_HEADS, (max(SEQ_LENS) + SEQ_BLOCK - 1) // SEQ_BLOCK) - - seq_len = torch.tensor(SEQ_LENS, device=DEVICE, dtype=torch.int32) - seq_start_indices = torch.zeros(len(SEQ_LENS), device=DEVICE, dtype=torch.int32) - seq_start_indices[1:] = torch.cumsum(seq_len[:-1], 0) - - update_kv_cache[grid]( - k_f, - v_f, - seq_len, - seq_start_indices, - k_cache, - v_cache, - torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32), - torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32), - MAX_SEQ_LEN, - N_KV_HEADS, - K_D_HEAD, - V_D_HEAD, - SEQ_BLOCK, - GENERATE_ONLY, - ) - - # Check if the cache was correctly updated: - for i, slot_idx in enumerate(SLOT_IDX): - assert torch.equal( - k_cache[slot_idx, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].squeeze(), - k[i].squeeze(), - ) - assert torch.equal( - v_cache[slot_idx, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].squeeze(), - v[i].squeeze(), - ) - - -@pytest.mark.parametrize("d_head", [16, 96]) -def test_attention_kv_flash_decoding(d_head): - DEVICE = "cuda" - DTYPE = torch.float16 - BATCH_SIZE = 1 - N_HEADS = 1 - D_HEAD = d_head - MAX_SEQ_LEN = 64 - SLOT_IDX = list(range(0, BATCH_SIZE)) - random.shuffle(SLOT_IDX) - INPUT_POS = [0] * BATCH_SIZE - # Q,K,V are computed using GEMM. - q = torch.randn(BATCH_SIZE, 1, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE) - k = torch.randn(BATCH_SIZE, 1, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE) - v = torch.randn(BATCH_SIZE, 1, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE) - k_cache = torch.zeros(BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE) - v_cache = torch.zeros(BATCH_SIZE, MAX_SEQ_LEN, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE) - - grid = (BATCH_SIZE, N_HEADS, 1) # - update_kv_cache[grid]( - k, - v, - torch.tensor([1] * BATCH_SIZE, device=DEVICE, dtype=torch.int32), - torch.arange(0, BATCH_SIZE, 1, device=DEVICE, dtype=torch.int32), - k_cache, - v_cache, - torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32), - torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32), - MAX_SEQ_LEN, - N_HEADS, - D_HEAD, - D_HEAD, - SEQ_BLOCK=1, - GENERATE_ONLY=True, - ) - SEQ_BLOCK_SIZE = 8 - num_blocks = MAX_SEQ_LEN // SEQ_BLOCK_SIZE - output_tensor = torch.zeros( - BATCH_SIZE, N_HEADS, num_blocks, D_HEAD, device=DEVICE, dtype=torch.float32 - ) - output_logsumexp = torch.zeros( - BATCH_SIZE, N_HEADS, num_blocks, device=DEVICE, dtype=torch.float32 - ) - float("inf") - - def run(q, k_cache, v_cache, output_tensor, output_logsumexp): - attention_kv_stage1[ - ( - BATCH_SIZE, - N_HEADS, - num_blocks, - ) - ]( - q, - k_cache, - v_cache, - torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32), - torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32), - output_tensor, - output_logsumexp, - num_blocks, - MAX_SEQ_LEN, - N_HEADS, - N_HEADS, - D_HEAD, - SEQ_BLOCK_SIZE, - ) - - run(q, k_cache, v_cache, output_tensor, output_logsumexp) - output = torch_reference_stage2(output_tensor, output_logsumexp) - - ref = [] - for i in range(BATCH_SIZE): - ref.append( - torch.nn.functional.scaled_dot_product_attention( - q[i, :, :, :].unsqueeze(0).transpose(1, 2), # [BSND]->[BNSD] - k_cache[SLOT_IDX[i], 0 : INPUT_POS[i] + 1, :, :].unsqueeze(0).transpose(1, 2), - v_cache[SLOT_IDX[i], 0 : INPUT_POS[i] + 1, :, :].unsqueeze(0).transpose(1, 2), - ) - ) - ref = torch.cat(ref, dim=0) - print(ref) - torch.allclose( - ref.squeeze().cpu().to(torch.float32), - output.squeeze().cpu().to(torch.float32), - atol=1e-2, - rtol=1e-2, - ) - - -@pytest.mark.parametrize("q_d_head", [16, 96]) -@pytest.mark.parametrize("v_d_head", [16, 96]) -@pytest.mark.parametrize("n_heads,n_kv_heads", [(8, 8), (8, 1)]) -@pytest.mark.parametrize("sliding_window", [-1, 16]) -def test_gqa_attention_kv_flash_decoding(q_d_head, v_d_head, n_heads, n_kv_heads, sliding_window): - DEVICE = "cuda" - DTYPE = torch.float16 - BATCH_SIZE = 64 - N_HEADS = n_heads - N_KV_HEADS = n_kv_heads - Q_D_HEAD = q_d_head - V_D_HEAD = v_d_head - MAX_SEQ_LEN = 64 - SLOT_IDX = list(range(0, BATCH_SIZE)) - INPUT_POS = [0] * BATCH_SIZE - # Q,K,V are computed using GEMM. - q = torch.randn(BATCH_SIZE, 1, N_HEADS, Q_D_HEAD, dtype=DTYPE, device=DEVICE) - k = torch.randn(BATCH_SIZE, 1, N_KV_HEADS, Q_D_HEAD, dtype=DTYPE, device=DEVICE) - v = torch.randn(BATCH_SIZE, 1, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE) - k_cache = torch.randn(BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, Q_D_HEAD, dtype=DTYPE, device=DEVICE) - v_cache = torch.randn(BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE) - - slot_idx = torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32) - input_pos = torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32) - - grid = (BATCH_SIZE, N_KV_HEADS, 1) # - update_kv_cache[grid]( - k, - v, - torch.tensor([1] * BATCH_SIZE, device=DEVICE, dtype=torch.int32), - torch.arange(0, BATCH_SIZE, 1, device=DEVICE, dtype=torch.int32), - k_cache, - v_cache, - input_pos, - slot_idx, - MAX_SEQ_LEN, - N_KV_HEADS, - Q_D_HEAD, - V_D_HEAD, - SEQ_BLOCK=1, - GENERATE_ONLY=True, - ) - SEQ_BLOCK_SIZE = 16 - num_blocks = MAX_SEQ_LEN // SEQ_BLOCK_SIZE - output_tensor = torch.zeros( - BATCH_SIZE, N_HEADS, num_blocks, V_D_HEAD, device=DEVICE, dtype=torch.float32 - ) - output_logsumexp = torch.zeros( - BATCH_SIZE, N_HEADS, num_blocks, device=DEVICE, dtype=torch.float32 - ) - float("inf") - - def run(q, k_cache, v_cache, output_tensor, output_logsumexp): - HEAD_BLOCK_SIZE = max(16, triton.next_power_of_2(N_HEADS // N_KV_HEADS)) - - gqa_attention_kv_stage1[ - ( - BATCH_SIZE, - N_KV_HEADS, - num_blocks, - ) - ]( - q, - k_cache, - v_cache, - slot_idx, - input_pos, - output_tensor, - output_logsumexp, - num_blocks, - 1.0 / math.sqrt(Q_D_HEAD), - MAX_SEQ_LEN, - N_HEADS, - N_KV_HEADS, - Q_D_HEAD, - V_D_HEAD, - SEQ_BLOCK_SIZE, - HEAD_BLOCK_SIZE, - sliding_window, # SLIDING_WINDOW: parameterized - ) - - run(q, k_cache, v_cache, output_tensor, output_logsumexp) - # This needs to be another kernel if torch-trt doesn't support broadcast + div. - output = torch_reference_stage2(output_tensor, output_logsumexp) - - ref = [] - for i in range(BATCH_SIZE): - kk = k_cache[SLOT_IDX[i], 0 : INPUT_POS[i] + 1, :, :].unsqueeze(0) - vv = v_cache[SLOT_IDX[i], 0 : INPUT_POS[i] + 1, :, :].unsqueeze(0) - - if N_HEADS != N_KV_HEADS: - kk = repeat_kv(q[i, :, :, :].unsqueeze(0), kk) - vv = repeat_kv(q[i, :, :, :].unsqueeze(0), vv) - ref.append( - torch.nn.functional.scaled_dot_product_attention( - q[i, :, :, :].unsqueeze(0).transpose(1, 2), # [BSND]->[BNSD] - kk.transpose(1, 2), - vv.transpose(1, 2), - ) - ) - ref = torch.cat(ref, dim=0) - assert torch.allclose( - ref.squeeze().cpu().to(torch.float32), - output.squeeze().cpu().to(torch.float32), - atol=1e-2, - rtol=1e-2, - ) - - -@pytest.mark.parametrize("has_sinks", [False, True]) -def test_attention_with_kv_stage2(has_sinks): - DEVICE = "cuda" - BATCH_SIZE = 4 - N_HEADS = 32 - D_HEAD = 96 - SEQ_BLOCK_SIZE = 8 - input_positions = torch.zeros(BATCH_SIZE, device=DEVICE, dtype=torch.int32) + 10 - num_blocks = 3 - # Produced by stage1 - values = torch.randn( - BATCH_SIZE, N_HEADS, num_blocks, D_HEAD, device=DEVICE, dtype=torch.float32 - ) - logsumexp = torch.randn(BATCH_SIZE, N_HEADS, num_blocks, device=DEVICE, dtype=torch.float32) - output = torch.zeros(BATCH_SIZE, N_HEADS, D_HEAD, device=DEVICE, dtype=torch.float32) - # Create sink tokens if needed - kernel expects [BATCH_SIZE, N_HEADS] shape - sinks = ( - torch.randn(BATCH_SIZE, N_HEADS, device=DEVICE, dtype=torch.float32) if has_sinks else None - ) - - def run(): - attention_kv_stage2[ - ( - BATCH_SIZE, - N_HEADS, - ) - ]( - values, - logsumexp, - output, - input_positions, - num_blocks, - N_HEADS, - D_HEAD, - SEQ_BLOCK_SIZE, - has_sinks, - sinks, - ) - - run() - ref = [] - for i in range(BATCH_SIZE): - block_id = input_positions[i].item() // SEQ_BLOCK_SIZE + 1 - batch_sinks = sinks[i : i + 1, :] if has_sinks else None # [1, N_HEADS] - ref.append( - torch_reference_stage2( - values[i, :, :block_id, :].unsqueeze(0), - logsumexp[i, :, :block_id].unsqueeze(0), - batch_sinks, - ) - ) - ref = torch.cat(ref, dim=0) - assert torch.allclose( - ref.squeeze().cpu().to(torch.float32), - output.squeeze().cpu().to(torch.float32), - atol=1e-2, - rtol=1e-2, - ) - - -@pytest.mark.parametrize("batch_size", [1, 4]) -@pytest.mark.parametrize("q_d_head", [32, 96]) -@pytest.mark.parametrize("v_d_head", [32, 96]) -@pytest.mark.parametrize("n_heads,n_kv_heads", [(8, 8), (8, 1)]) -@pytest.mark.parametrize( - "dtype_cache", - [ - torch.bfloat16, - pytest.param( - torch.float8_e4m3fn, - marks=pytest.mark.skipif( - torch.cuda.get_device_capability(0) < (8, 9), reason="Requires fp8 support" - ), - ), - ], -) -def test_context_attention_kv(batch_size, q_d_head, v_d_head, n_heads, n_kv_heads, dtype_cache): - DEVICE = "cuda" - DTYPE = torch.bfloat16 - BATCH_SIZE = batch_size - N_HEADS = n_heads - N_KV_HEADS = n_kv_heads - Q_D_HEAD = K_D_HEAD = q_d_head - V_D_HEAD = v_d_head - MAX_SEQ_LEN = 64 - SEQ = 36 - # Q,K,V are computed using GEMM. - q = torch.randn(BATCH_SIZE, SEQ, N_HEADS, Q_D_HEAD, dtype=DTYPE, device=DEVICE) - k = torch.randn(BATCH_SIZE, SEQ, N_KV_HEADS, K_D_HEAD, dtype=DTYPE, device=DEVICE) - v = torch.randn(BATCH_SIZE, SEQ, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE) - k_cache = torch.zeros( - BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, K_D_HEAD, dtype=dtype_cache, device=DEVICE - ) - v_cache = torch.zeros( - BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, V_D_HEAD, dtype=dtype_cache, device=DEVICE - ) - - SEQ_BLOCK = 16 - output_tensor = torch.empty((BATCH_SIZE, SEQ, N_HEADS, V_D_HEAD), dtype=DTYPE, device=DEVICE) - grid = (BATCH_SIZE, N_HEADS, (SEQ + SEQ_BLOCK - 1) // SEQ_BLOCK) - context_attention_kv[grid]( - q, - k, - v, - k_cache, - v_cache, - SEQ, - output_tensor, - 1.0 / math.sqrt(Q_D_HEAD), - N_HEADS, - N_KV_HEADS, - Q_D_HEAD, - V_D_HEAD, - SEQ_BLOCK, - MAX_SEQ_LEN, - num_stages=2, - ) - - if N_HEADS != N_KV_HEADS: - k = repeat_kv(q, k) - v = repeat_kv(q, v) - ref = torch.nn.functional.scaled_dot_product_attention( - q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True - ).transpose(2, 1) - assert torch.allclose(ref, output_tensor, atol=1e-2, rtol=1e-2) - - -@pytest.mark.parametrize( - "dtype", - ["float16", "float32", "bfloat16"], -) -@pytest.mark.parametrize("n_heads,n_kv_heads", [(8, 8), (8, 1)]) -@pytest.mark.parametrize("q_d_head", [32, 96]) -@pytest.mark.parametrize("v_d_head", [32, 96]) -@pytest.mark.parametrize("sliding_window", [-1, 16]) -def test_context_attention_kv_flattened( - q_d_head, v_d_head, n_heads, n_kv_heads, dtype, sliding_window -): - DEVICE = "cuda" - DTYPE = getattr(torch, dtype) - N_HEADS = n_heads - N_KV_HEADS = n_kv_heads - K_D_HEAD = Q_D_HEAD = q_d_head - V_D_HEAD = v_d_head - MAX_SEQ_LEN = 64 - SEQ_LENS = [36, 44, 12, 1, 1] - SLOT_IDX = list(range(0, len(SEQ_LENS))) - random.shuffle(SLOT_IDX) - INPUT_POS = [2, 4, 8, 0, 1] # The starting position for in the cache for each of the sequences. - q = [] - k = [] - v = [] - for i, s in enumerate(SEQ_LENS): - q.append(torch.ones(1, s, N_HEADS, Q_D_HEAD, dtype=DTYPE, device=DEVICE)) - k.append(torch.ones(1, s, N_KV_HEADS, K_D_HEAD, dtype=DTYPE, device=DEVICE)) - v.append(torch.ones(1, s, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE)) - - (q_f, k_f, v_f) = tuple(map(lambda x: torch.cat(x, 1).contiguous(), (q, k, v))) - - k_cache = torch.zeros( - len(SEQ_LENS), MAX_SEQ_LEN, N_KV_HEADS, K_D_HEAD, dtype=DTYPE, device=DEVICE - ) - v_cache = torch.zeros( - len(SEQ_LENS), MAX_SEQ_LEN, N_KV_HEADS, V_D_HEAD, dtype=DTYPE, device=DEVICE - ) - - def compute_reference(q, k_cache, v_cache): - ref = [] - for i in range(len(SEQ_LENS)): - kk = k_cache[SLOT_IDX[i], : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].view( - 1, INPUT_POS[i] + SEQ_LENS[i], N_KV_HEADS, K_D_HEAD - ) - vv = v_cache[SLOT_IDX[i], : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].view( - 1, INPUT_POS[i] + SEQ_LENS[i], N_KV_HEADS, V_D_HEAD - ) - - if N_HEADS != N_KV_HEADS: - kk = repeat_kv(q[i], kk) - vv = repeat_kv(q[i], vv) - - mask = torch.tril( - torch.ones(q[i].shape[1], kk.shape[1], dtype=torch.bool), - diagonal=kk.shape[1] - q[i].shape[1], - ) - - # Apply sliding window constraints if enabled - if sliding_window > 0: - seq_len_q = q[i].shape[1] # Current sequence length - seq_len_k = kk.shape[1] # Total KV sequence length - - # Create sliding window mask - sliding_mask = torch.zeros_like(mask) - for q_pos in range(seq_len_q): - # For each query position, determine its absolute position in the cache - abs_q_pos = INPUT_POS[i] + q_pos - # Calculate sliding window range - sliding_start = max(0, abs_q_pos - sliding_window + 1) - sliding_end = abs_q_pos + 1 - # Apply to KV cache positions - k_start = max(0, sliding_start) - k_end = min(seq_len_k, sliding_end) - if k_start < k_end: - sliding_mask[q_pos, k_start:k_end] = True - - # Combine causal and sliding window masks - mask = mask & sliding_mask - - ref.append( - torch.nn.functional.scaled_dot_product_attention( - q[i].transpose(1, 2), - kk.transpose(1, 2), - vv.transpose(1, 2), - attn_mask=mask.to(DEVICE), - ).transpose(2, 1) - ) - return torch.cat(ref, 1) - - seq_len = torch.tensor(SEQ_LENS, device=DEVICE, dtype=torch.int32) - seq_start_indices = torch.zeros(len(SEQ_LENS), device=DEVICE, dtype=torch.int32) - seq_start_indices[1:] = torch.cumsum(seq_len[:-1], 0) - input_pos = torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32) - slot_idx = torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32) - SEQ_BLOCK = 32 - output_tensor = torch.empty((1, sum(SEQ_LENS), N_HEADS, V_D_HEAD), dtype=DTYPE, device=DEVICE) - grid = (len(SEQ_LENS), N_KV_HEADS, (max(SEQ_LENS) + SEQ_BLOCK - 1) // SEQ_BLOCK) - update_kv_cache[grid]( - k_f, - v_f, - seq_len, - seq_start_indices, - k_cache, - v_cache, - input_pos, - slot_idx, - MAX_SEQ_LEN, - N_KV_HEADS, - K_D_HEAD, - V_D_HEAD, - SEQ_BLOCK, - GENERATE_ONLY=False, - ) - - # Check if the cache was correctly updated: - for i, cl in enumerate(SLOT_IDX): - assert torch.equal( - k_cache[cl, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].squeeze(), - k[i].squeeze(), - ) - assert torch.equal( - v_cache[cl, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i], :N_KV_HEADS, :].squeeze(), - v[i].squeeze(), - ) - ref = compute_reference(q, k_cache, v_cache) - grid = (len(SEQ_LENS), N_HEADS, (max(SEQ_LENS) + SEQ_BLOCK - 1) // SEQ_BLOCK) - context_attention_kv_flattened[grid]( - q_f, - seq_len, - seq_start_indices, - k_cache, - v_cache, - input_pos, - slot_idx, - output_tensor, - 1.0 / math.sqrt(Q_D_HEAD), - N_HEADS, - N_KV_HEADS, - K_D_HEAD, - V_D_HEAD, - SEQ_BLOCK, - MAX_SEQ_LEN, - sliding_window, # SLIDING_WINDOW: parameterized - False, # HAS_SINKS: no sink tokens used - None, # sinks_ptr: no sink tokens used - ) - assert torch.allclose(ref, output_tensor, atol=1e-2, rtol=1e-2) - - -@pytest.mark.parametrize( - "seq_lens", - [ - [16, 8, 9, 21], # context only sequences - [1, 1, 1, 1, 1, 1], # decode only sequences - [5, 10, 4, 1, 1, 1], # context + decode sequences - ], -) -@pytest.mark.parametrize("n_heads,n_kv_heads", [(8, 8), (8, 1)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) -def test_update_kv_cache_rope_fusion(seq_lens, n_heads, n_kv_heads, dtype): - DEVICE = "cuda" - DTYPE = dtype - N_HEADS = n_heads - N_KV_HEADS = n_kv_heads - D_HEAD = 16 - MAX_SEQ_LEN = 64 - MAX_BATCH_SIZE = 16 - SEQ_LENS = seq_lens - BATCH_SIZE = len(SEQ_LENS) - SLOT_IDX = list(range(0, BATCH_SIZE)) - random.shuffle(SLOT_IDX) - INPUT_POS = [ - random.randint(0, 16) for _ in range(BATCH_SIZE) - ] # The starting position for in the cache for each of the sequences. - q = [] - k = [] - v = [] - for i, s in enumerate(SEQ_LENS): - q.append(torch.randn(1, s, N_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE)) - k.append(torch.randn(1, s, N_KV_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE)) - v.append(torch.randn(1, s, N_KV_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE)) - - (q_f, k_f, v_f) = tuple(map(lambda x: torch.cat(x, 1), (q, k, v))) - # kernel handles interleaved input - q_f = q_f.unflatten(-1, (D_HEAD // 2, 2)).transpose(-1, -2).flatten(-2).contiguous() - k_f = k_f.unflatten(-1, (D_HEAD // 2, 2)).transpose(-1, -2).flatten(-2).contiguous() - - q_rope = torch.zeros_like(q_f) - k_cache = torch.zeros( - MAX_BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE - ) - v_cache = torch.zeros( - MAX_BATCH_SIZE, MAX_SEQ_LEN, N_KV_HEADS, D_HEAD, dtype=DTYPE, device=DEVICE - ) - - GENERATE_ONLY = all(s == 1 for s in SEQ_LENS) - SEQ_BLOCK = 1 if GENERATE_ONLY else 32 - grid = (BATCH_SIZE, N_KV_HEADS, (max(SEQ_LENS) + SEQ_BLOCK - 1) // SEQ_BLOCK) - - seq_len = torch.tensor(SEQ_LENS, device=DEVICE, dtype=torch.int32) - seq_start_indices = torch.zeros(BATCH_SIZE, device=DEVICE, dtype=torch.int32) - seq_start_indices[1:] = torch.cumsum(seq_len[:-1], 0) - - freqs = torch.rand([MAX_SEQ_LEN, D_HEAD // 2, 2], device=DEVICE, dtype=torch.float32) - HEAD_BLOCK_SIZE = max(16, triton.next_power_of_2(N_HEADS // N_KV_HEADS)) - - update_kv_cache_rope_fusion[grid]( - q_f, - k_f, - v_f, - seq_len, - seq_start_indices, - q_rope, - k_cache, - v_cache, - torch.tensor(INPUT_POS, device=DEVICE, dtype=torch.int32), - torch.tensor(SLOT_IDX, device=DEVICE, dtype=torch.int32), - freqs, - MAX_SEQ_LEN, - N_HEADS, - N_KV_HEADS, - D_HEAD, - SEQ_BLOCK, - HEAD_BLOCK_SIZE, - GENERATE_ONLY, - ) - - q_ref = [] - k_ref = [] - for i in range(BATCH_SIZE): - # result of torch reference is in normal order - # simulate interleaved rope result - q_batch = q[i] - k_batch = k[i] - q_ref.append( - torch_rope_reference(q_batch, freqs, INPUT_POS[i]) - .unflatten(-1, (D_HEAD // 2, 2)) - .transpose(-1, -2) - .flatten(-2) - .contiguous() - ) - k_ref.append( - torch_rope_reference(k_batch, freqs, INPUT_POS[i]) - .unflatten(-1, (D_HEAD // 2, 2)) - .transpose(-1, -2) - .flatten(-2) - .contiguous() - ) - - # Check if q_rope was correctly updated: - start = 0 - for i in range(BATCH_SIZE): - end = start + SEQ_LENS[i] - assert torch.allclose( - q_rope[0, start:end].squeeze(), - q_ref[i].squeeze(), - atol=1e-2, - rtol=1e-2, - ) - start = end - - # Check if the cache was correctly updated: - for i, cl in enumerate(SLOT_IDX): - assert torch.allclose( - k_cache[cl, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i]].squeeze(), - k_ref[i].squeeze(), - atol=1e-2, - rtol=1e-2, - ) - assert torch.equal( - v_cache[cl, INPUT_POS[i] : INPUT_POS[i] + SEQ_LENS[i]].squeeze(), - v[i].squeeze(), - ) diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py index f2b3a0ac82ae..74dfe2fe95ff 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kv_cache.py @@ -434,7 +434,7 @@ def test_insert_cached_attention_lowers_semantic_mask_for_torch_backend(): @torch.inference_mode() -def test_insert_cached_attention_lowers_semantic_mask_for_triton_paged_backend(): +def test_insert_cached_attention_lowers_semantic_mask_for_triton_backend(): model = SpanMaskedAttentionModel().eval() input_ids = torch.tensor([[1, 2, 3, 4, 5]], dtype=torch.int64) position_ids = torch.arange(input_ids.shape[1], dtype=torch.int64).repeat(input_ids.shape[0], 1) @@ -459,7 +459,7 @@ def test_insert_cached_attention_lowers_semantic_mask_for_triton_paged_backend() { "insert_cached_attention": { "stage": "cache_init", - "backend": "triton_paged", + "backend": "triton", }, }, )(cm, gm) @@ -471,7 +471,7 @@ def test_insert_cached_attention_lowers_semantic_mask_for_triton_paged_backend() cached_nodes = [ node for node in gm_transformed.graph.nodes - if is_op(node, torch.ops.auto_deploy.triton_paged_mha_with_cache) + if is_op(node, torch.ops.auto_deploy.triton_mha_with_cache) ] assert len(cached_nodes) == 1 assert is_op(cached_nodes[0].args[-1], torch.ops.auto_deploy.gemma4_prepare_multimodal_mask) @@ -503,7 +503,7 @@ def test_insert_cached_attention_rejects_unsupported_semantic_mask_backend(): RuntimeError, match=( "Cached attention backend 'flashinfer' does not support lowering semantic mask op" - ".*gemma4_multimodal_mask.*Supported backends: torch, triton_paged" + ".*gemma4_multimodal_mask.*Supported backends: torch, triton" ), ): InferenceOptimizer( @@ -706,10 +706,10 @@ def test_insert_cached_attention_uses_add_resource(): # Verify resources were added assert len(cm._resource_lookup) > 0 - # Should have k_cache and v_cache resources registered + # Triton attention registers one combined paged KV cache resource. resource_names = list(cm._resource_lookup.keys()) - assert any("k_cache" in name for name in resource_names) - assert any("v_cache" in name for name in resource_names) + assert any(name.endswith("kv_cache") for name in resource_names) + assert any(handler.is_paged for handler in cm._resource_lookup.values()) def test_insert_cached_attention_passes_kv_cache_config(): @@ -784,9 +784,7 @@ def test_insert_cached_attention_passes_kv_cache_config(): # Initialize resources cm.initialize_resources() - assert not any(handler.is_paged for handler in cm._resource_lookup.values()), ( - "triton should not use paged resources" - ) + assert any(handler.is_paged for handler in cm._resource_lookup.values()) assert cm._caches, "at least some resources should be present" # Verify cache dtype matches config @@ -951,7 +949,7 @@ def forward( return x + self.o_proj_1(a1.reshape(b, s, -1)) -def _build_optimizer_with_backend(model, backend="triton_paged"): +def _build_optimizer_with_backend(model, backend="triton"): """Helper to create an InferenceOptimizer for testing.""" return InferenceOptimizer( DummyFactory(model, cache_config_updates={}), @@ -1146,7 +1144,7 @@ def test_kv_cache_manager_initialized_with_sliding_window(): }, "insert_cached_attention": { "stage": "cache_init", - "backend": "triton_paged", + "backend": "triton", }, "initialize_cache": { "stage": "cache_init", diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py index 6e9043bc96b7..2c98ee893551 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_kvcache_vswa_metadata.py @@ -94,7 +94,7 @@ def test_vswa_registers_per_group_seq_len_with_cache_placeholders(): This is required so the SWA group's window-capped value reaches the kernel and the prepare-extra-metadata op. """ - gm, info, cm = _run_transform(backend="triton_paged") + gm, info, cm = _run_transform(backend="triton") assert info.num_matches == 2, "expected 2 cached-attention insertions" names = _placeholder_names(gm) @@ -118,9 +118,9 @@ def test_vswa_extra_metadata_is_per_group(): This is the regression guard against the pre-fix transform behavior described in the autodeploy-kvcache-vswa-meta-nodes-extra backlog. """ - gm, info, cm = _run_transform(backend="triton_paged") + gm, info, cm = _run_transform(backend="triton") - prep_meta_op = torch.ops.auto_deploy.triton_paged_prepare_metadata.default + prep_meta_op = torch.ops.auto_deploy.triton_prepare_metadata.default prep_calls = [ node for node in gm.graph.nodes @@ -161,9 +161,9 @@ def test_vswa_each_layer_routes_to_its_groups_extra_metadata(): The cached-attn call for group 1 (the SWA layer) must consume the group-1 prepare-extra outputs, not group 0's. """ - gm, info, cm = _run_transform(backend="triton_paged") + gm, info, cm = _run_transform(backend="triton") - cached_op = torch.ops.auto_deploy.triton_paged_mha_with_cache.default + cached_op = torch.ops.auto_deploy.triton_mha_with_cache.default cached_calls = [ node for node in gm.graph.nodes if node.op == "call_function" and node.target == cached_op ] diff --git a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_shared_kv_attention.py b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_shared_kv_attention.py index c13270d34fb0..f915fc233945 100644 --- a/tests/unittest/auto_deploy/singlegpu/transformations/library/test_shared_kv_attention.py +++ b/tests/unittest/auto_deploy/singlegpu/transformations/library/test_shared_kv_attention.py @@ -12,9 +12,7 @@ from tensorrt_llm._torch.auto_deploy.custom_ops.attention.torch_backend_attention import ( TorchBackendAttention, ) -from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_paged_attention import ( - TritonPagedAttention, -) +from tensorrt_llm._torch.auto_deploy.custom_ops.attention.triton_attention import TritonAttention from tensorrt_llm._torch.auto_deploy.custom_ops.attention_interface import BatchInfo from tensorrt_llm._torch.auto_deploy.export import torch_export_to_gm from tensorrt_llm._torch.auto_deploy.shim.interface import CachedSequenceInterface @@ -352,7 +350,7 @@ def __init__(self, target): ) -def test_triton_paged_backend_attention_metadata_for_shared_kv_node(): +def test_triton_backend_attention_metadata_for_shared_kv_node(): module = _TinySharedKVModule().eval() gm = torch_export_to_gm(module, (torch.randn(1, 4, 8),)) source_nodes = [ @@ -366,18 +364,18 @@ def test_triton_paged_backend_attention_metadata_for_shared_kv_node(): for node in source_nodes if node.target == torch.ops.auto_deploy.torch_attention.default ) - shared = next(node for node in source_nodes if TritonPagedAttention.get_layer_idx(node) == 1) + shared = next(node for node in source_nodes if TritonAttention.get_layer_idx(node) == 1) - assert TritonPagedAttention.get_layer_idx(regular) == 0 - assert TritonPagedAttention.get_layer_idx(shared) == 1 - assert TritonPagedAttention.get_shared_kv_source_layer_idx(regular) is None - assert TritonPagedAttention.get_shared_kv_source_layer_idx(shared) == 0 - assert TritonPagedAttention.get_cached_attention_op() == ( - torch.ops.auto_deploy.triton_paged_mha_with_cache.default + assert TritonAttention.get_layer_idx(regular) == 0 + assert TritonAttention.get_layer_idx(shared) == 1 + assert TritonAttention.get_shared_kv_source_layer_idx(regular) is None + assert TritonAttention.get_shared_kv_source_layer_idx(shared) == 0 + assert TritonAttention.get_cached_attention_op() == ( + torch.ops.auto_deploy.triton_mha_with_cache.default ) -def test_shared_kv_transform_aliases_source_cache_placeholders_for_triton_paged(): +def test_shared_kv_transform_aliases_source_cache_placeholders_for_triton(): module = _TinySharedKVModule().eval() gm = torch_export_to_gm(module, (torch.randn(1, 4, 8),)) @@ -388,7 +386,7 @@ def test_shared_kv_transform_aliases_source_cache_placeholders_for_triton_paged( device="cpu", ) transform = _InsertCachedOperator( - InsertCachedAttentionConfig(stage=Stages.CACHE_INIT, backend="triton_paged") + InsertCachedAttentionConfig(stage=Stages.CACHE_INIT, backend="triton") ) gm, info = transform._apply(gm, cm, factory=None, shared_config=SharedConfig()) @@ -403,27 +401,27 @@ def test_shared_kv_transform_aliases_source_cache_placeholders_for_triton_paged( regular_node = next( node for node in cached_nodes - if node.target == torch.ops.auto_deploy.triton_paged_mha_with_cache.default + if node.target == torch.ops.auto_deploy.triton_mha_with_cache.default and node.args[-1] is False ) shared_node = next( node for node in cached_nodes - if node.target == torch.ops.auto_deploy.triton_paged_mha_with_cache.default + if node.target == torch.ops.auto_deploy.triton_mha_with_cache.default and node.args[-1] is True ) assert regular_node.args[13] is shared_node.args[13] - assert regular_node.target == torch.ops.auto_deploy.triton_paged_mha_with_cache.default - assert shared_node.target == torch.ops.auto_deploy.triton_paged_mha_with_cache.default + assert regular_node.target == torch.ops.auto_deploy.triton_mha_with_cache.default + assert shared_node.target == torch.ops.auto_deploy.triton_mha_with_cache.default assert regular_node.args[-1] is False assert shared_node.args[-1] is True @torch.no_grad() -def test_triton_paged_shared_kv_cached_attention_reads_aliased_cache_without_writing(): +def test_triton_shared_kv_cached_attention_reads_aliased_cache_without_writing(): if not torch.cuda.is_available(): - pytest.skip("CUDA is required for triton_paged shared-KV runtime coverage") + pytest.skip("CUDA is required for triton shared-KV runtime coverage") device = torch.device("cuda") head_dim = 16 @@ -462,14 +460,14 @@ def test_triton_paged_shared_kv_cached_attention_reads_aliased_cache_without_wri seq_len_with_cache_host = torch.tensor([3], dtype=torch.int32) position_ids = torch.tensor([[0]], dtype=torch.int32, device=device) - triton_batch_indices, triton_positions = torch.ops.auto_deploy.triton_paged_prepare_metadata( + triton_batch_indices, triton_positions = torch.ops.auto_deploy.triton_prepare_metadata( position_ids, batch_info_host.serialize(), cu_seqlen_host.to(device), seq_len_with_cache_host.to(device), ) - output = torch.ops.auto_deploy.triton_paged_mha_with_cache( + output = torch.ops.auto_deploy.triton_mha_with_cache( q, dummy_k, dummy_v, @@ -488,7 +486,7 @@ def test_triton_paged_shared_kv_cached_attention_reads_aliased_cache_without_wri None, True, ) - expected = torch.ops.auto_deploy.triton_paged_mha_with_cache( + expected = torch.ops.auto_deploy.triton_mha_with_cache( q, owner_k[:, 2:3], owner_v[:, 2:3], @@ -514,9 +512,9 @@ def test_triton_paged_shared_kv_cached_attention_reads_aliased_cache_without_wri @torch.no_grad() -def test_triton_paged_shared_kv_prefill_sdpa_reads_aliased_cache_without_writing(): +def test_triton_shared_kv_prefill_sdpa_reads_aliased_cache_without_writing(): if not torch.cuda.is_available(): - pytest.skip("CUDA is required for triton_paged shared-KV SDPA runtime coverage") + pytest.skip("CUDA is required for triton shared-KV SDPA runtime coverage") from unittest.mock import patch @@ -567,7 +565,7 @@ def tracking_sdpa(*args, **kwargs): return original_sdpa(*args, **kwargs) with patch.object(torch.nn.functional, "scaled_dot_product_attention", tracking_sdpa): - output = torch.ops.auto_deploy.triton_paged_mha_with_cache( + output = torch.ops.auto_deploy.triton_mha_with_cache( q, dummy_k, dummy_v, @@ -586,7 +584,7 @@ def tracking_sdpa(*args, **kwargs): None, True, ) - expected = torch.ops.auto_deploy.triton_paged_mha_with_cache( + expected = torch.ops.auto_deploy.triton_mha_with_cache( q, owner_k, owner_v, From 12ea8ef22b4106011cff774b0d85372c1fdb8deb Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:48:50 +0800 Subject: [PATCH 170/308] [None][test] Waive 5 failed cases for main in QA CI (#14789) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 30da247dcaa2..3ece58081333 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -135,6 +135,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_5_9B::test_bf16[mtp_on] SKIP (https: accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response SKIP (https://nvbugs/6200128) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6211189) accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6211189) +accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6215689) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6248744) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) @@ -302,6 +303,7 @@ test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-t test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] SKIP (https://nvbugs/6190759) +test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[etcd] SKIP (https://nvbugs/6211193) test_e2e.py::test_openai_kv_cache_contamination SKIP (https://nvbugs/6227203) test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] SKIP (https://nvbugs/5448523) From b9453fcf76c63c3e744c04b3d136dce3dbaad970 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:49:31 +0800 Subject: [PATCH 171/308] [None][test] Waive 7 failed cases for main in QA CI (#14791) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3ece58081333..d2e7e5ef2c60 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -3,6 +3,7 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[Fals accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6184575) +accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram SKIP (https://nvbugs/6245651) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph SKIP (https://nvbugs/5772995) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] SKIP (https://nvbugs/5346443) @@ -32,6 +33,8 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_s accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_mtp1] SKIP (https://nvbugs/6185196) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp2] SKIP (https://nvbugs/6241842) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp3] SKIP (https://nvbugs/6241845) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[baseline] SKIP (https://nvbugs/6185196) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[mtp3_fp8kv_chunked] SKIP (https://nvbugs/5989920) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) @@ -172,6 +175,8 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) +disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp1-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6223556) +disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6223556) disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) From c482b8151363a24e64923bb3812a538271884ef8 Mon Sep 17 00:00:00 2001 From: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:13:39 -0700 Subject: [PATCH 172/308] [https://nvbugs/6240561][fix] Fix AutoDeploy DeepSeek-R1 accuracy drop (#14793) Signed-off-by: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> --- .../_torch/auto_deploy/utils/fp8_dequant.py | 16 ++++- .../custom_ops/quantization/test_quant.py | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/utils/fp8_dequant.py b/tensorrt_llm/_torch/auto_deploy/utils/fp8_dequant.py index 055aa51d004a..578d32ae72a5 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/fp8_dequant.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/fp8_dequant.py @@ -39,9 +39,19 @@ def dequant_fp8_weight_two_dim_block_grid( """ N, K = weight_fp8.shape scale_n, scale_k = weight_scale.shape - # Ceil division so the expanded scale covers non-divisible dims (see torch_quant). - actual_block_n = math.ceil(N / scale_n) if scale_n > 0 else block_n - actual_block_k = math.ceil(K / scale_k) if scale_k > 0 else block_k + # How each scale value maps to weight rows depends on the scale layout: + # - coarse 128-block scale (scale_n == ceil(N/128)): expand by the canonical 128. Using + # ceil(N/scale_n) equals 128 only when N is a 128-multiple; for a partial last block + # (e.g. N=576 -> 116) it maps scales onto the wrong rows and corrupts the weights. + # - per-row scale (scale_n == N): expand by 1, which ceil(N/scale_n) already yields. + n_is_block_scale = scale_n > 0 and scale_n == math.ceil(N / 128) + k_is_block_scale = scale_k > 0 and scale_k == math.ceil(K / 128) + actual_block_n = ( + 128 if n_is_block_scale else (math.ceil(N / scale_n) if scale_n > 0 else block_n) + ) + actual_block_k = ( + 128 if k_is_block_scale else (math.ceil(K / scale_k) if scale_k > 0 else block_k) + ) scale_expanded = weight_scale.repeat_interleave(actual_block_n, dim=0).repeat_interleave( actual_block_k, dim=1 ) diff --git a/tests/unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py b/tests/unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py index 5d669e7ff480..240904397e39 100644 --- a/tests/unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py +++ b/tests/unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py @@ -12,12 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math + import pytest import torch import torch.nn.functional as F from _torch_test_utils import fp4_compatible, fp8_compatible, trtllm_ops_available import tensorrt_llm._torch.auto_deploy.custom_ops # noqa: F401 +from tensorrt_llm._torch.auto_deploy.utils.fp8_dequant import dequant_fp8_weight_two_dim_block_grid from tensorrt_llm._torch.auto_deploy.utils.quantization_utils import ( fp4_global_scale, pack_int4_in_uint8, @@ -466,3 +469,70 @@ def test_nvfp4_prequant_linear_wrapper_matches_direct_gemm(use_bias, input_dtype assert out_wrapped.shape == out_ref.shape assert out_wrapped.dtype == input_dtype torch.testing.assert_close(out_wrapped, out_ref, rtol=1e-3, atol=5e-3) + + +# --------------------------------------------------------------------------- +# FineGrained FP8 BF16-fallback dequant (dequant_fp8_weight_two_dim_block_grid) +# +# Regression coverage for the block-alignment bug: a coarse 128-block scale whose +# weight dim is NOT a multiple of 128 (e.g. kv_a N=576) was expanded by +# ceil(N/scale_n) (=116) instead of the canonical 128, mapping scales onto the +# wrong rows. These are pure-tensor (CPU-ok) reference comparisons. +# --------------------------------------------------------------------------- +_FG_BLOCK = 128 + + +def _fg_make_block_scaled_weight(n, k): + """Random FP8 weight + a coarse 128-block scale of shape [ceil(N/128), ceil(K/128)].""" + weight = (torch.randn(n, k) * 0.1).to(torch.float8_e4m3fn) + scale = (torch.rand(math.ceil(n / _FG_BLOCK), math.ceil(k / _FG_BLOCK)) + 0.5).to( + torch.bfloat16 + ) + return weight, scale + + +def _fg_reference_128_block(weight, scale): + """Ground truth: expand the block scale by the canonical 128 and clip to [N, K].""" + n, k = weight.shape + expanded = scale.repeat_interleave(_FG_BLOCK, dim=0).repeat_interleave(_FG_BLOCK, dim=1)[:n, :k] + return weight.to(torch.bfloat16) * expanded.to(torch.bfloat16) + + +@pytest.mark.parametrize( + "n,k", + [ + (576, 7168), # kv_a: N not a 128-multiple (the regressing shape) + (192, 256), # N not a 128-multiple + (256, 192), # K not a 128-multiple + (256, 256), # fully aligned (must stay correct) + ], +) +def test_finegrained_fp8_dequant_coarse_block_uses_canonical_128(n, k): + """A coarse 128-block scale must dequant exactly, even when N/K aren't 128-multiples.""" + weight, scale = _fg_make_block_scaled_weight(n, k) + out = dequant_fp8_weight_two_dim_block_grid(weight, scale, _FG_BLOCK, _FG_BLOCK) + torch.testing.assert_close(out, _fg_reference_128_block(weight, scale), atol=0.0, rtol=0.0) + + +def test_finegrained_fp8_dequant_differs_from_buggy_ceil_expansion(): + """Lock in the fix: canonical-128 expansion must differ from the old ceil(N/scale_n) one.""" + n, k = 576, 256 # ceil(576/5) = 116 != 128 -> old behavior was wrong + weight, scale = _fg_make_block_scaled_weight(n, k) + correct = dequant_fp8_weight_two_dim_block_grid(weight, scale, _FG_BLOCK, _FG_BLOCK) + + buggy_block_n = math.ceil(n / scale.shape[0]) + assert buggy_block_n != _FG_BLOCK + buggy_scale = scale.repeat_interleave(buggy_block_n, dim=0).repeat_interleave(_FG_BLOCK, dim=1) + buggy = weight.to(torch.bfloat16) * buggy_scale[:n, :k].to(torch.bfloat16) + assert not torch.allclose(correct, buggy), "fix must not reproduce the misaligned expansion" + + +def test_finegrained_fp8_dequant_per_row_scale_expands_by_one(): + """A per-row scale (scale_n == N) keeps the ceil path -> block_n == 1 (each row its own scale).""" + n, k = 72, 256 + weight = (torch.randn(n, k) * 0.1).to(torch.float8_e4m3fn) + scale = (torch.rand(n, math.ceil(k / _FG_BLOCK)) + 0.5).to(torch.bfloat16) # per-row along N + out = dequant_fp8_weight_two_dim_block_grid(weight, scale, _FG_BLOCK, _FG_BLOCK) + expanded = scale.repeat_interleave(1, dim=0).repeat_interleave(_FG_BLOCK, dim=1)[:n, :k] + expected = weight.to(torch.bfloat16) * expanded.to(torch.bfloat16) + torch.testing.assert_close(out, expected, atol=0.0, rtol=0.0) From 6222112ff9a487a9e307583a62a537a2f1436776 Mon Sep 17 00:00:00 2001 From: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:17:27 -0700 Subject: [PATCH 173/308] [#14588][fix] [AutoDeploy] Fix OOM of DeepSeek-R1 NVFP4 for tp=4 (#14477) Signed-off-by: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) --- .../transform/library/fused_moe.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fused_moe.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fused_moe.py index 99293eceade7..aac21bac9af0 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fused_moe.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fused_moe.py @@ -1593,17 +1593,19 @@ def remove_original_experts(gm: GraphModule, weight_lists: List[List[Node]]) -> weight_lists_flat = [w for weights in weight_lists for w in weights] for w in weight_lists_flat: - w_param = get_attr_by_name(gm, w.target) - if w_param is not None: - owner_module, owner_module_path, param_name = get_submodule_of_param(gm, w.target) - owner_param = get_attr_by_name(owner_module, param_name) - if owner_param is w_param: - gm.delete_submodule(owner_module_path) - else: - # param w is not owned by owner_module, skip - continue - else: + # Experts may share an owning container (e.g. nn.ParameterList): deleting it for the + # first expert removes its siblings too, so later targets resolve to a missing attr. + try: + w_param = get_attr_by_name(gm, w.target) + except AttributeError: + continue + if w_param is None: continue + owner_module, owner_module_path, param_name = get_submodule_of_param(gm, w.target) + owner_param = get_attr_by_name(owner_module, param_name) + # delete_submodule requires a non-empty submodule path; root params can't be dropped this way. + if owner_param is w_param and owner_module_path: + gm.delete_submodule(owner_module_path) def _stack_fp8_moe_weights( @@ -2297,7 +2299,15 @@ def _prepare_args_cutlass_format_nvfp4(): ) node.replace_all_uses_with(new_node) + input_nodes = node.all_input_nodes graph.erase_node(node) + for input_node in input_nodes: + if input_node.op == "get_attr" and len(input_node.users) == 0: + graph.erase_node(input_node) + # Free per-layer to avoid accumulating ~ per_layer_weight_bytes across all MoE layers. + # Only pass weight lists; scales/alphas live as buffers under the same submodules and + # are removed together when delete_submodule(...) drops the owning module. + remove_original_experts(gm, [w1_list, w2_list, w3_list]) # Clean up after processing all nodes # eliminate_dead_code will remove unused get_attr nodes, then delete_all_unused_submodules @@ -2854,7 +2864,15 @@ def _shuffle_scale_stack(scale_3d_u8: torch.Tensor, is_gated: bool) -> torch.Ten ) node.replace_all_uses_with(new_node) + input_nodes = node.all_input_nodes graph.erase_node(node) + for input_node in input_nodes: + if input_node.op == "get_attr" and len(input_node.users) == 0: + graph.erase_node(input_node) + # Free per-layer to avoid accumulating ~ per_layer_weight_bytes across all MoE layers. + # Only pass weight lists; scales/alphas live as buffers under the same submodules and + # are removed together when delete_submodule(...) drops the owning module. + remove_original_experts(gm, [w1_list, w2_list, w3_list]) fused_key_counter += 1 eliminate_dead_code(gm) From 5db9414cbeefd0f53cb5c8fd9a80927feba17fd4 Mon Sep 17 00:00:00 2001 From: Yibin Li <109242046+yibinl-nvidia@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:02:59 -0400 Subject: [PATCH 174/308] [https://nvbugs/6179761][fix] Save LTX-2 BF16 weights to speed up perf (#14639) Signed-off-by: yibinl-nvidia <109242046+yibinl-nvidia@users.noreply.github.com> --- .../models/ltx2/pipeline_ltx2_two_stages.py | 71 ++++++++++++++---- .../_torch/visual_gen/test_ltx2_pipeline.py | 73 +++++++++++++++++-- 2 files changed, 122 insertions(+), 22 deletions(-) diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py index 1747df003619..6a8e365d41cf 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py @@ -39,6 +39,8 @@ STAGE_2_DISTILLED_SIGMA_VALUES = [0.909375, 0.725, 0.421875, 0.0] _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2) +# Baseline BF16 peak memory ~75 GiB, saving BF16 weights snopshot total ~108 GiB. +_BF16_WEIGHTS_SNAPSHOT_FREE_MEMORY_THRESHOLD_GIB = 115.0 # --------------------------------------------------------------------------- @@ -46,6 +48,37 @@ # --------------------------------------------------------------------------- +def _get_free_gpu_memory_gib() -> Optional[float]: + """Return free memory on the current CUDA device, or ``None`` if unavailable.""" + if not torch.cuda.is_available(): + return None + + try: + free_bytes, _ = torch.cuda.mem_get_info() + except (RuntimeError, OSError) as exc: + logger.warning(f"Unable to query CUDA free memory for BF16 weight snapshots: {exc}") + return None + + return free_bytes / (1024**3) + + +def _should_save_bf16_weights( + threshold_gib: float = _BF16_WEIGHTS_SNAPSHOT_FREE_MEMORY_THRESHOLD_GIB, +) -> bool: + free_gib = _get_free_gpu_memory_gib() + if free_gib is None: + logger.debug("BF16 weight snapshots disabled: CUDA free memory is unavailable") + return False + + save_state = free_gib > threshold_gib + relation = ">" if save_state else "<=" + logger.debug( + f"BF16 weight snapshots {'enabled' if save_state else 'disabled'}: " + f"free GPU memory {free_gib:.2f} GiB {relation} {threshold_gib:.2f} GiB threshold" + ) + return save_state + + def _load_lora_deltas( lora_path: str, transformer: torch.nn.Module, @@ -338,11 +371,12 @@ def _apply_lora_deltas( module: torch.nn.Module, deltas: Dict[str, torch.Tensor], sign: float = 1.0, + save_bf16_weights: bool = False, ) -> tuple: """Add (sign=+1) or remove (sign=-1) pre-computed LoRA deltas. - For standard (BF16/FP16) weights the delta is added directly and - later removed by subtracting the same delta. + For BF16 weights the delta is added directly and later removed either + by restoring an optional saved snapshot or by subtracting the same delta. For FP8-quantized weights (same shape, float8 dtype), we dequantize → apply → requantize. FP4 weights are handled through the packed-FP4 branch because the current static and dynamic NVFP4 @@ -355,13 +389,15 @@ def _apply_lora_deltas( state afterwards. Returns ``(applied_count, saved_lora_state, snapshot_required_count)`` where - *saved_lora_state* maps each touched quantized parameter name to its - original tensor. For packed FP4 it also stores the parent + *saved_lora_state* maps each touched snapshotted parameter name to its + original tensor. BF16 weights are snapshotted only when + *save_bf16_weights* is true. This does not change FP8 or FP4 LoRA + handling: those paths always snapshot quantized state. For packed FP4 it + also stores the parent ``quant_method`` so that stage 2 can run with BF16 weights and then restore the exact original FP4 state without another quantization round - trip. Dense BF16/FP16/FP32 weights are not stored in - *saved_lora_state*. *snapshot_required_count* is the number of - quantized weights that must be restored from saved snapshots. + trip. *snapshot_required_count* is the number of weights that must be + restored from saved snapshots. """ applied = 0 snapshot_required = 0 @@ -424,8 +460,11 @@ def _apply_lora_deltas( ws_param.data.copy_(new_scale) snapshot_required += 1 else: - # BF16/FP16/FP32: direct in-place addition. These are - # restored by subtracting the same delta after stage 2. + if save_bf16_weights and param.dtype == torch.bfloat16: + saved_state[param_name] = param.data.clone() + snapshot_required += 1 + # BF16: direct in-place addition, then restore by snapshot copy + # when memory allows. Other dense weights restore by subtraction. param.data.add_( delta.to(param.device, param.dtype), alpha=sign, @@ -808,11 +847,16 @@ def forward( # ================================================================ # For FP4 models (static-packed or dynamic), stage 2 always runs in # BF16: the quant_method is swapped to UnquantizedLinearMethod inside - # _apply_lora_deltas and restored afterwards. BF16 models are unaffected. + # _apply_lora_deltas and restored afterwards. FP8 handling is also + # unchanged and always restores from saved quantized state. BF16 weights + # save snapshots only when enough free GPU memory is available; + # otherwise they fall back to on-the-fly LoRA subtraction. + save_bf16_weights = _should_save_bf16_weights() n, saved_lora_state, snapshot_required = _apply_lora_deltas( self.transformer, self._distilled_lora_deltas, sign=1.0, + save_bf16_weights=save_bf16_weights, ) logger.info(f"Merged distilled LoRA ({n} params) for stage 2 (BF16 weights)") @@ -840,8 +884,7 @@ def forward( self.transformer.set_ulysses_enabled(True) if snapshot_required and not saved_lora_state: raise RuntimeError( - "Quantized LoRA state was not saved; cannot safely restore " - "FP8/FP4 stage 2 weights." + "LoRA state was not saved; cannot safely restore stage 2 weights." ) snapshot_restored = 0 @@ -851,8 +894,8 @@ def forward( _restore_lora_state(self.transformer, saved_lora_state) snapshot_restored = _count_saved_lora_weight_tensors(saved_lora_state) - # Dense BF16/FP16/FP32 weights are not snapshotted; restore them by - # subtracting the LoRA deltas directly. + # BF16 weights that were not snapshotted, plus any other dense + # floating-point weights, are restored by subtracting LoRA deltas. dense_restored = _subtract_dense_lora_deltas( self.transformer, self._distilled_lora_deltas, diff --git a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py index c9a38ad819cb..a5109b3cb01e 100644 --- a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -537,6 +537,58 @@ def test_latent_shape_matches_batch(self): class TestTwoStageLoRAHelpers: """Test LoRA delta loading and application without checkpoints.""" + def test_bf16_weight_snapshot_gate_uses_cuda_free_memory(self, monkeypatch): + from tensorrt_llm._torch.visual_gen.models.ltx2.pipeline_ltx2_two_stages import ( + _should_save_bf16_weights, + ) + + gib = 1024**3 + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "mem_get_info", lambda: (116 * gib, 180 * gib)) + assert _should_save_bf16_weights() + + monkeypatch.setattr(torch.cuda, "mem_get_info", lambda: (115 * gib, 180 * gib)) + assert not _should_save_bf16_weights() + + def _raise_mem_query_error(): + raise RuntimeError("mem_get_info failed") + + monkeypatch.setattr(torch.cuda, "mem_get_info", _raise_mem_query_error) + assert not _should_save_bf16_weights() + + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + assert not _should_save_bf16_weights() + + def test_bf16_weight_snapshot_saved_when_requested(self): + from tensorrt_llm._torch.visual_gen.models.ltx2.pipeline_ltx2_two_stages import ( + _apply_lora_deltas, + _restore_lora_state, + _subtract_dense_lora_deltas, + ) + + linear = torch.nn.Linear(32, 32, bias=False).bfloat16() + original_weight = linear.weight.data.clone() + deltas = {"weight": torch.randn(32, 32) * 0.1} + + applied, saved_state, snapshot_required = _apply_lora_deltas( + linear, + deltas, + sign=1.0, + save_bf16_weights=True, + ) + + assert applied == 1 + assert snapshot_required == 1 + assert "weight" in saved_state + assert torch.allclose(saved_state["weight"], original_weight) + assert not torch.allclose(linear.weight.data, original_weight) + + removed = _subtract_dense_lora_deltas(linear, deltas, saved_state) + assert removed == 0 + + _restore_lora_state(linear, saved_state) + assert torch.allclose(linear.weight.data, original_weight) + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") def test_apply_and_remove_deltas_bf16(self): """Merge then unmerge in BF16 should leave weights approximately unchanged.""" @@ -554,7 +606,7 @@ def test_apply_and_remove_deltas_bf16(self): applied, saved_state, snapshot_required = _apply_lora_deltas(linear, deltas, sign=1.0) assert applied == 1, "Expected one parameter to be modified" - assert saved_state == {}, "Dense BF16 weights should not be snapshotted" + assert saved_state == {}, "BF16 weights should not be snapshotted by default" assert snapshot_required == 0 assert not torch.allclose(linear.weight.data, original_weight), ( "Weights should have changed after applying delta" @@ -597,7 +649,7 @@ def test_multi_round_drift_bounded(self): rounds = 10 for _ in range(rounds): _, saved_state, snapshot_required = _apply_lora_deltas(model, deltas, sign=1.0) - assert saved_state == {}, "Dense BF16 weights should not be snapshotted" + assert saved_state == {}, "BF16 weights should not be snapshotted by default" assert snapshot_required == 0 _subtract_dense_lora_deltas(model, deltas, saved_state) @@ -605,8 +657,8 @@ def test_multi_round_drift_bounded(self): assert drift < 0.1, f"bf16 drift after {rounds} rounds too large: {drift:.2e}" @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") - def test_dense_lora_state_not_saved_and_subtract_restores(self): - """Dense weights are restored by subtraction without snapshot storage.""" + def test_fp32_state_not_saved_and_subtract_restores(self): + """FP32 weights restore by subtraction, even when BF16 snapshots are enabled.""" from tensorrt_llm._torch.visual_gen.models.ltx2.pipeline_ltx2_two_stages import ( _apply_lora_deltas, _subtract_dense_lora_deltas, @@ -617,16 +669,21 @@ def test_dense_lora_state_not_saved_and_subtract_restores(self): original_weight = linear.weight.data.clone() deltas = {"weight": torch.randn(32, 32, device=device) * 0.1} - _, saved_state, snapshot_required = _apply_lora_deltas(linear, deltas, sign=1.0) + _, saved_state, snapshot_required = _apply_lora_deltas( + linear, + deltas, + sign=1.0, + save_bf16_weights=True, + ) - assert saved_state == {}, "Dense FP32 weights should not be snapshotted" + assert saved_state == {}, "FP32 weights should not use the BF16 snapshot path" assert snapshot_required == 0 assert not torch.allclose(linear.weight.data, original_weight) removed = _subtract_dense_lora_deltas(linear, deltas, saved_state) assert removed == 1 assert torch.allclose(linear.weight.data, original_weight), ( - "Dense LoRA subtraction should restore weights" + "FP32 LoRA subtraction should restore weights" ) @@ -1111,7 +1168,7 @@ def test_two_stage_lora_deltas_match_transformer(self, ltx2_two_stage_assets_exi print(f"\n[Two-Stage] LoRA apply rate: {match_rate:.1f}% ({applied}/{total})") assert match_rate > 99.0, f"Expected >99% LoRA match rate, got {match_rate:.1f}%" - assert saved_state == {}, "BF16 checkpoint should not snapshot dense weights" + assert saved_state == {}, "BF16 checkpoint should not snapshot weights by default" assert snapshot_required == 0 # Verify dense unmerge by subtraction From 4ba59c0e78ddac295099ebc6b69953f253066bb8 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 03:10:35 +0000 Subject: [PATCH 175/308] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- security_scanning/examples/apps/poetry.lock | 6 +- .../examples/auto_deploy/poetry.lock | 246 +++++----- .../examples/draft_target_model/poetry.lock | 246 +++++----- security_scanning/examples/eagle/poetry.lock | 246 +++++----- .../llm-eval/lm-eval-harness/poetry.lock | 246 +++++----- .../examples/lookahead/poetry.lock | 246 +++++----- security_scanning/examples/medusa/poetry.lock | 246 +++++----- .../models/contrib/baichuan/poetry.lock | 246 +++++----- .../examples/models/contrib/bloom/poetry.lock | 246 +++++----- .../models/contrib/chatglm-6b/poetry.lock | 246 +++++----- .../models/contrib/chatglm2-6b/poetry.lock | 246 +++++----- .../contrib/chatglm3-6b-32k/poetry.lock | 246 +++++----- .../examples/models/contrib/dbrx/poetry.lock | 246 +++++----- .../models/contrib/deepseek_v1/poetry.lock | 246 +++++----- .../models/contrib/deepseek_v2/poetry.lock | 246 +++++----- .../models/contrib/falcon/poetry.lock | 246 +++++----- .../examples/models/contrib/gptj/poetry.lock | 246 +++++----- .../models/contrib/gptneox/poetry.lock | 246 +++++----- .../examples/models/contrib/grok/poetry.lock | 246 +++++----- .../models/contrib/internlm/poetry.lock | 246 +++++----- .../examples/models/contrib/jais/poetry.lock | 246 +++++----- .../examples/models/contrib/mmdit/poetry.lock | 42 +- .../examples/models/contrib/mpt/poetry.lock | 246 +++++----- .../examples/models/contrib/opt/poetry.lock | 246 +++++----- .../models/contrib/skywork/poetry.lock | 246 +++++----- .../examples/models/contrib/smaug/poetry.lock | 246 +++++----- .../examples/models/core/commandr/poetry.lock | 246 +++++----- .../examples/models/core/gemma/poetry.lock | 246 +++++----- .../examples/models/core/glm-4-9b/poetry.lock | 246 +++++----- .../examples/models/core/gpt/poetry.lock | 246 +++++----- .../examples/models/core/llama/poetry.lock | 246 +++++----- .../examples/models/core/mamba/poetry.lock | 246 +++++----- .../examples/models/core/nemotron/poetry.lock | 246 +++++----- .../examples/models/core/phi/poetry.lock | 246 +++++----- .../examples/models/core/qwen/poetry.lock | 254 +++++----- .../examples/models/core/qwen/pyproject.toml | 2 +- .../models/core/qwen2audio/poetry.lock | 246 +++++----- .../examples/models/core/qwenvl/poetry.lock | 246 +++++----- .../models/core/recurrentgemma/poetry.lock | 246 +++++----- .../examples/models/core/whisper/poetry.lock | 246 +++++----- security_scanning/examples/ngram/poetry.lock | 246 +++++----- .../examples/quantization/poetry.lock | 246 +++++----- .../examples/ray_orchestrator/poetry.lock | 364 +++++++-------- .../examples/redrafter/poetry.lock | 246 +++++----- security_scanning/examples/serve/poetry.lock | 32 +- .../examples/trtllm-eval/poetry.lock | 246 +++++----- security_scanning/metadata.json | 4 +- security_scanning/poetry.lock | 436 +++++++++--------- security_scanning/pyproject.toml | 2 +- security_scanning/triton_backend/poetry.lock | 246 +++++----- 50 files changed, 5603 insertions(+), 5625 deletions(-) diff --git a/security_scanning/examples/apps/poetry.lock b/security_scanning/examples/apps/poetry.lock index 793fdbb76a67..8ad66cc2de75 100644 --- a/security_scanning/examples/apps/poetry.lock +++ b/security_scanning/examples/apps/poetry.lock @@ -282,14 +282,14 @@ files = [ [[package]] name = "openai" -version = "2.38.0" +version = "2.40.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c"}, - {file = "openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3"}, + {file = "openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff"}, + {file = "openai-2.40.0.tar.gz", hash = "sha256:9a756f91f274a24ad6026cbcb2042fd356c8d4a10e8f347b08d34465e585f7a2"}, ] [package.dependencies] diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index e5e04ecc6a41..d493cd26c661 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -110,132 +110,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -246,10 +245,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/draft_target_model/poetry.lock b/security_scanning/examples/draft_target_model/poetry.lock index 368d809f4267..ad163ae2e9df 100644 --- a/security_scanning/examples/draft_target_model/poetry.lock +++ b/security_scanning/examples/draft_target_model/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/eagle/poetry.lock b/security_scanning/examples/eagle/poetry.lock index 73d25b03ea36..13ff80c8f683 100644 --- a/security_scanning/examples/eagle/poetry.lock +++ b/security_scanning/examples/eagle/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index cf6f9c6a75ae..1ca4024f5e3b 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -59,132 +59,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -195,10 +194,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/lookahead/poetry.lock b/security_scanning/examples/lookahead/poetry.lock index 368d809f4267..ad163ae2e9df 100644 --- a/security_scanning/examples/lookahead/poetry.lock +++ b/security_scanning/examples/lookahead/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/medusa/poetry.lock b/security_scanning/examples/medusa/poetry.lock index 368d809f4267..ad163ae2e9df 100644 --- a/security_scanning/examples/medusa/poetry.lock +++ b/security_scanning/examples/medusa/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/baichuan/poetry.lock b/security_scanning/examples/models/contrib/baichuan/poetry.lock index 03c1f0e7ce8e..055673bd2a41 100644 --- a/security_scanning/examples/models/contrib/baichuan/poetry.lock +++ b/security_scanning/examples/models/contrib/baichuan/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/bloom/poetry.lock b/security_scanning/examples/models/contrib/bloom/poetry.lock index dc682784cc28..65d39788db17 100644 --- a/security_scanning/examples/models/contrib/bloom/poetry.lock +++ b/security_scanning/examples/models/contrib/bloom/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock index eaaf906664b5..30900cf5e0be 100644 --- a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock index eaaf906664b5..30900cf5e0be 100644 --- a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock index eaaf906664b5..30900cf5e0be 100644 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/dbrx/poetry.lock b/security_scanning/examples/models/contrib/dbrx/poetry.lock index e8979a0c7455..74aeddde2e13 100644 --- a/security_scanning/examples/models/contrib/dbrx/poetry.lock +++ b/security_scanning/examples/models/contrib/dbrx/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v1/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock index 83b7157abd35..2d4c50596cdd 100644 --- a/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock +++ b/security_scanning/examples/models/contrib/deepseek_v2/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/falcon/poetry.lock b/security_scanning/examples/models/contrib/falcon/poetry.lock index 804f96084df6..67ab7f11f29c 100644 --- a/security_scanning/examples/models/contrib/falcon/poetry.lock +++ b/security_scanning/examples/models/contrib/falcon/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/gptj/poetry.lock b/security_scanning/examples/models/contrib/gptj/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/contrib/gptj/poetry.lock +++ b/security_scanning/examples/models/contrib/gptj/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/gptneox/poetry.lock b/security_scanning/examples/models/contrib/gptneox/poetry.lock index 73d25b03ea36..13ff80c8f683 100644 --- a/security_scanning/examples/models/contrib/gptneox/poetry.lock +++ b/security_scanning/examples/models/contrib/gptneox/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/grok/poetry.lock b/security_scanning/examples/models/contrib/grok/poetry.lock index 41a75eaabd28..8373a68565ce 100644 --- a/security_scanning/examples/models/contrib/grok/poetry.lock +++ b/security_scanning/examples/models/contrib/grok/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/internlm/poetry.lock b/security_scanning/examples/models/contrib/internlm/poetry.lock index 368d809f4267..ad163ae2e9df 100644 --- a/security_scanning/examples/models/contrib/internlm/poetry.lock +++ b/security_scanning/examples/models/contrib/internlm/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/jais/poetry.lock b/security_scanning/examples/models/contrib/jais/poetry.lock index dc682784cc28..65d39788db17 100644 --- a/security_scanning/examples/models/contrib/jais/poetry.lock +++ b/security_scanning/examples/models/contrib/jais/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/mmdit/poetry.lock b/security_scanning/examples/models/contrib/mmdit/poetry.lock index cdaa56ecf86e..e06da37ae370 100644 --- a/security_scanning/examples/models/contrib/mmdit/poetry.lock +++ b/security_scanning/examples/models/contrib/mmdit/poetry.lock @@ -981,35 +981,35 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "safetensors" -version = "0.8.0rc0" +version = "0.8.0rc1" description = "" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "safetensors-0.8.0rc0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c1e7a6a1c0dd0128888bc47aca0a9625855673f44f275bf4073088563bf7121b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c052d1706567487bc103088fe02daf05132dbccbbc3d798753541b66eb37fb14"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79547625fa84f3a9b28b933e44c67d012edf22a0c7170ed68835b9f467dda836"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a132d3cf5f63c3f02b82c4abf65c58d33a8422199ae34e09a9a7edb661bd2ca9"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d42f6c44773901ce1a021d2372747a559e9ec5aa59d044c0d711c273bff21c67"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b40d25911c5f241cad874ad1ea4100a9a9e3c2d469a73a38b47af759d239f44"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf0d366f75f63867f1ede90f87090450c7cec320da1fc2a597f9bb8cb73460db"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:50c56d7b6a2f44c3f4ab130bfeb6a8a51ce72bec152805f9c5a46bdf6addb6c5"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:94d9c0d569a124fe3074b9934031c2cdcfab12d4d7b64ae17343fac4a92081e8"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b4fcccda047df747e2463744428cba352d99527c4e52545d07f8c3a8583136f1"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:2ef8ab6704ea895cb13c89d5825f49e87328cac2093e7e45fb3cb615bd457fb2"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:35bf158d1555df7a529c844ae8ab89355c9df34546de0f94c47d538902bcc07c"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:98b0f6f2a14a6bde7f6acaa5f0381baef9a87c6a3124338affe4e4bb40bf826b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win32.whl", hash = "sha256:7e7cc49c69d8df5aaaf332532cd636609727599f81294bf4e5de56a2e3b70a10"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win_amd64.whl", hash = "sha256:d6532e381c492f5a6b4e82706b232f003e9e697b77d6c2eb7e806d11b578d00b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win_arm64.whl", hash = "sha256:b7f8180f8c119dce85da7913904ccf4a0227adf095eb63f1732a6729c2672cb1"}, - {file = "safetensors-0.8.0rc0.tar.gz", hash = "sha256:b4168a839ff287dc26b0d843e1760962b2e92ed5645f95e8ab3f4b9401807e6a"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7e57730ae523085fda4a80eef74ad40c6d67af60b38498d9995cc9fcb639103b"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ba66ebb7eaa5914ff41cd2b2cd7ccd22c844854be2a5179289e748c70ffa90a4"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a72817e309ed17a6b805168bca500af711c8bf50cccf7bf790ad247788c676c"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c945f1ec6fc5a04abc174e79bce69c4613ae536c5276a3812a2a89b62ae009d7"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba0397739b71400eab5d1f492d68484595cf8b5d13dbfef274e3bcf518348bd8"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f61b7d2c1babc6271d778138788c57c8a95898be6ad3b559971fce20ec1c9c23"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53f59971f435fb5c23bb7bfa3c00e6cdbb26486d41f3aaf04c6d9e2d91bc8e4c"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:b070ba9428e6b2b3b820152b1e1583931cfbd692cda595442d5fbc8b5a46e824"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:69f73c2ec4f76e89deaefe485fec0c6fc42c04a70e318aaefd235da8edacccb1"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87aad206a0bb02fa3ddaeb2feb1c6f7f39a87bea1976750ba28a857fbfcd6b31"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:6ea00be4f7066ba834064fd7b104ba4b10d2e42cd915c9ece31f0e2b42e6b6d2"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:2445135cda6018a095ed951dec94a2c393f929ee3b7f7a9ca4630db854be6b89"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf949f6a37287572de1c47294b36131bf49528436724eec2f96015f75a3d0bc8"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win32.whl", hash = "sha256:4633355aaa0da80e789cb7c014c462b6dabe0ecc5f5bb56738fca01511b99975"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win_amd64.whl", hash = "sha256:d62fad383627979d80b640174c679f3304b3a21614c4d8e71f76910aecac9c8d"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win_arm64.whl", hash = "sha256:2b8ce46f7f16376eaf7527ee1dc830a32f74542767c0779f446f76d0be6b5da8"}, + {file = "safetensors-0.8.0rc1.tar.gz", hash = "sha256:a4bacbcd2ab9efe4eb5f1ea44afc9ac5f3b40e103ebde146e370e885ba46f2fc"}, ] [package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]"] +dev = ["safetensors[all]", "safetensors[pinned-tf]"] jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] mlx = ["mlx (>=0.0.9)"] numpy = ["numpy (>=1.24.6)"] @@ -1018,7 +1018,7 @@ pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] quality = ["ruff"] tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -testingfree = ["hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] +tf-nightly = ["safetensors[numpy]", "tf-nightly"] torch = ["safetensors[numpy]", "torch (>=2.4)"] [[package]] diff --git a/security_scanning/examples/models/contrib/mpt/poetry.lock b/security_scanning/examples/models/contrib/mpt/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/contrib/mpt/poetry.lock +++ b/security_scanning/examples/models/contrib/mpt/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/opt/poetry.lock b/security_scanning/examples/models/contrib/opt/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/contrib/opt/poetry.lock +++ b/security_scanning/examples/models/contrib/opt/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/skywork/poetry.lock b/security_scanning/examples/models/contrib/skywork/poetry.lock index dc682784cc28..65d39788db17 100644 --- a/security_scanning/examples/models/contrib/skywork/poetry.lock +++ b/security_scanning/examples/models/contrib/skywork/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/contrib/smaug/poetry.lock b/security_scanning/examples/models/contrib/smaug/poetry.lock index dc682784cc28..65d39788db17 100644 --- a/security_scanning/examples/models/contrib/smaug/poetry.lock +++ b/security_scanning/examples/models/contrib/smaug/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/commandr/poetry.lock b/security_scanning/examples/models/core/commandr/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/core/commandr/poetry.lock +++ b/security_scanning/examples/models/core/commandr/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/gemma/poetry.lock b/security_scanning/examples/models/core/gemma/poetry.lock index bdd43e9bee06..92cd9bb33b1a 100644 --- a/security_scanning/examples/models/core/gemma/poetry.lock +++ b/security_scanning/examples/models/core/gemma/poetry.lock @@ -38,132 +38,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -174,10 +173,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/glm-4-9b/poetry.lock b/security_scanning/examples/models/core/glm-4-9b/poetry.lock index eaaf906664b5..30900cf5e0be 100644 --- a/security_scanning/examples/models/core/glm-4-9b/poetry.lock +++ b/security_scanning/examples/models/core/glm-4-9b/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/gpt/poetry.lock b/security_scanning/examples/models/core/gpt/poetry.lock index dc682784cc28..65d39788db17 100644 --- a/security_scanning/examples/models/core/gpt/poetry.lock +++ b/security_scanning/examples/models/core/gpt/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/llama/poetry.lock b/security_scanning/examples/models/core/llama/poetry.lock index a0bd689a9c87..9697792c476e 100644 --- a/security_scanning/examples/models/core/llama/poetry.lock +++ b/security_scanning/examples/models/core/llama/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/mamba/poetry.lock b/security_scanning/examples/models/core/mamba/poetry.lock index 7feb4e0f77e1..b62dbb0dbe72 100644 --- a/security_scanning/examples/models/core/mamba/poetry.lock +++ b/security_scanning/examples/models/core/mamba/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/nemotron/poetry.lock b/security_scanning/examples/models/core/nemotron/poetry.lock index 297d2dac7d24..645a66767965 100644 --- a/security_scanning/examples/models/core/nemotron/poetry.lock +++ b/security_scanning/examples/models/core/nemotron/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/phi/poetry.lock b/security_scanning/examples/models/core/phi/poetry.lock index dce1be46e2fd..af2ab71f6e4b 100644 --- a/security_scanning/examples/models/core/phi/poetry.lock +++ b/security_scanning/examples/models/core/phi/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/qwen/poetry.lock b/security_scanning/examples/models/core/qwen/poetry.lock index 8078881ef87d..03b23490ca7e 100644 --- a/security_scanning/examples/models/core/qwen/poetry.lock +++ b/security_scanning/examples/models/core/qwen/poetry.lock @@ -38,132 +38,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -174,10 +173,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiohttp-sse-client" @@ -1706,14 +1706,14 @@ files = [ [[package]] name = "openai" -version = "2.38.0" +version = "2.40.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c"}, - {file = "openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3"}, + {file = "openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff"}, + {file = "openai-2.40.0.tar.gz", hash = "sha256:9a756f91f274a24ad6026cbcb2042fd356c8d4a10e8f347b08d34465e585f7a2"}, ] [package.dependencies] @@ -3712,4 +3712,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "875c0b8dca04bc34d76fbbd349e66235063c8433ee5d4baf5a2337286367ede2" +content-hash = "3b711af70db45a67fd686b90123891638877ef237a3bd5d7659dc002c5917a17" diff --git a/security_scanning/examples/models/core/qwen/pyproject.toml b/security_scanning/examples/models/core/qwen/pyproject.toml index d2108040cc3c..1c9ef820d4b5 100644 --- a/security_scanning/examples/models/core/qwen/pyproject.toml +++ b/security_scanning/examples/models/core/qwen/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "mdtex2html (>=1.3.2,<2.0.0)", "sse-starlette (>=3.4.4,<4.0.0)", "aiohttp-sse-client (>=0.2.1,<0.3.0)", - "openai (>=2.38.0,<3.0.0)" + "openai (>=2.40.0,<3.0.0)" ] diff --git a/security_scanning/examples/models/core/qwen2audio/poetry.lock b/security_scanning/examples/models/core/qwen2audio/poetry.lock index f3132bb8b010..66843173078d 100644 --- a/security_scanning/examples/models/core/qwen2audio/poetry.lock +++ b/security_scanning/examples/models/core/qwen2audio/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 83eb04386171..29cb9d0e9e7a 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/recurrentgemma/poetry.lock b/security_scanning/examples/models/core/recurrentgemma/poetry.lock index 58cb2b6ffe4b..1530af8fbb05 100644 --- a/security_scanning/examples/models/core/recurrentgemma/poetry.lock +++ b/security_scanning/examples/models/core/recurrentgemma/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/models/core/whisper/poetry.lock b/security_scanning/examples/models/core/whisper/poetry.lock index 0997bda7deed..05d15c574dcc 100644 --- a/security_scanning/examples/models/core/whisper/poetry.lock +++ b/security_scanning/examples/models/core/whisper/poetry.lock @@ -14,132 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -150,10 +149,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/ngram/poetry.lock b/security_scanning/examples/ngram/poetry.lock index 7b20c0a00de0..1cb1abc25378 100644 --- a/security_scanning/examples/ngram/poetry.lock +++ b/security_scanning/examples/ngram/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/quantization/poetry.lock b/security_scanning/examples/quantization/poetry.lock index a513fa11f384..3b25e4775ad6 100644 --- a/security_scanning/examples/quantization/poetry.lock +++ b/security_scanning/examples/quantization/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 02cdf3669285..8316fd5fc2e4 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -14,132 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -150,10 +149,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiohttp-cors" @@ -816,80 +816,70 @@ grpc = ["grpcio (>=1.44.0,<2.0.0)"] [[package]] name = "grpcio" -version = "1.80.0" +version = "1.81.0" description = "HTTP/2-based RPC framework" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, - {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81"}, - {file = "grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069"}, - {file = "grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58"}, - {file = "grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a"}, - {file = "grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6"}, - {file = "grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440"}, - {file = "grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9"}, - {file = "grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0"}, - {file = "grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f"}, - {file = "grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193"}, - {file = "grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff"}, - {file = "grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad"}, - {file = "grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7"}, - {file = "grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294"}, - {file = "grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50"}, - {file = "grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e"}, - {file = "grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e"}, - {file = "grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae"}, - {file = "grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f"}, - {file = "grpcio-1.80.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4"}, - {file = "grpcio-1.80.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614"}, - {file = "grpcio-1.80.0-cp39-cp39-win32.whl", hash = "sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141"}, - {file = "grpcio-1.80.0-cp39-cp39-win_amd64.whl", hash = "sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de"}, - {file = "grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257"}, + {file = "grpcio-1.81.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:b4108e5d9d0f651b7eea749116181fe6c315b145661a80ec31f05ec2dbe21af7"}, + {file = "grpcio-1.81.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:b76ea9d55cd08fcdbda25d28e0f76679536710acb7fbd5b1f70cb4ac49317265"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4e032feb3bfb4e2749b140a2302a6baa8ead1b9781ff5cf7094e4402b5e9372e"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:725801c7086d7e4cd160e42bb2f54e0aeb976b9568df3cc6f843b15d29b79fb1"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f750a091fff3a3991731abc1f818bdc64874bb3528162732cb4d45f2e07821a6"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8226ba097eed660ef14d36c6a69b85038552bb8b6d17b44a5aa6f9abf48b8e08"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:40edffb4ec3689373825d367c4457727047a6e554f03245265ecc8cc03215f22"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f85570a016d794c29b1e76cf22f67af4486ddbe779e0f30674f138fa4e1769ec"}, + {file = "grpcio-1.81.0-cp310-cp310-win32.whl", hash = "sha256:3755c9669307cad18e7e009860fdea98118978d2300451bd8530a53048e741e7"}, + {file = "grpcio-1.81.0-cp310-cp310-win_amd64.whl", hash = "sha256:909bb3222b53235498d2c5817a0596d82b0aaea490ba93fdf1b060e2938a543c"}, + {file = "grpcio-1.81.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:794e6aa648e8df47d8f908dc8c3b42347d04ec58438f1dcd4e445f09b4f6b0ce"}, + {file = "grpcio-1.81.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cd78145b7f7784661c524624f3526c9c6f891b30a4b54cb93a40806d0d0d61e9"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:638ccc1b86f7540170a169cb900799b9296a1381e47879ce60b0de9d3db73d33"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:21ec30b9ea320c8207ea7cd05873ad64aa69fdd0e81b6758b3347983ba20b50a"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dbdb99986548a7e87f8343805ef315fd4eb50ffaabf4fb1206e42f2542bb805d"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c36f5d5e97944cbda2d4096b4ae262e6e68506246b61582acf1b8591607f3ccc"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9f355384e5543ab77a755a7085225ecc19f32b76032e851cbd8145715d79dec8"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:77eb4e9fe61486bd1198cc7236ebb0f70e66234e63c0348f40bc2553ed16a88b"}, + {file = "grpcio-1.81.0-cp311-cp311-win32.whl", hash = "sha256:7915a2e63acdc05264a206e1bddfd8e1fb8a29e406c18d72d30f8c124e021374"}, + {file = "grpcio-1.81.0-cp311-cp311-win_amd64.whl", hash = "sha256:5e925a70fe99fe5794f7beca0ea034c75f068afcc356d79047e73f99cdcca34c"}, + {file = "grpcio-1.81.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:57b3b0e73a518fa286959b40c3eddd02703504ca186e8b7b2945954519bd8b2c"}, + {file = "grpcio-1.81.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:8bb1789c94322a13336a2b6c58d9c14d68f8628b6e24205a799c69f5bf8516ce"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e4d053900a0d24b75d7521139a3872150301b3d6bde3bed5e12318fb25791e4d"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:db217c2e52931719f9937bd12082cd4d7b495b35803d5760686975c285924bf8"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:19f201da7b4e5c0559198abe5a97157e726f3abe6e8f5e832d4a50740f6dcc22"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:275144b0115353339dbb8a6f28a9cf8997b5bf40e37f8f66ac0b0ea57e95b43f"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5192857589f223e5a98ff0e31f6e551b19040e647d17bfe10116c8a2ce3b8696"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6ff087cb1f563f47b504b4e29e684129fc5ae4863faf3ebca08a327764ee6cb"}, + {file = "grpcio-1.81.0-cp312-cp312-win32.whl", hash = "sha256:98c6240f563178fc5877bd50e6ff274463e53e1472128f4110742450739659fa"}, + {file = "grpcio-1.81.0-cp312-cp312-win_amd64.whl", hash = "sha256:87e33b7afcfb3585121b5f007d2c52b8c534104d18f556e840d35193ca2a9141"}, + {file = "grpcio-1.81.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:62bbe463c9f0f2ff24e31bd25f8dd8b4bae78900e315915a3195a0ef1471a855"}, + {file = "grpcio-1.81.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43c121e135ae44d1559b430db2b2dfad7421cbbe40e1deba506c7dc62b439719"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f345de40ef2e65f63645d53d251824e6070e07804827c5b00ec2e44555f9f901"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8c0855a350886f713b9e458e2a10d208009dcaa849f574e39cd6067db1fe1279"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a524cd530900bd24511fcb7f2ed144da4ea37711c4b094475d0bceca7a93a170"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e7746ba3e6efc9e2b748eff59470a2b8684d5a9ec607c6580bcaa5be175820bc"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:aaaa4f7f2057d795952e4eacf3f342be8b5b156992f6ac85023c8b98794ebd47"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0fba53cb96004b2b7fb758b46b2288cb49d0b658316a4e73f3ef67230616ee65"}, + {file = "grpcio-1.81.0-cp313-cp313-win32.whl", hash = "sha256:c197e2ef75a442528072b29e9755da299110e8610e8bcbb59a6b4cf55384f005"}, + {file = "grpcio-1.81.0-cp313-cp313-win_amd64.whl", hash = "sha256:194eddfacc84d80f50512e9fd4ee851d5f2499f18f299c95aa8fb4748f0537e0"}, + {file = "grpcio-1.81.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:a9351055f52660b58f3d4890ea66188b5134399f82b11aa0c55bd4b99eff5390"}, + {file = "grpcio-1.81.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:300f3337b6425fd16ead9a4f9b2ac25801acb64aa5bc0b99eb69901645b2b1d2"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:97bbd623f7ded558fd4f7cb5a4f600c4d4de65c5dd364c83a5b14b2a10a2d3b5"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ff83d889e3ebf6341c8c7864ad8031591ad5ca61599072fc511644d1eb962d2b"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c4fe218c5a35e1d87a5a26544237f1fa41dfd9cbd3c856b0810a30061f8b0aaf"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b8b025b6af43ee0ad4a70307025d77bcab5adde7c4597786010d802c203e9fc5"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:3d4e0ce5a40a998cf608c8ba60ecfe18fdf364a9aa193ae4ac3faeecd0e86757"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa948712c8e5fa40ec250870bda14bc7578e1bb832a8912d9d2a0f720518edbe"}, + {file = "grpcio-1.81.0-cp314-cp314-win32.whl", hash = "sha256:fbbe81314a9d92156abce8b62c09364eb8bafc0ca2a19919a45ec64b5c6cb664"}, + {file = "grpcio-1.81.0-cp314-cp314-win_amd64.whl", hash = "sha256:b93cee313cae4e113fbb3a0ce1ea5633db6f63cfde2b2dc1d817429026b2a50b"}, + {file = "grpcio-1.81.0.tar.gz", hash = "sha256:a5acd7efd3b1fe9b4eb0bcaaa1507eed68a0ad0678b654c3f7b464df9ba9dca5"}, ] [package.dependencies] typing-extensions = ">=4.12,<5.0" [package.extras] -protobuf = ["grpcio-tools (>=1.80.0)"] +protobuf = ["grpcio-tools (>=1.81.0)"] [[package]] name = "idna" diff --git a/security_scanning/examples/redrafter/poetry.lock b/security_scanning/examples/redrafter/poetry.lock index 368d809f4267..ad163ae2e9df 100644 --- a/security_scanning/examples/redrafter/poetry.lock +++ b/security_scanning/examples/redrafter/poetry.lock @@ -26,132 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -162,10 +161,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index 9f94c9bdfe07..e354fe4bc439 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -2620,28 +2620,28 @@ dill = ">=0.4.1" [[package]] name = "narwhals" -version = "2.21.2" +version = "2.22.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "narwhals-2.21.2-py3-none-any.whl", hash = "sha256:7bb57c3700486039215455b9bf2d64261915cc0fd845cc30272d631df696b251"}, - {file = "narwhals-2.21.2.tar.gz", hash = "sha256:5c5b2d0b47aef7c73ea412cfcbcd467f2f2d5be73e3c2ab19d78f4a97718790a"}, + {file = "narwhals-2.22.0-py3-none-any.whl", hash = "sha256:1421797ede01789cc1537619dbc3f36f840737240f748fdb24a60a0225fc80be"}, + {file = "narwhals-2.22.0.tar.gz", hash = "sha256:6486282bb7e4b4ab55963efbd8be1451b764cc4874b74d1fd625eba9dc60b86f"}, ] [package.extras] -cudf = ["cudf-cu12 (>=24.10.0)"] +cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging", "pyarrow-hotfix", "rich"] -modin = ["modin"] -pandas = ["pandas (>=1.1.3)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +modin = ["modin (>=0.22.0)"] +pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] pyarrow = ["pyarrow (>=13.0.0)"] pyspark = ["pyspark (>=3.5.0)"] pyspark-connect = ["pyspark[connect] (>=3.5.0)"] -sql = ["duckdb (>=1.1)", "sqlparse"] +sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] [[package]] @@ -2807,26 +2807,26 @@ files = [ [[package]] name = "nvidia-ml-py" -version = "13.595.45" +version = "13.610.43" description = "Python Bindings for the NVIDIA Management Library" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "nvidia_ml_py-13.595.45-py3-none-any.whl", hash = "sha256:b65a7977f503d56154b14d683710125ef93594adb63fbf7e559336e3318f1376"}, - {file = "nvidia_ml_py-13.595.45.tar.gz", hash = "sha256:c9f34897fe0441ff35bc8f35baf80f830a20b0f4e6ce71e0a325bc0e66acf079"}, + {file = "nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8"}, + {file = "nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2"}, ] [[package]] name = "optuna" -version = "4.8.0" +version = "4.9.0" description = "A hyperparameter optimization framework" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "optuna-4.8.0-py3-none-any.whl", hash = "sha256:c57a7682679c36bfc9bca0da430698179e513874074b71bebedb0334964ab930"}, - {file = "optuna-4.8.0.tar.gz", hash = "sha256:6f7043e9f8ecb5e607af86a7eb00fb5ec2be26c3b08c201209a73d36aff37a38"}, + {file = "optuna-4.9.0-py3-none-any.whl", hash = "sha256:f52f3be6148654850c92a5860d398fd88ec6b2c84ab68d9c3d07dcff02e7afee"}, + {file = "optuna-4.9.0.tar.gz", hash = "sha256:b322e5cbdf1655fb84c37646c4a7a1f391de1b47806bbe222e015825d0a82b87"}, ] [package.dependencies] @@ -2839,10 +2839,8 @@ sqlalchemy = ">=1.4.2" tqdm = "*" [package.extras] -checking = ["mypy", "mypy_boto3_s3", "ruff", "scipy-stubs ; python_version >= \"3.10\"", "types-PyYAML", "types-redis", "types-setuptools", "types-tqdm", "typing_extensions (>=3.10.0.0)"] -document = ["ase", "cmaes (>=0.12.0)", "fvcore", "kaleido (<0.4)", "lightgbm", "matplotlib (!=3.6.0)", "pandas", "pillow", "plotly (>=4.9.0)", "scikit-learn", "sphinx", "sphinx-copybutton", "sphinx-gallery", "sphinx-notfound-page", "sphinx_rtd_theme (>=1.2.0)", "torch", "torchvision"] +document = ["ase", "cmaes (>=0.12.0)", "fvcore", "kaleido (!=0.2.1.post1,<0.4)", "lightgbm", "matplotlib (!=3.6.0)", "pandas", "pillow", "plotly (>=4.9.0)", "scikit-learn", "sphinx", "sphinx-copybutton", "sphinx-gallery", "sphinx-notfound-page", "sphinx_rtd_theme (>=1.2.0)", "torch", "torchvision"] optional = ["boto3", "cmaes (>=0.12.0)", "google-cloud-storage", "greenlet", "grpcio", "matplotlib (!=3.6.0)", "pandas", "plotly (>=4.9.0)", "protobuf (>=5.28.1)", "redis", "scikit-learn (>=0.24.2)", "scipy", "torch"] -test = ["fakeredis[lua]", "greenlet", "grpcio", "kaleido (<0.4)", "moto", "protobuf (>=5.28.1)", "pytest", "pytest-xdist", "scipy (>=1.9.2)", "torch"] [[package]] name = "orjson" diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index 1ee122f0bc39..9c8b3a3a8e77 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -59,132 +59,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -195,10 +194,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 3fafefd4a7d2..d1ae0f9420e8 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "5f5b77239ad5ccad19a944e40aee9ef7d6016101", - "timestamp": "2026-06-01T02:47:52Z" + "commit_hash": "6222112ff9a487a9e307583a62a537a2f1436776", + "timestamp": "2026-06-02T02:46:47Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index 873a7edd0167..fe0e4a0e4ab4 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -60,132 +60,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -196,10 +195,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -1889,80 +1889,70 @@ test = ["coverage", "pytest (>=7,<8.1)", "pytest-cov", "pytest-mock (>=3)"] [[package]] name = "grpcio" -version = "1.80.0" +version = "1.81.0" description = "HTTP/2-based RPC framework" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, - {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc"}, - {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199"}, - {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81"}, - {file = "grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069"}, - {file = "grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58"}, - {file = "grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a"}, - {file = "grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21"}, - {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106"}, - {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6"}, - {file = "grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440"}, - {file = "grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9"}, - {file = "grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0"}, - {file = "grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921"}, - {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f"}, - {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f"}, - {file = "grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193"}, - {file = "grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff"}, - {file = "grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad"}, - {file = "grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6"}, - {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7"}, - {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7"}, - {file = "grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294"}, - {file = "grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50"}, - {file = "grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e"}, - {file = "grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14"}, - {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f"}, - {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e"}, - {file = "grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae"}, - {file = "grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f"}, - {file = "grpcio-1.80.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4"}, - {file = "grpcio-1.80.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957"}, - {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4"}, - {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614"}, - {file = "grpcio-1.80.0-cp39-cp39-win32.whl", hash = "sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141"}, - {file = "grpcio-1.80.0-cp39-cp39-win_amd64.whl", hash = "sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de"}, - {file = "grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257"}, + {file = "grpcio-1.81.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:b4108e5d9d0f651b7eea749116181fe6c315b145661a80ec31f05ec2dbe21af7"}, + {file = "grpcio-1.81.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:b76ea9d55cd08fcdbda25d28e0f76679536710acb7fbd5b1f70cb4ac49317265"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4e032feb3bfb4e2749b140a2302a6baa8ead1b9781ff5cf7094e4402b5e9372e"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:725801c7086d7e4cd160e42bb2f54e0aeb976b9568df3cc6f843b15d29b79fb1"}, + {file = "grpcio-1.81.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f750a091fff3a3991731abc1f818bdc64874bb3528162732cb4d45f2e07821a6"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8226ba097eed660ef14d36c6a69b85038552bb8b6d17b44a5aa6f9abf48b8e08"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:40edffb4ec3689373825d367c4457727047a6e554f03245265ecc8cc03215f22"}, + {file = "grpcio-1.81.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f85570a016d794c29b1e76cf22f67af4486ddbe779e0f30674f138fa4e1769ec"}, + {file = "grpcio-1.81.0-cp310-cp310-win32.whl", hash = "sha256:3755c9669307cad18e7e009860fdea98118978d2300451bd8530a53048e741e7"}, + {file = "grpcio-1.81.0-cp310-cp310-win_amd64.whl", hash = "sha256:909bb3222b53235498d2c5817a0596d82b0aaea490ba93fdf1b060e2938a543c"}, + {file = "grpcio-1.81.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:794e6aa648e8df47d8f908dc8c3b42347d04ec58438f1dcd4e445f09b4f6b0ce"}, + {file = "grpcio-1.81.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cd78145b7f7784661c524624f3526c9c6f891b30a4b54cb93a40806d0d0d61e9"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:638ccc1b86f7540170a169cb900799b9296a1381e47879ce60b0de9d3db73d33"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:21ec30b9ea320c8207ea7cd05873ad64aa69fdd0e81b6758b3347983ba20b50a"}, + {file = "grpcio-1.81.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dbdb99986548a7e87f8343805ef315fd4eb50ffaabf4fb1206e42f2542bb805d"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c36f5d5e97944cbda2d4096b4ae262e6e68506246b61582acf1b8591607f3ccc"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9f355384e5543ab77a755a7085225ecc19f32b76032e851cbd8145715d79dec8"}, + {file = "grpcio-1.81.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:77eb4e9fe61486bd1198cc7236ebb0f70e66234e63c0348f40bc2553ed16a88b"}, + {file = "grpcio-1.81.0-cp311-cp311-win32.whl", hash = "sha256:7915a2e63acdc05264a206e1bddfd8e1fb8a29e406c18d72d30f8c124e021374"}, + {file = "grpcio-1.81.0-cp311-cp311-win_amd64.whl", hash = "sha256:5e925a70fe99fe5794f7beca0ea034c75f068afcc356d79047e73f99cdcca34c"}, + {file = "grpcio-1.81.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:57b3b0e73a518fa286959b40c3eddd02703504ca186e8b7b2945954519bd8b2c"}, + {file = "grpcio-1.81.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:8bb1789c94322a13336a2b6c58d9c14d68f8628b6e24205a799c69f5bf8516ce"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e4d053900a0d24b75d7521139a3872150301b3d6bde3bed5e12318fb25791e4d"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:db217c2e52931719f9937bd12082cd4d7b495b35803d5760686975c285924bf8"}, + {file = "grpcio-1.81.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:19f201da7b4e5c0559198abe5a97157e726f3abe6e8f5e832d4a50740f6dcc22"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:275144b0115353339dbb8a6f28a9cf8997b5bf40e37f8f66ac0b0ea57e95b43f"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5192857589f223e5a98ff0e31f6e551b19040e647d17bfe10116c8a2ce3b8696"}, + {file = "grpcio-1.81.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6ff087cb1f563f47b504b4e29e684129fc5ae4863faf3ebca08a327764ee6cb"}, + {file = "grpcio-1.81.0-cp312-cp312-win32.whl", hash = "sha256:98c6240f563178fc5877bd50e6ff274463e53e1472128f4110742450739659fa"}, + {file = "grpcio-1.81.0-cp312-cp312-win_amd64.whl", hash = "sha256:87e33b7afcfb3585121b5f007d2c52b8c534104d18f556e840d35193ca2a9141"}, + {file = "grpcio-1.81.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:62bbe463c9f0f2ff24e31bd25f8dd8b4bae78900e315915a3195a0ef1471a855"}, + {file = "grpcio-1.81.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43c121e135ae44d1559b430db2b2dfad7421cbbe40e1deba506c7dc62b439719"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f345de40ef2e65f63645d53d251824e6070e07804827c5b00ec2e44555f9f901"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8c0855a350886f713b9e458e2a10d208009dcaa849f574e39cd6067db1fe1279"}, + {file = "grpcio-1.81.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a524cd530900bd24511fcb7f2ed144da4ea37711c4b094475d0bceca7a93a170"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e7746ba3e6efc9e2b748eff59470a2b8684d5a9ec607c6580bcaa5be175820bc"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:aaaa4f7f2057d795952e4eacf3f342be8b5b156992f6ac85023c8b98794ebd47"}, + {file = "grpcio-1.81.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0fba53cb96004b2b7fb758b46b2288cb49d0b658316a4e73f3ef67230616ee65"}, + {file = "grpcio-1.81.0-cp313-cp313-win32.whl", hash = "sha256:c197e2ef75a442528072b29e9755da299110e8610e8bcbb59a6b4cf55384f005"}, + {file = "grpcio-1.81.0-cp313-cp313-win_amd64.whl", hash = "sha256:194eddfacc84d80f50512e9fd4ee851d5f2499f18f299c95aa8fb4748f0537e0"}, + {file = "grpcio-1.81.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:a9351055f52660b58f3d4890ea66188b5134399f82b11aa0c55bd4b99eff5390"}, + {file = "grpcio-1.81.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:300f3337b6425fd16ead9a4f9b2ac25801acb64aa5bc0b99eb69901645b2b1d2"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:97bbd623f7ded558fd4f7cb5a4f600c4d4de65c5dd364c83a5b14b2a10a2d3b5"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ff83d889e3ebf6341c8c7864ad8031591ad5ca61599072fc511644d1eb962d2b"}, + {file = "grpcio-1.81.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c4fe218c5a35e1d87a5a26544237f1fa41dfd9cbd3c856b0810a30061f8b0aaf"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b8b025b6af43ee0ad4a70307025d77bcab5adde7c4597786010d802c203e9fc5"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:3d4e0ce5a40a998cf608c8ba60ecfe18fdf364a9aa193ae4ac3faeecd0e86757"}, + {file = "grpcio-1.81.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa948712c8e5fa40ec250870bda14bc7578e1bb832a8912d9d2a0f720518edbe"}, + {file = "grpcio-1.81.0-cp314-cp314-win32.whl", hash = "sha256:fbbe81314a9d92156abce8b62c09364eb8bafc0ca2a19919a45ec64b5c6cb664"}, + {file = "grpcio-1.81.0-cp314-cp314-win_amd64.whl", hash = "sha256:b93cee313cae4e113fbb3a0ce1ea5633db6f63cfde2b2dc1d817429026b2a50b"}, + {file = "grpcio-1.81.0.tar.gz", hash = "sha256:a5acd7efd3b1fe9b4eb0bcaaa1507eed68a0ad0678b654c3f7b464df9ba9dca5"}, ] [package.dependencies] typing-extensions = ">=4.12,<5.0" [package.extras] -protobuf = ["grpcio-tools (>=1.80.0)"] +protobuf = ["grpcio-tools (>=1.81.0)"] [[package]] name = "h11" @@ -3309,28 +3299,28 @@ dill = ">=0.3.8" [[package]] name = "narwhals" -version = "2.21.2" +version = "2.22.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "narwhals-2.21.2-py3-none-any.whl", hash = "sha256:7bb57c3700486039215455b9bf2d64261915cc0fd845cc30272d631df696b251"}, - {file = "narwhals-2.21.2.tar.gz", hash = "sha256:5c5b2d0b47aef7c73ea412cfcbcd467f2f2d5be73e3c2ab19d78f4a97718790a"}, + {file = "narwhals-2.22.0-py3-none-any.whl", hash = "sha256:1421797ede01789cc1537619dbc3f36f840737240f748fdb24a60a0225fc80be"}, + {file = "narwhals-2.22.0.tar.gz", hash = "sha256:6486282bb7e4b4ab55963efbd8be1451b764cc4874b74d1fd625eba9dc60b86f"}, ] [package.extras] -cudf = ["cudf-cu12 (>=24.10.0)"] +cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging", "pyarrow-hotfix", "rich"] -modin = ["modin"] -pandas = ["pandas (>=1.1.3)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +modin = ["modin (>=0.22.0)"] +pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] pyarrow = ["pyarrow (>=13.0.0)"] pyspark = ["pyspark (>=3.5.0)"] pyspark-connect = ["pyspark[connect] (>=3.5.0)"] -sql = ["duckdb (>=1.1)", "sqlparse"] +sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] [[package]] @@ -3805,14 +3795,14 @@ typing-extensions = "*" [[package]] name = "nvidia-ml-py" -version = "13.595.45" +version = "13.610.43" description = "Python Bindings for the NVIDIA Management Library" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "nvidia_ml_py-13.595.45-py3-none-any.whl", hash = "sha256:b65a7977f503d56154b14d683710125ef93594adb63fbf7e559336e3318f1376"}, - {file = "nvidia_ml_py-13.595.45.tar.gz", hash = "sha256:c9f34897fe0441ff35bc8f35baf80f830a20b0f4e6ce71e0a325bc0e66acf079"}, + {file = "nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8"}, + {file = "nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2"}, ] [[package]] @@ -4028,14 +4018,14 @@ onnx = ">=1.14.0" [[package]] name = "openai" -version = "2.38.0" +version = "2.40.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c"}, - {file = "openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3"}, + {file = "openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff"}, + {file = "openai-2.40.0.tar.gz", hash = "sha256:9a756f91f274a24ad6026cbcb2042fd356c8d4a10e8f347b08d34465e585f7a2"}, ] [package.dependencies] @@ -5963,35 +5953,35 @@ files = [ [[package]] name = "safetensors" -version = "0.8.0rc0" +version = "0.8.0rc1" description = "" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "safetensors-0.8.0rc0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c1e7a6a1c0dd0128888bc47aca0a9625855673f44f275bf4073088563bf7121b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c052d1706567487bc103088fe02daf05132dbccbbc3d798753541b66eb37fb14"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79547625fa84f3a9b28b933e44c67d012edf22a0c7170ed68835b9f467dda836"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a132d3cf5f63c3f02b82c4abf65c58d33a8422199ae34e09a9a7edb661bd2ca9"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d42f6c44773901ce1a021d2372747a559e9ec5aa59d044c0d711c273bff21c67"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b40d25911c5f241cad874ad1ea4100a9a9e3c2d469a73a38b47af759d239f44"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf0d366f75f63867f1ede90f87090450c7cec320da1fc2a597f9bb8cb73460db"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:50c56d7b6a2f44c3f4ab130bfeb6a8a51ce72bec152805f9c5a46bdf6addb6c5"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:94d9c0d569a124fe3074b9934031c2cdcfab12d4d7b64ae17343fac4a92081e8"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b4fcccda047df747e2463744428cba352d99527c4e52545d07f8c3a8583136f1"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:2ef8ab6704ea895cb13c89d5825f49e87328cac2093e7e45fb3cb615bd457fb2"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:35bf158d1555df7a529c844ae8ab89355c9df34546de0f94c47d538902bcc07c"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:98b0f6f2a14a6bde7f6acaa5f0381baef9a87c6a3124338affe4e4bb40bf826b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win32.whl", hash = "sha256:7e7cc49c69d8df5aaaf332532cd636609727599f81294bf4e5de56a2e3b70a10"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win_amd64.whl", hash = "sha256:d6532e381c492f5a6b4e82706b232f003e9e697b77d6c2eb7e806d11b578d00b"}, - {file = "safetensors-0.8.0rc0-cp310-abi3-win_arm64.whl", hash = "sha256:b7f8180f8c119dce85da7913904ccf4a0227adf095eb63f1732a6729c2672cb1"}, - {file = "safetensors-0.8.0rc0.tar.gz", hash = "sha256:b4168a839ff287dc26b0d843e1760962b2e92ed5645f95e8ab3f4b9401807e6a"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7e57730ae523085fda4a80eef74ad40c6d67af60b38498d9995cc9fcb639103b"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ba66ebb7eaa5914ff41cd2b2cd7ccd22c844854be2a5179289e748c70ffa90a4"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a72817e309ed17a6b805168bca500af711c8bf50cccf7bf790ad247788c676c"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c945f1ec6fc5a04abc174e79bce69c4613ae536c5276a3812a2a89b62ae009d7"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba0397739b71400eab5d1f492d68484595cf8b5d13dbfef274e3bcf518348bd8"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f61b7d2c1babc6271d778138788c57c8a95898be6ad3b559971fce20ec1c9c23"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53f59971f435fb5c23bb7bfa3c00e6cdbb26486d41f3aaf04c6d9e2d91bc8e4c"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:b070ba9428e6b2b3b820152b1e1583931cfbd692cda595442d5fbc8b5a46e824"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:69f73c2ec4f76e89deaefe485fec0c6fc42c04a70e318aaefd235da8edacccb1"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87aad206a0bb02fa3ddaeb2feb1c6f7f39a87bea1976750ba28a857fbfcd6b31"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:6ea00be4f7066ba834064fd7b104ba4b10d2e42cd915c9ece31f0e2b42e6b6d2"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:2445135cda6018a095ed951dec94a2c393f929ee3b7f7a9ca4630db854be6b89"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf949f6a37287572de1c47294b36131bf49528436724eec2f96015f75a3d0bc8"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win32.whl", hash = "sha256:4633355aaa0da80e789cb7c014c462b6dabe0ecc5f5bb56738fca01511b99975"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win_amd64.whl", hash = "sha256:d62fad383627979d80b640174c679f3304b3a21614c4d8e71f76910aecac9c8d"}, + {file = "safetensors-0.8.0rc1-cp310-abi3-win_arm64.whl", hash = "sha256:2b8ce46f7f16376eaf7527ee1dc830a32f74542767c0779f446f76d0be6b5da8"}, + {file = "safetensors-0.8.0rc1.tar.gz", hash = "sha256:a4bacbcd2ab9efe4eb5f1ea44afc9ac5f3b40e103ebde146e370e885ba46f2fc"}, ] [package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]"] +dev = ["safetensors[all]", "safetensors[pinned-tf]"] jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] mlx = ["mlx (>=0.0.9)"] numpy = ["numpy (>=1.24.6)"] @@ -6000,7 +5990,7 @@ pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] quality = ["ruff"] tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -testingfree = ["hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] +tf-nightly = ["safetensors[numpy]", "tf-nightly"] torch = ["safetensors[numpy]", "torch (>=2.4)"] [[package]] @@ -7474,4 +7464,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "26cbc35ce62c9e6822456f64742b16d54b72a783b19a2ebb08b9e2fc2a5a4bf1" +content-hash = "442b985b06ee9a2f18ffd12f8661192e162749843e42f4856b814fc2fdc6f404" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index c0cdd6e82f90..929ece110637 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "onnx (>=1.21.0)", "onnx-graphsurgeon (>=0.5.2)", "graphviz (>=0.21,<0.22)", - "openai (>=2.38.0,<3.0.0)", + "openai (>=2.40.0,<3.0.0)", "polygraphy (>=0.50.3,<0.51.0)", "psutil (>=7.2.2,<8.0.0)", "nvidia-ml-py (>=13)", diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 9ffe8afebe53..dfbe06ac11f2 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -14,132 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, - {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, - {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, - {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, - {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, - {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, - {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, - {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, - {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, - {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, - {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, - {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, - {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, - {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, - {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, - {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, - {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, - {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, - {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, - {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, - {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, - {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, - {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, - {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, - {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, - {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, - {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, - {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, - {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, - {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, - {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, - {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, - {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, - {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, - {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, - {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2"}, + {file = "aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f"}, + {file = "aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b"}, + {file = "aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25"}, + {file = "aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594"}, + {file = "aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803"}, + {file = "aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee"}, + {file = "aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93"}, + {file = "aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996"}, + {file = "aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5"}, + {file = "aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c"}, + {file = "aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2"}, + {file = "aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae"}, + {file = "aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066"}, + {file = "aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a"}, + {file = "aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127"}, + {file = "aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2"}, + {file = "aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c"}, + {file = "aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096"}, + {file = "aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec"}, + {file = "aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869"}, + {file = "aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11"}, + {file = "aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b"}, + {file = "aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e"}, + {file = "aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a"}, + {file = "aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb"}, + {file = "aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52"}, + {file = "aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7"}, + {file = "aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228"}, + {file = "aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b"}, + {file = "aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928"}, + {file = "aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2"}, + {file = "aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24"}, + {file = "aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b"}, ] [package.dependencies] @@ -150,10 +149,11 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" +typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" From c3f6d981b133a6fa5dcbfb0f762c57d9de2095cd Mon Sep 17 00:00:00 2001 From: Zhenhua Wang <4936589+zhenhuaw-me@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:19:06 +0800 Subject: [PATCH 176/308] [TRTLLM-13028][doc] Add VisualGen API walkthrough example and docs page (#14685) Signed-off-by: Zhenhua Wang --- docs/source/helper.py | 33 ++++++++- docs/source/index.rst | 1 + examples/visual_gen/api_walkthrough.py | 71 +++++++++++++++++++ .../defs/examples/test_visual_gen.py | 22 ++++++ .../test_lists/test-db/l0_dgx_b200.yml | 1 + .../test_lists/test-db/l0_gh200.yml | 1 + .../test_lists/test-db/l0_h100.yml | 1 + .../test_lists/test-db/l0_l40s.yml | 1 + 8 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 examples/visual_gen/api_walkthrough.py diff --git a/docs/source/helper.py b/docs/source/helper.py index 9dc0cac1d0ba..ae09cbcfcbd8 100644 --- a/docs/source/helper.py +++ b/docs/source/helper.py @@ -64,8 +64,18 @@ def extract_meta_info(filename: str) -> Optional[DocMeta]: def generate_examples(): root_dir = Path(__file__).parent.parent.parent.resolve() ignore_list = { - '__init__.py', 'quickstart_example.py', 'quickstart_advanced.py', - 'quickstart_multimodal.py', 'star_attention.py' + '__init__.py', + 'quickstart_example.py', + 'quickstart_advanced.py', + 'quickstart_multimodal.py', + 'star_attention.py', + # Older VisualGen example scripts without ### :title metadata; opt + # in by adding the metadata block and removing the entry below. + 'visual_gen_flux.py', + 'visual_gen_ltx2.py', + 'visual_gen_wan_i2v.py', + 'visual_gen_wan_t2v.py', + 'visual_gen_mgmn_distributed.sh' } doc_dir = root_dir / "docs/source/examples" @@ -95,6 +105,13 @@ def collect_script_paths(examples_subdir: str) -> list[Path]: ] serve_script_base_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/{commit_hash}/examples/serve" + # Collect source paths for VisualGen examples + visual_gen_script_paths = collect_script_paths("visual_gen") + visual_gen_doc_paths = [ + doc_dir / f"{path.stem}.rst" for path in visual_gen_script_paths + ] + visual_gen_script_base_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/{commit_hash}/examples/visual_gen" + def _get_lines_without_metadata(filename: str) -> str: """Get line ranges that exclude metadata lines. Returns a string like "5-10,15-20" for use in :lines: directive. @@ -267,6 +284,18 @@ def write_index(metas: list[DocMeta], doc_template_path: Path, example_name="Online Serving Examples", section_order=[]) + # Generate the toctree for VisualGen example scripts. No section_order + # while the example set is small; add one alongside ### :section + # metadata on the scripts once we have enough examples to group. + visual_gen_metas = write_scripts(visual_gen_script_base_url, + visual_gen_script_paths, + visual_gen_doc_paths) + write_index(metas=visual_gen_metas, + doc_template_path=doc_dir / "llm_examples_index.template.rst_", + doc_path=doc_dir / "visual_gen_examples.rst", + example_name="VisualGen Examples", + section_order=[]) + def extract_all_and_eval(file_path): ''' Extract the __all__ variable from a Python file. diff --git a/docs/source/index.rst b/docs/source/index.rst index c4853512f426..2f96834c3a3f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,6 +23,7 @@ Welcome to TensorRT LLM's Documentation! :name: Deployment Guide examples/llm_api_examples.rst + examples/visual_gen_examples.rst examples/trtllm_serve_examples examples/dynamo_k8s_example.rst deployment-guide/index.rst diff --git a/examples/visual_gen/api_walkthrough.py b/examples/visual_gen/api_walkthrough.py new file mode 100644 index 000000000000..a47315d11ab6 --- /dev/null +++ b/examples/visual_gen/api_walkthrough.py @@ -0,0 +1,71 @@ +### :title API walkthrough +### :order 0 +from tensorrt_llm import VisualGen, VisualGenArgs +from tensorrt_llm.visual_gen.args import CompilationConfig + + +def main(): + # 1. List supported models registered with the pipeline registry. + print("\n=== Supported models ===") + for hf_id in VisualGen.supported_models(): + print(f" - {hf_id}") + + # 2. Inspect default pipeline_config knobs for the chosen model. These + # are per-architecture runtime knobs (e.g. Lightricks/LTX-2's + # ``text_encoder_path``); Wan-AI/Wan2.1-T2V-1.3B-Diffusers registers + # none, so the dict is empty. + pipeline_defaults = VisualGen.pipeline_config("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") + print("\n=== Pipeline config defaults for Wan-AI/Wan2.1-T2V-1.3B-Diffusers ===") + print(f" {pipeline_defaults or '(none)'}") + + # 3. Build VisualGenArgs. ``pipeline_config`` carries the per-architecture + # knobs from step 2 (here we just forward the registered defaults; + # real callers would override entries like ``text_encoder_path``). + # ``compilation_config.skip_warmup`` skips the post-load warmup pass. + visual_gen = VisualGen( + model="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + args=VisualGenArgs( + pipeline_config=pipeline_defaults, + compilation_config=CompilationConfig(skip_warmup=True), + ), + ) + + # 4. Discover model-specific ``extra_params`` accepted by the loaded + # pipeline. Wan-AI/Wan2.1-T2V-1.3B-Diffusers declares none; + # Wan-AI/Wan2.2-T2V-A14B-Diffusers surfaces ``guidance_scale_2`` and + # ``boundary_ratio`` here. + specs = visual_gen.extra_param_specs + print("\n=== Extra param specs (extra_params keys) ===") + for name, spec in specs.items(): + print(f" - {name}: {spec}") + if not specs: + print(" (none for this model)") + + # 5. Take the pipeline's resolved defaults (height/width/steps/etc.) + # and override fields. ``default_params`` already pre-populates + # ``params.extra_params`` with each declared spec's default, so the + # override below shows how a caller would set a model-specific knob + # -- no-op on Wan-AI/Wan2.1-T2V-1.3B-Diffusers, but the wiring is + # the same on Wan-AI/Wan2.2-T2V-A14B-Diffusers where + # ``extra_params["guidance_scale_2"]`` is honored. + params = visual_gen.default_params + # Wan requires num_frames of the form 4k+1; 1.25x the model default (81) + # is 101.25, so we round to the nearest valid value, 101 (= 4*25 + 1). + params.num_frames = 101 + for name, spec in specs.items(): + params.extra_params[name] = spec.default + + print("\n=== Request params ===") + print(params.model_dump_json(indent=2)) + + output = visual_gen.generate(inputs="A cute cat playing piano in a sunny room", params=params) + + # 6. Persist to disk. ``save`` infers the container from the file + # extension (.avi/.mp4) and uses the frame_rate carried on the + # output. + saved = output.save("api_walkthrough_output.avi") + print(f"\nSaved: {saved}") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/defs/examples/test_visual_gen.py b/tests/integration/defs/examples/test_visual_gen.py index 1a025a0e294c..bf711d1b40e3 100644 --- a/tests/integration/defs/examples/test_visual_gen.py +++ b/tests/integration/defs/examples/test_visual_gen.py @@ -1188,6 +1188,28 @@ def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(output_path), f"Quickstart did not produce output.avi at {output_path}" +def test_visual_gen_api_walkthrough(_visual_gen_deps, llm_root, llm_venv): + """Run examples/visual_gen/api_walkthrough.py end-to-end.""" + scratch_space = conftest.llm_models_root() + model_src = os.path.join(scratch_space, WAN_T2V_MODEL_SUBPATH) + if not os.path.isdir(model_src): + pytest.skip( + f"Model not found: {model_src} " + f"(set LLM_MODELS_ROOT or place {WAN_T2V_MODEL_SUBPATH} under scratch)" + ) + + model_dst = os.path.join(llm_venv.get_working_directory(), "Wan-AI", WAN_T2V_MODEL_SUBPATH) + if not os.path.islink(model_dst): + os.makedirs(os.path.dirname(model_dst), exist_ok=True) + os.symlink(model_src, model_dst, target_is_directory=True) + + script_path = os.path.join(llm_root, "examples", "visual_gen", "api_walkthrough.py") + venv_check_call(llm_venv, [script_path]) + + output_path = os.path.join(llm_venv.get_working_directory(), "api_walkthrough_output.avi") + assert os.path.isfile(output_path), f"API walkthrough did not produce {output_path}" + + # ============================================================================= # Core example tests — run per-model scripts from examples/visual_gen/models/ # with shared YAML configs from examples/visual_gen/configs/. diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 586e5986f230..00b9c02e1c48 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -304,6 +304,7 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] - examples/test_visual_gen.py::test_visual_gen_quickstart + - examples/test_visual_gen.py::test_visual_gen_api_walkthrough - examples/test_visual_gen.py::test_wan_t2v_example - examples/test_visual_gen.py::test_flux1_lpips_against_golden - examples/test_visual_gen.py::test_flux2_lpips_against_golden diff --git a/tests/integration/test_lists/test-db/l0_gh200.yml b/tests/integration/test_lists/test-db/l0_gh200.yml index 8a82797cbfef..8f799029f613 100644 --- a/tests/integration/test_lists/test-db/l0_gh200.yml +++ b/tests/integration/test_lists/test-db/l0_gh200.yml @@ -24,6 +24,7 @@ l0_gh200: - unittest/llmapi/test_llm_quant.py - llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit - examples/test_visual_gen.py::test_visual_gen_quickstart + - examples/test_visual_gen.py::test_visual_gen_api_walkthrough - unittest/test_model_runner_cpp.py - accuracy/test_cli_flow.py::TestGptNext::test_auto_dtype - examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index c61a9f1ed392..7fce83c94dd4 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -272,6 +272,7 @@ l0_h100: - test_e2e.py::test_mistral_large_hidden_vocab_size - llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit - examples/test_visual_gen.py::test_visual_gen_quickstart + - examples/test_visual_gen.py::test_visual_gen_api_walkthrough - unittest/trt/attention/test_gpt_attention_IFB.py - accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_fp8_prequantized - accuracy/test_cli_flow.py::TestLlama2_7B::test_fp8 diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index 9c72f9dccb86..c0fb7b7cbfba 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -64,6 +64,7 @@ l0_l40s: - examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] - llmapi/test_llm_examples.py::test_llmapi_quickstart - examples/test_visual_gen.py::test_visual_gen_quickstart + - examples/test_visual_gen.py::test_visual_gen_api_walkthrough - llmapi/test_llm_examples.py::test_llmapi_example_inference - llmapi/test_llm_examples.py::test_llmapi_example_inference_async - llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming From 702e39d2a6236f5d075e24872c853214f85333fe Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Tue, 2 Jun 2026 11:46:28 +0800 Subject: [PATCH 177/308] [None][chore] Update flashinfer-python from 0.6.12rc2 to 0.6.12 (#14805) Signed-off-by: yihwang-nv --- ATTRIBUTIONS-Python.md | 2 +- requirements.txt | 2 +- security_scanning/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index 4ae2f5070ff1..0ba92e7c6bc8 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -5261,7 +5261,7 @@ For more information, please refer to - `Tracker`: https://github.com/tox-dev/py-filelock/issues -## flashinfer-python (0.6.12rc2) +## flashinfer-python (0.6.12) ### Licenses License: `Apache-2.0` diff --git a/requirements.txt b/requirements.txt index a083e8cad59f..9fe0fbf59814 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ ordered-set peft>=0.18.1,<0.19.0 patchelf einops -flashinfer-python==0.6.12rc2 +flashinfer-python==0.6.12 opencv-python-headless xgrammar==0.1.32 llguidance==0.7.29 diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 929ece110637..22701cbe8440 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python (==0.6.12rc2)", + "flashinfer-python (==0.6.12)", "opencv-python-headless (>=4.13.0.92,<5.0.0.0)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", From 260b80fd691c5c898006408bca85ba59f29b1d04 Mon Sep 17 00:00:00 2001 From: Bala Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 22:16:52 -0700 Subject: [PATCH 178/308] [None][fix] AutoDeploy: Unwaive llmc standalone tests (#14700) Signed-off-by: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> --- .../llmc/create_standalone_package.py | 19 ++++- .../custom_ops/normalization/rms_norm.py | 23 +++++- .../models/custom/modeling_minimax_m2.py | 3 +- .../auto_deploy/models/quant_config_reader.py | 26 +++++-- .../transform/library/fuse_rope_mla.py | 3 +- .../transform/library/sharding_ir.py | 12 ++-- .../_torch/auto_deploy/utils/_graph.py | 20 ++++++ .../_torch/auto_deploy/utils/node_utils.py | 22 ++++-- tests/integration/test_lists/waives.txt | 2 - .../standalone/test_standalone_package.py | 70 ++++++++++++++----- 10 files changed, 158 insertions(+), 42 deletions(-) diff --git a/examples/auto_deploy/llmc/create_standalone_package.py b/examples/auto_deploy/llmc/create_standalone_package.py index 67222e371a2e..4a2bc641f81f 100644 --- a/examples/auto_deploy/llmc/create_standalone_package.py +++ b/examples/auto_deploy/llmc/create_standalone_package.py @@ -158,10 +158,17 @@ "test_flashinfer_mamba_cached_op.py", # Require TRT-LLM custom ops (dsv3_router_gemm_op, noaux_tc_op, etc.) "test_deepseek_custom.py", + "test_glm4_moe_modeling.py", "test_glm4_moe_lite_modeling.py", + "test_glm_moe_dsa_modeling.py", + # Full-model tests hit standalone-incompatible HF cache behavior. + "test_granite_moe_hybrid_modeling.py", + # Imports triton_kernels, which is not a standalone dependency. + "test_mxfp4_moe_layout.py", # Require TRT-LLM distributed ops (trtllm_dist_all_gather) "test_gather_logits_before_lm_head.py", - # Multimodal types are None in standalone (MultimodalInput guard) + # Multimodal processors depend on TensorRT-LLM multimodal request types. + "test_gemma4_modeling.py", "test_qwen3_5_moe.py", # Hardware-specific (requires H100+ shared memory) "test_triton_mla_op.py", @@ -377,8 +384,16 @@ def _create_test_conftest(tests_dir: str) -> None: # limitations under the License. \"\"\"Conftest for standalone auto_deploy tests.\"\"\" - import sys + import importlib.util import os + import sys + + _trtllm_spec = importlib.util.find_spec("tensorrt_llm") + if _trtllm_spec is not None: + raise RuntimeError( + "Standalone llmc tests must not be able to import tensorrt_llm; " + f"found {getattr(_trtllm_spec, 'origin', None)!r}" + ) # Add _utils_test to the Python path so test files can import from it sys.path.insert(0, os.path.join(os.path.dirname(__file__), "_utils_test")) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py index 2d042705423f..4e2b43d78b5c 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py @@ -21,10 +21,13 @@ import torch.nn.functional as F from einops import rearrange -import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils - from ..quantization.quant import TRTLLM_NVFP4_SCALING_VECTOR_SIZE +try: + import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils +except (ModuleNotFoundError, ImportError): + fp4_utils = None + try: from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl except (ModuleNotFoundError, ImportError): @@ -41,8 +44,22 @@ def get_env_enable_pdl() -> bool: from .triton_rms_norm import rms_norm +def _pad_up(x, y: int): + return ((x + y - 1) // y) * y + + def _get_nvfp4_fake_shapes(x: torch.Tensor) -> tuple[tuple[int, ...], int]: - output_shape, sf_size = fp4_utils.get_fp4_shape(x.shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + if fp4_utils is not None: + output_shape, sf_size = fp4_utils.get_fp4_shape(x.shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + return tuple(output_shape), sf_size + + input_shape = tuple(x.shape) + m = 1 + for dim in input_shape[:-1]: + m *= dim + output_shape = list(input_shape) + output_shape[-1] //= 2 + sf_size = _pad_up(m, 128) * _pad_up(input_shape[-1] // TRTLLM_NVFP4_SCALING_VECTOR_SIZE, 4) return tuple(output_shape), sf_size diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py index c14c1a8fedaf..2539ee043396 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py @@ -36,8 +36,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput -from tensorrt_llm._torch.utils import ActivationType - +from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache diff --git a/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py b/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py index 92ca23fd272b..167d09b182b6 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py +++ b/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py @@ -25,13 +25,29 @@ from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Optional, Tuple, Type -from tensorrt_llm.quantization.modelopt_config import ( - is_modelopt_quant_config, - read_modelopt_quant_config, -) - +from .._compat import TRTLLM_AVAILABLE from ..utils.logger import ad_logger +# Importing ``tensorrt_llm.quantization.modelopt_config`` triggers +# ``tensorrt_llm.quantization.__init__`` which pulls in ``tensorrt_llm.mapping`` +# and thus ``tensorrt_llm._torch.device_mesh``; none of these exist in the +# standalone llmc package. Gate the import so this module loads cleanly when +# TRT-LLM is absent; the ModelOPT reader path itself is unreachable in that case. +if TRTLLM_AVAILABLE: + from tensorrt_llm.quantization.modelopt_config import ( + is_modelopt_quant_config, + read_modelopt_quant_config, + ) +else: + + def is_modelopt_quant_config(raw: Any) -> bool: + return False + + def read_modelopt_quant_config(raw: Any) -> Dict[str, Any]: + raise NotImplementedError( + "ModelOPT quant config parsing requires TensorRT-LLM; not available in standalone mode." + ) + class QuantConfigReader(ABC): """Base class for reading and parsing quantization config.""" diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py index 2e293ffbc745..a59eecdb314d 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py @@ -35,13 +35,14 @@ import torch from torch.fx import GraphModule, Node -from ...custom_ops.mla.trtllm_mla import _TRTLLM_MLA_ROPE_INFO_KEY from ...models.factory import ModelFactory from ...shim.interface import CachedSequenceInterface from ...utils.logger import ad_logger from ...utils.node_utils import is_op from ..interface import BaseTransform, SharedConfig, TransformInfo, TransformRegistry +_TRTLLM_MLA_ROPE_INFO_KEY = "_trtllm_mla_rope_info" + # At post_load_fusion, only the backend-agnostic torch_rope_* IR ops are # present (optimize_rope has not yet replaced them with flashinfer_rope). _ROPE_OP_TARGETS = [] diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py b/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py index 0e49da0b52cf..9bd9c428b171 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py @@ -111,6 +111,13 @@ def _shard_scale_and_hook( } ) + +def _auto_deploy_ops(*names: str) -> Tuple[OpOverloadPacket, ...]: + return tuple( + op for name in names if (op := getattr(torch.ops.auto_deploy, name, None)) is not None + ) + + # ============================================================================= # ShardableNode abstract base class # ============================================================================= @@ -524,10 +531,7 @@ def apply(self, gm: GraphModule, dc: DistConfig, max_num_tokens: int = 0) -> int return 1 if count > 0 else 0 -@ShardableNode.register( - torch.ops.auto_deploy.torch_rmsnorm_gated, - torch.ops.auto_deploy.triton_rmsnorm_gated, -) +@ShardableNode.register(*_auto_deploy_ops("torch_rmsnorm_gated", "triton_rmsnorm_gated")) class NormShardableNode(ShardableNode): """Gated RMSNorm op: shard weight parameter.""" diff --git a/tensorrt_llm/_torch/auto_deploy/utils/_graph.py b/tensorrt_llm/_torch/auto_deploy/utils/_graph.py index f2d0ab640b51..f4159a502277 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/_graph.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/_graph.py @@ -357,7 +357,27 @@ def lint(gm: GraphModule) -> None: gm.graph.lint() +def _restore_topological_order(gm: GraphModule) -> None: + """Move nodes after their direct inputs if graph rewrites left them out of order.""" + while True: + node_order = {node: idx for idx, node in enumerate(gm.graph.nodes)} + for node in list(gm.graph.nodes): + input_nodes = [ + input_node for input_node in node.all_input_nodes if input_node in node_order + ] + if not input_nodes: + continue + latest_input = max(input_nodes, key=node_order.__getitem__) + if node_order[node] < node_order[latest_input]: + latest_input.append(node) + break + else: + return + + def _canonicalize_single_gm(gm: GraphModule) -> None: + _restore_topological_order(gm) + # clean up graph (needs to be done repeatedly until no more dead code) eliminate_dead_code(gm, is_impure_node=_is_impure_node) diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index f89634df51af..286242ce5228 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -46,6 +46,10 @@ OperatorLike = Union[OpOrOverload, Callable] +def _auto_deploy_op(name: str) -> Optional[OpOverloadPacket]: + return getattr(torch.ops.auto_deploy, name, None) + + class LayerType(Enum): """Enum for layer type.""" @@ -670,13 +674,17 @@ def is_any_moe_op(node: Node) -> bool: return is_op( node, ops=[ - torch.ops.auto_deploy.torch_moe, - torch.ops.auto_deploy.torch_quant_fp8_moe, - torch.ops.auto_deploy.torch_quant_nvfp4_moe, - torch.ops.auto_deploy.torch_quant_finegrained_fp8_moe, - torch.ops.auto_deploy.triton_mxfp4_moe, - torch.ops.auto_deploy.torch_moe_fused, - torch.ops.auto_deploy.torch_moe_dense_mlp, + op + for op in [ + _auto_deploy_op("torch_moe"), + _auto_deploy_op("torch_quant_fp8_moe"), + _auto_deploy_op("torch_quant_nvfp4_moe"), + _auto_deploy_op("torch_quant_finegrained_fp8_moe"), + _auto_deploy_op("triton_mxfp4_moe"), + _auto_deploy_op("torch_moe_fused"), + _auto_deploy_op("torch_moe_dense_mlp"), + ] + if op is not None ], ) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d2e7e5ef2c60..e0f5e7bf2342 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -366,8 +366,6 @@ unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vision_attention unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vision_block_matches_reference SKIP (https://nvbugs/6189450) unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vlm_wrapper_delta_is_request_scoped_no_cross_call_leakage SKIP (https://nvbugs/6189450) unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py::test_build_ad[deepseek-ai/DeepSeek-V3-llm_extra_args10] SKIP (https://nvbugs/5888827) -unittest/auto_deploy/standalone SKIP (https://nvbugs/6160629) -unittest/auto_deploy/standalone/test_standalone_package.py::TestStandalonePackage::test_run_unit_tests SKIP (https://nvbugs/6160629) unittest/disaggregated/test_agent_multi_backends.py::test_run_with_different_env[1] SKIP (https://nvbugs/5979673) unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) diff --git a/tests/unittest/auto_deploy/standalone/test_standalone_package.py b/tests/unittest/auto_deploy/standalone/test_standalone_package.py index 9aa11aecfc09..5dff65ae2a00 100644 --- a/tests/unittest/auto_deploy/standalone/test_standalone_package.py +++ b/tests/unittest/auto_deploy/standalone/test_standalone_package.py @@ -174,27 +174,65 @@ def test_run_unit_tests(self, standalone_package): if not os.path.isdir(tests_dir): pytest.skip("No tests directory in standalone package") - # Pass through the host env but override PYTHONPATH to use standalone tests. - # The venv's pip install already put llmc in the venv's site-packages, - # so `import llmc` resolves there (not to the host TRT-LLM). + # Keep runtime/compiler environment, but remove host Python import paths. standalone_env = { - **os.environ, - "PYTHONPATH": tests_dir + os.pathsep + os.path.join(tests_dir, "_utils_test"), - # Override PATH to prefer venv's python/pytest - "PATH": os.path.join(standalone_package["venv_dir"], "bin") - + os.pathsep - + os.environ.get("PATH", ""), - # FlashInfer JIT-compiles kernels and caches .so files under - # FLASHINFER_WORKSPACE_BASE (default: $HOME). Ninja records absolute - # source paths from the flashinfer package. Since this venv is - # ephemeral, a subsequent run would find stale cache entries pointing - # at the old (deleted) venv paths, causing all JIT builds to fail. - # Redirect the cache into the venv so it's discarded with it. - "FLASHINFER_WORKSPACE_BASE": standalone_package["venv_dir"], + key: value + for key, value in os.environ.items() + if key not in {"PYTHONHOME", "PYTHONPATH", "PYTHONUSERBASE"} } + standalone_env.update( + { + # Override PATH to prefer venv's python/pytest. + "PATH": os.path.join(standalone_package["venv_dir"], "bin") + + os.pathsep + + os.environ.get("PATH", ""), + # Keep subprocess workers from adding cwd/user paths that can + # expose the TensorRT-LLM checkout to standalone tests. + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + # FlashInfer JIT-compiles kernels and caches .so files under + # FLASHINFER_WORKSPACE_BASE (default: $HOME). Ninja records absolute + # source paths from the flashinfer package. Since this venv is + # ephemeral, a subsequent run would find stale cache entries pointing + # at the old (deleted) venv paths, causing all JIT builds to fail. + # Redirect the cache into the venv so it's discarded with it. + "FLASHINFER_WORKSPACE_BASE": standalone_package["venv_dir"], + } + ) + + isolation_probe = subprocess.run( + [ + python, + "-I", + "-c", + textwrap.dedent( + """ + import importlib.util + import sys + + spec = importlib.util.find_spec("tensorrt_llm") + if spec is not None: + raise SystemExit( + "tensorrt_llm is importable in standalone test env: " + "origin=%r, sys.path=%r" % (getattr(spec, "origin", None), sys.path) + ) + """ + ), + ], + capture_output=True, + text=True, + timeout=30, + cwd=pkg_dir, + env=standalone_env, + ) + assert isolation_probe.returncode == 0, ( + f"Standalone env leaked TensorRT-LLM\n" + f"stdout:\n{isolation_probe.stdout}\nstderr:\n{isolation_probe.stderr}" + ) cmd = [ python, + "-I", "-m", "pytest", os.path.join(tests_dir, "singlegpu"), From ca004112c71e37df2b9db4a99db94eb77cc49ce2 Mon Sep 17 00:00:00 2001 From: Li Min <11663212+limin2021@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:46:05 +0800 Subject: [PATCH 179/308] [TRTLLM-35882][feat] Add cute dsl gvr top-k decode kernel (#14602) Signed-off-by: Mindy Li <11663212+limin2021@users.noreply.github.com> --- .../_torch/custom_ops/cute_dsl_custom_ops.py | 308 +++ .../blackwell/top_k/__init__.py | 3 + .../blackwell/top_k/gvr_topk_decode.py | 1680 +++++++++++++++++ .../cute_dsl_kernels/top_k/run_gvr_topk.py | 506 +++++ .../sparse/test_cute_dsl_gvr_topk_decode.py | 240 +++ 5 files changed, 2737 insertions(+) create mode 100644 tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py create mode 100644 tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py create mode 100644 tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py index 7f1efabeff99..681cc3d3a0d1 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py @@ -5690,6 +5690,314 @@ def _( ) -> None: return None + # ------------------------------------------------------------------ # + # CuTe DSL GVR Top-K Decode # + # ------------------------------------------------------------------ # + from ..cute_dsl_kernels.blackwell.top_k.gvr_topk_decode import \ + GvrTopKKernel as _GvrTopKKernel + + class CuteDSLGvrTopKDecodeRunner: + """Runner for the GVR Top-K cuTe DSL kernel (Blackwell SM100). + + Owns the JIT compile cache keyed on + ``(dtype, top_k, next_n, tuning_knobs, compress_ratio)`` and an + auto-heuristic that resolves T (threads/block), V (vec-load width), + ``min_blocks_per_mp`` (ptxas ``__launch_bounds`` cap) and + ``enable_warp_parallel_reduce`` from input shape + dtype. + + Production knobs (tuning parameters) are fixed: ``enable_unroll_4`` + and ``enable_phase3_unroll`` always on, ``use_constant_hint`` off. + """ + kernel_cache: dict = {} + + @classmethod + def _compile( + cls, + dtype, + top_k: int, + next_n: int, + enable_unroll_4: bool, + enable_phase3_unroll: bool, + use_constant_hint: bool, + min_blocks_per_mp: int, + use_256bit_load: bool, + num_threads_per_block: int, + enable_warp_parallel_reduce: bool, + compress_ratio: int, + return_output_values: bool, + ) -> None: + key = (dtype, top_k, next_n, enable_unroll_4, enable_phase3_unroll, + use_constant_hint, min_blocks_per_mp, use_256bit_load, + num_threads_per_block, enable_warp_parallel_reduce, + compress_ratio, return_output_values) + if key in cls.kernel_cache: + return + n_rows = cute.sym_int() + n_cols = cute.sym_int() + n_batch = cute.sym_int() + # 256-bit vec loads require 32-byte aligned input addresses + # (PyTorch CUDA allocations are 256-byte aligned; Phase 2/3 + # offsets are multiples of vec_w * elem_bytes = 32 bytes). + in_align = 32 if use_256bit_load else 16 + input_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, n_cols), + stride_order=(1, 0), + assumed_align=in_align) + pre_idx_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_batch, top_k), + stride_order=(1, 0), + assumed_align=16) + seq_lens_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_batch, ), stride_order=(0, )) + # When return_output_values=False the kernel skips all STG.value + # writes; pass None so cute.compile doesn't materialize a fake + # value-output placeholder (matches the optional-tensor pattern + # used by CuteDSLTopKDecodeMultiCTARunner above). + out_values_fake = (cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, top_k), stride_order=(1, 0), assumed_align=16) + if return_output_values else None) + out_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows, top_k), + stride_order=(1, 0), + assumed_align=16) + fake_stream = cute.runtime.make_fake_stream( + use_tvm_ffi_env_stream=True) + + kernel = _GvrTopKKernel( + dtype=dtype, + top_k=top_k, + next_n=next_n, + num_threads=num_threads_per_block, + enable_unroll_4=enable_unroll_4, + enable_phase3_unroll=enable_phase3_unroll, + use_constant_hint=use_constant_hint, + min_blocks_per_mp=min_blocks_per_mp, + use_256bit_load=use_256bit_load, + enable_warp_parallel_reduce=enable_warp_parallel_reduce, + compress_ratio=compress_ratio, + return_output_values=return_output_values, + ) + cls.kernel_cache[key] = cute.compile( + kernel, + input_fake, + pre_idx_fake, + seq_lens_fake, + out_values_fake, + out_indices_fake, + stream=fake_stream, + options="--enable-tvm-ffi", + ) + logger.debug(f"[compile cute_dsl gvr_topk_decode] {key}") + + @classmethod + def forward( + cls, + logits: torch.Tensor, + pre_idx: torch.Tensor, + seq_lens: torch.Tensor, + output_indices: torch.Tensor, + top_k: int, + next_n: int = 1, + compress_ratio: int = 1, + max_seq_len: Optional[int] = None, + ) -> None: + cute_dtype = _TORCH_TO_CUTLASS_DTYPE[logits.dtype] + num_rows = logits.shape[0] + # Op-level hardcodes return_output_values=False — the DSA indexer + # pipeline only consumes indices, mirroring CUDA's + # ``indexer_topk_decode`` (which also doesn't expose value + # outputs). The compiled kernel skips all STG.value writes and + # accepts None for its value-output slot at launch time. + # + # The kernel keeps both ``return_output_values=True/False`` + # branches to support enabling value writeback in the future + # if a downstream caller needs it. + return_output_values = False + # max_seq_len: graph-safe hint. Eager mode: leave None and the + # heuristic adapts to actual N each call. Graph capture mode: + # caller passes peak runtime N so the captured kernel selects + # the large-N (T=1024, V=256) variant instead of the + # capture-time-small-N variant. + N_dec = max_seq_len if max_seq_len is not None else logits.shape[1] + num_sms = _get_num_sms() + + # Production tuning knobs (fixed across shapes). + enable_unroll_4 = True + enable_phase3_unroll = True + use_constant_hint = False + + # ---- T (num_threads_per_block) + V (use_256bit_load) ---- + # T=1024 only when grid fits 1 CTA/SM and each thread has + # enough vec-loop work (N >= 65536). For half-prec under + # graph capture, raise the bar to 131072 so a small capture-N + # never forces T=1024 onto small-N replays (~14-16% regression). + if max_seq_len is not None and logits.dtype != torch.float32: + n_thresh_t = 131072 + else: + n_thresh_t = 65536 + num_threads_per_block = (1024 if (num_rows <= num_sms + and N_dec >= n_thresh_t) else 512) + # V=256-bit only for fp32 above N=16K. Half-prec cvt-to-fp32 + # doubles fragment reg footprint and regresses 5-11% on K=512/1024. + # Caller contract: when this fires, ``logits.data_ptr()`` must + # be 32-byte aligned (satisfied by torch.empty() and row slices; + # column slices / stride-padded layouts may violate it). + use_256bit_load = (logits.dtype == torch.float32 and N_dec >= 16384) + if use_256bit_load: + ptr = logits.data_ptr() + assert ptr % 32 == 0, ( + f"use_256bit_load=True requires 32B-aligned " + f"logits.data_ptr(), got {ptr} % 32 = {ptr % 32}. " + f"Pass a contiguous tensor (column slices / stride-" + f"padded layouts violate alignment).") + # Warp-parallel reduce only pays off at 32-warp (T=1024); at + # 16-warp (T=512) it costs ~2pp on synth data. + enable_warp_parallel_reduce = num_threads_per_block == 1024 + + # ---- min_blocks_per_mp (ptxas __launch_bounds cap) ---- + # Reg-vs-occupancy 3-tier heuristic. Tier-3 ordering differs + # by dtype: half-prec puts tier-3 first (cvt-ILP fits in 40 + # regs, extra CTA/SM hides cvt latency); fp32 keeps tier-0 + # first (4-LDG ILP wants ~70 regs, mb=2 cap=64 caps ILP). + vec_bits_host = 256 if use_256bit_load else 128 + vec_w_host = vec_bits_host // (32 if logits.dtype == torch.float32 + else 16) + n_vec_iters = max(1, N_dec // (num_threads_per_block * vec_w_host)) + is_fp32 = logits.dtype == torch.float32 + if is_fp32: + if n_vec_iters < 4: + min_blocks_per_mp = 0 + elif num_rows <= num_sms: + min_blocks_per_mp = 1 + elif (num_sms * 2 < num_rows <= num_sms * 3 and N_dec <= 32768): + # Wave-fit + latency-bound: mb=3 caps each CTA to fit + # all num_rows CTAs in a single wave. At N>=65K the + # kernel becomes bandwidth-bound and mb=2 wins. + min_blocks_per_mp = 3 + else: + min_blocks_per_mp = 2 + else: + if num_rows > num_sms: + min_blocks_per_mp = 3 + elif n_vec_iters < 4: + min_blocks_per_mp = 0 + else: + min_blocks_per_mp = 1 + + cls._compile( + cute_dtype, + top_k, + next_n, + enable_unroll_4, + enable_phase3_unroll, + use_constant_hint, + min_blocks_per_mp, + use_256bit_load, + num_threads_per_block, + enable_warp_parallel_reduce, + compress_ratio, + return_output_values, + ) + key = (cute_dtype, top_k, next_n, enable_unroll_4, + enable_phase3_unroll, use_constant_hint, min_blocks_per_mp, + use_256bit_load, num_threads_per_block, + enable_warp_parallel_reduce, compress_ratio, + return_output_values) + # TVM FFI: pass raw torch tensors directly, env stream picked + # up automatically (no from_dlpack, no stream argument). + # ``output_values=None`` matches the kernel's compile-time + # ``return_output_values=False`` constexpr — no writes for top-k values. + cls.kernel_cache[key](logits, pre_idx, seq_lens, None, + output_indices) + + @torch.library.custom_op("trtllm::cute_dsl_gvr_topk_decode", + mutates_args=("output_indices", ), + device_types="cuda") + def cute_dsl_gvr_topk_decode( + logits: torch.Tensor, + pre_idx: torch.Tensor, + seq_lens: torch.Tensor, + output_indices: torch.Tensor, + top_k: int, + next_n: int = 1, + compress_ratio: int = 1, + max_seq_len: Optional[int] = None, + ) -> None: + """CuTe DSL GVR (Guess-Verify-Refine) Top-K decode for Blackwell. + + Writes per-row top-K indices into the caller-allocated + ``output_indices`` buffer (no return). Values are NOT written — + the kernel is compiled with ``return_output_values=False`` so the + STG.value path is skipped. Mirrors CUDA ``indexer_topk_decode`` + which also only exposes indices to callers. + + Args: + logits: ``[num_rows, max_seq_len]`` fp32 / bf16 / fp16. + pre_idx: ``[num_rows // next_n, top_k]`` int32. ``pre_idx[..., 0]`` + must be the argmax index (indexer invariant). + seq_lens: ``[num_rows // next_n]`` int32. Effective seq length per group. + output_indices: ``[num_rows, top_k]`` int32. + top_k: K ∈ {512, 1024, 2048} — compile-time specialized. + next_n: Temporal stride (V3.2 ``preIdxOffset = (row % next_n) + 1``). + compress_ratio: KV-indexer compression factor (1 = DSv3.2, 4 = DSv4). + max_seq_len: Graph-safe hint for peak ``logits.shape[1]`` at replay. + Pass under CUDA graph capture so the heuristic picks the + large-N kernel; leave ``None`` in eager mode. + """ + if not is_sm_100f(): + raise ValueError( + f"CuteDSL: SM version {get_sm_version()} is not supported. " + f"CuteDSL GVR Top-K Decode only supports SM 100 family.") + # num_rows = batch_size * next_n contract; downstream sizing of + # pre_idx/seq_lens/output_indices derives batch_size from this. + # Caught here so the user sees a clear message instead of an OOB + # write or ZeroDivisionError deep inside the kernel. + if logits.shape[0] % next_n != 0: + raise ValueError( + f"logits.shape[0] (={logits.shape[0]}) must be divisible by " + f"next_n (={next_n}); the kernel derives batch_size as " + f"logits.shape[0] / next_n.") + # Key includes the shape + dtype signature so a NEW input shape + # gets its own one-shot log line; without the signature only the + # first shape would ever be logged, hiding follow-up shapes from + # production diagnostics. + _log_sig = ( + f"{logits.dtype}|{tuple(logits.shape)}|" + f"k={top_k}|nn={next_n}|cr={compress_ratio}|msl={max_seq_len}") + logger.info_once( + f"cute_dsl_gvr_topk_decode inputs: " + f"logits dtype={logits.dtype} shape={tuple(logits.shape)} stride={logits.stride()}; " + f"pre_idx dtype={pre_idx.dtype} shape={tuple(pre_idx.shape)}; " + f"seq_lens dtype={seq_lens.dtype} shape={tuple(seq_lens.shape)}; " + f"output_indices dtype={output_indices.dtype} shape={tuple(output_indices.shape)}; " + f"top_k={top_k} next_n={next_n} compress_ratio={compress_ratio} " + f"max_seq_len={max_seq_len}", + key=f"cute_dsl_gvr_topk_decode_inputs|{_log_sig}", + ) + CuteDSLGvrTopKDecodeRunner.forward( + logits=logits, + pre_idx=pre_idx, + seq_lens=seq_lens, + output_indices=output_indices, + top_k=top_k, + next_n=next_n, + compress_ratio=compress_ratio, + max_seq_len=max_seq_len, + ) + + @torch.library.register_fake("trtllm::cute_dsl_gvr_topk_decode") + def _( + logits: torch.Tensor, + pre_idx: torch.Tensor, + seq_lens: torch.Tensor, + output_indices: torch.Tensor, + top_k: int, + next_n: int = 1, + compress_ratio: int = 1, + max_seq_len: Optional[int] = None, + ) -> None: + return None + def warmup_cute_dsl_indexer_topk( dtype: torch.dtype, top_k: int, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/__init__.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/__init__.py index 0c2c3191a5c5..f9ee2ddde4e2 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/__init__.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/__init__.py @@ -16,10 +16,13 @@ from .filtered_top_k_decode_varlen import FilteredTopKKernelVarlenDecode from .filtered_top_k_varlen_util import FilteredTopKKernelVarlen +from .gvr_topk_decode import GvrParams, GvrTopKKernel from .single_pass_multi_cta_radix_topk import SinglePassMultiCTARadixTopKKernel __all__ = [ "SinglePassMultiCTARadixTopKKernel", "FilteredTopKKernelVarlen", "FilteredTopKKernelVarlenDecode", + "GvrParams", + "GvrTopKKernel", ] diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py new file mode 100644 index 000000000000..1fdd39fb044f --- /dev/null +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py @@ -0,0 +1,1680 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GVR (Guess-Verify-Refine) heuristic Top-K kernel using cuTe DSL on Blackwell sm_100. + +Supported (dtype, K): fp32/bf16/fp16 x 512/1024/2048. + +TODO: could see if smem_ptcnt part and fmin/fmax vectorization could be improved. +""" + +from dataclasses import dataclass +from typing import Optional + +import cutlass +import cutlass.cute as cute +from cutlass._mlir.dialects import llvm +from cutlass.utils.distributed import atomicAdd +from cutlass.utils.smem_allocator import SmemAllocator + +from ..utils import TRTLLM_ENABLE_PDL, griddepcontrol_launch_dependents, griddepcontrol_wait +from .block_scan import warp_scan + + +def float_as_uint32(float_val): + """Interpret FP32 value as uint32 bit pattern (cuTe DSL bit-cast).""" + return llvm.bitcast(cutlass.Uint32.mlir_type, float_val.ir_value()) + + +def _fmin_f32_inline(a, b): + """Emit single PTX `min.f32` -> single SASS FMNMX instruction. + + cuTe DSL has cute.arch.fmax but NOT cute.arch.fmin; the canonical + workaround `-fmax(-a, -b)` lowers to a 3-instruction sequence + (FADD R, RZ, -a; FADD R, RZ, -b; FMNMX R, ...; FADD R, RZ, -R). This pattern + is concentrated in block_fused_snap_iter's inner loop and accounts + for ~8-10 us of the cuTe vs prod GVR gap at fp32 K=2048 BS=1. + """ + return cutlass.Float32( + llvm.inline_asm( + cutlass.Float32.mlir_type, + [a.ir_value(), b.ir_value()], + "min.f32 $0, $1, $2;", + "=f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + ) + ) + + +# ============================================================================= +# GvrParams — parameters for different (dtype, K, compress_ratio) combinations. +# ============================================================================= + + +@dataclass(frozen=True) +class GvrParams: + kFTarget: int + kC: int # candidate buffer cap + kNumBins: int # histogram bin count + + @staticmethod + def get(dtype_name: str, top_k: int, compress_ratio: int = 1) -> "GvrParams": + """Returns the per-(dtype, K, cr) specialization parameters. + + Mirrors CUDA template specialization GvrParams. For K ∈ {512, 1024}, + cr=1 (DSv3.2) and cr=4 (DSv4, PR #14413) use different kFTarget values + — V4's kFTarget=kK alignment eliminates upper-clamp saturation on + tight-σ + high-A2 layers; cross-prompt swe-bench shows 1.5-2.2× + P2-iter reduction vs V3.2's kFTarget=384/2560. K=2048 is identical + across cr (V4 doesn't natively use K=2048; values reused for safety). + """ + TABLE = { + # --- cr = 1 (DSv3.2): tuned on V3.2 swe-bench data --- + ("float32", 512, 1): GvrParams(kFTarget=384, kC=5120, kNumBins=1024), + ("float32", 1024, 1): GvrParams(kFTarget=2560, kC=5120, kNumBins=1024), + ("float32", 2048, 1): GvrParams(kFTarget=3072, kC=6144, kNumBins=2048), + ("bfloat16", 512, 1): GvrParams(kFTarget=384, kC=5120, kNumBins=512), + ("bfloat16", 1024, 1): GvrParams(kFTarget=2560, kC=5120, kNumBins=512), + ("bfloat16", 2048, 1): GvrParams(kFTarget=4096, kC=5120, kNumBins=2048), + ("float16", 512, 1): GvrParams(kFTarget=384, kC=5120, kNumBins=512), + ("float16", 1024, 1): GvrParams(kFTarget=2560, kC=5120, kNumBins=1024), + ("float16", 2048, 1): GvrParams(kFTarget=4096, kC=5120, kNumBins=2048), + # --- cr = 4 (DSv4): tuned on V4 Flash/Pro swe-bench data --- + ("float32", 512, 4): GvrParams(kFTarget=512, kC=5120, kNumBins=1024), + ("float32", 1024, 4): GvrParams(kFTarget=1024, kC=5120, kNumBins=1024), + ("float32", 2048, 4): GvrParams(kFTarget=3072, kC=6144, kNumBins=2048), + ("bfloat16", 512, 4): GvrParams(kFTarget=512, kC=5120, kNumBins=512), + ("bfloat16", 1024, 4): GvrParams(kFTarget=1024, kC=5120, kNumBins=512), + ("bfloat16", 2048, 4): GvrParams(kFTarget=4096, kC=5120, kNumBins=2048), + ("float16", 512, 4): GvrParams(kFTarget=512, kC=5120, kNumBins=512), + ("float16", 1024, 4): GvrParams(kFTarget=1024, kC=5120, kNumBins=1024), + ("float16", 2048, 4): GvrParams(kFTarget=4096, kC=5120, kNumBins=2048), + } + key = (dtype_name, top_k, compress_ratio) + if key not in TABLE: + raise ValueError(f"Unsupported GvrParams<{dtype_name}, {top_k}, cr={compress_ratio}>") + return TABLE[key] + + +class GvrTopKKernel: + """GVR (Guess-Verify-Refine) heuristic top-K kernel using cuTe DSL. + + One CTA processes one row. + Block size = 512/1024, as specified by num_threads. + Smem region sized to GvrParams. + + Algorithm phases: + P1: preIdx Min/Max/Mean → initial threshold + P2: Secant threshold search loop (count-only) + P3: Ballot-free candidate collect into smem keys[]/vals[] + P4: Histogram snap (cand → exact top-K) + writeback + + For different compress_ratio: + cr = 1: preIdxOffset = (row_idx % next_n) + 1. V3.2 decode +1 temporal shift. + cr = 4: preIdxOffset = 0. V4 decode no temporal shift. + """ + + def __init__( + self, + dtype: cutlass.Numeric, + top_k: int, + next_n: int = 1, + num_threads: int = 512, + enable_unroll_4: Optional[bool] = None, + enable_phase3_unroll: Optional[bool] = None, + use_constant_hint: bool = False, + min_blocks_per_mp: int = 3, + use_256bit_load: bool = False, + enable_warp_parallel_reduce: Optional[bool] = None, + compress_ratio: int = 1, + return_output_values: bool = True, + ): + # e.g., dtype = cutlass.Float32 / cutlass.BFloat16 / cutlass.Float16 + self.dtype = dtype + self.top_k = top_k + self.next_n = next_n + # KV compression ratio of the indexer feeding this kernel: + # 1 → DSv3.2 (no compressor); preIdxOffset = (row % next_n) + 1 + # reflects "newest token appended" + MTP windowing. + # 4 → DSv4 (overlap compressor); logits / preIdx live in compressed- + # token-index space. New compressed entries are appended at the + # end so prev-step indices remain valid as-is → preIdxOffset = 0. + assert compress_ratio in (1, 4), ( + f"compress_ratio must be 1 (V3.2) or 4 (V4); got {compress_ratio}" + ) + self.compress_ratio = compress_ratio + + self.WARP_SIZE = 32 + self.num_threads = num_threads + self.num_warps = num_threads // self.WARP_SIZE + # __launch_bounds__(num_threads, min_blocks_per_mp) hint to ptxas. + # B200 sm_100: 65536 regs/SM, BS=512 → max_regs_per_thread caps at: + # min_blocks=1: 128 (loosest — allows ptxas to use many regs) + # min_blocks=2: 64 + # min_blocks=3: 42 (current default — keeps 3 CTA/SM occupancy) + # When num_rows ≤ #SMs (e.g., 148 on B200), grid is undersubscribed so high + # occupancy isn't useful → set min_blocks=1 to let ptxas spend more + # regs covering LDG latency (4-LDG back-to-back pattern survives). + self.min_blocks_per_mp = min_blocks_per_mp + # Vector load width for Phase 2/3 vec loops: + # False (default): 128-bit per LDG → LDG.E.128 (fp32: 4 elems; bf16/fp16: 8) + # True: 256-bit per LDG → LDG.E.256 (fp32: 8 elems; bf16/fp16: 16) + # 256-bit halves the LDG count per byte loaded, but requires: + # * 32-byte aligned addresses (we pass assumed_align=32 below) + # * sm_90+ (Blackwell sm_100 supports it) + # * fragment reg usage doubles → potentially more reg pressure + self.use_256bit_load = use_256bit_load + self.vec_bits = 256 if use_256bit_load else 128 + self.vec_align_bytes = self.vec_bits // 8 # 32 for 256-bit, 16 for 128-bit + # Vec-loop unrolling switches. + # * enable_unroll_4: 4-way fast path in block_count_ge (4 LDG.E.128 + # in flight). Controls block_count_ge's fast path only. + # * enable_phase3_unroll: 4-way fast path in phase3_collect only. + # Independent of enable_unroll_4 because the perf trade-offs + # differ — phase3 has thread-local wc state and smem writes, + # making fast-path more expensive at large grid. + # * use_constant_hint: True → CopyG2ROp(invariant=True) → emits + # SASS LDG.E.*.CONSTANT (read-only data cache, matches CUDA + # __ldg path). False → CopyUniversalOp → plain LDG.E.* (no + # .CONSTANT modifier). + if enable_unroll_4 is None: + enable_unroll_4 = True + if enable_phase3_unroll is None: + enable_phase3_unroll = True + self.enable_unroll_4 = enable_unroll_4 + self.enable_phase3_unroll = enable_phase3_unroll + self.use_constant_hint = use_constant_hint + # Replace tid==0 serial loops over num_warps with warp-parallel + # reduce/scan in warp 0. Default policy is auto-coupled to + # num_threads: enabled iff num_threads == 1024 (32 warps), where + # the serial tid==0 cost is meaningful. At num_threads == 512 + # (16 warps) the warp-parallel path measured a ~2pp regression on + # synth, so it stays off. Explicit True / False overrides the + # policy for A/B testing. + if enable_warp_parallel_reduce is None: + enable_warp_parallel_reduce = num_threads == 1024 + self.enable_warp_parallel_reduce = enable_warp_parallel_reduce + + # When False, the kernel skips all STG writes to ``output_values``. + # Callers that only consume top-K indices (e.g. the DSA indexer + # pipeline) save the LSU bandwidth + reg pressure of dtype-cast + + # store; ``output_values`` is then a dummy buffer the kernel never + # touches. When True (kernel default), values are written for + # bench / standalone driver usage. + self.return_output_values = return_output_values + + # Map cutlass dtype → GvrParams lookup name + if dtype == cutlass.Float32: + self._dtype_name = "float32" + elif dtype == cutlass.BFloat16: + self._dtype_name = "bfloat16" + elif dtype == cutlass.Float16: + self._dtype_name = "float16" + else: + raise ValueError(f"Unsupported dtype for GvrTopKKernel: {dtype}") + + params = GvrParams.get(self._dtype_name, top_k, self.compress_ratio) + self.kC = params.kC + self.kNumBins = params.kNumBins + self.kFTarget = params.kFTarget + + # Kernel-wide constants. + # self.MAX_REFINE_ITERS: Phase-2 secant refine iteration cap. + # self.FLT_MAX / self.NEG_FLT_MAX: fp32 IEEE-754 max / negative-max + # sentinels used as reduction identities and pad values. + self.MAX_REFINE_ITERS = 15 + self.FLT_MAX = 3.4028235e38 + self.NEG_FLT_MAX = -self.FLT_MAX + + # ------------------------------------------------------------------ + # Build a vectorized copy atom for the input scan loops. With + # use_constant_hint=True we use CopyG2ROp+invariant to get + # xxx.E.*.CONSTANT (read-only cache, matches CUDA __ldg). Defined as + # a plain Python method (not @cute.jit) so the if-else branches both + # bind copy_atom in the same trace scope. + # ------------------------------------------------------------------ + def _make_load_copy_atom(self): + # num_bits_per_copy matches self.vec_bits (128 default; 256 when + # use_256bit_load=True). + if self.use_constant_hint: + return cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + self.dtype, + num_bits_per_copy=self.vec_bits, + invariant=True, + ) + return cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + self.dtype, + num_bits_per_copy=self.vec_bits, + ) + + # ------------------------------------------------------------------ + # Input load helper — casts to fp32 regardless of self.dtype. + # ------------------------------------------------------------------ + @cute.jit + def _load_fp32(self, ptr_view, idx): + # TODO: instructions? + v = ptr_view[idx] + if cutlass.const_expr(self.dtype == cutlass.Float32): + return v + else: + return cutlass.Float32(v) + + # ------------------------------------------------------------------ + # Warp-level reductions + # + # ------------------------------------------------------------------ + @cute.jit + def warp_reduce_sum_i32(self, val): + # REDUX.SYNC.ADD.S32 (sm_80+) + return cute.arch.warp_redux_sync(val, "add") + + @cute.jit + def warp_reduce_sum_f32(self, val): + # PTX redux.sync has no fadd. + # will lower to SHFL.BFLY 5-step tree. + return cute.arch.warp_reduction_sum(val) + + @cute.jit + def warp_reduce_min_f32(self, val): + # PTX redux.sync.fmin.f32 (sm_100). + return cute.arch.warp_redux_sync(val, "fmin") + + @cute.jit + def warp_reduce_max_f32(self, val): + # PTX redux.sync.fmax.f32 (sm_100). + return cute.arch.warp_redux_sync(val, "fmax") + + # ------------------------------------------------------------------ + # Phase 1: preIdx Min/Max/Mean -> initial threshold + # ------------------------------------------------------------------ + @cute.jit + def phase1_preidx_stats( + self, + input_row, # cute.Tensor [N] fp32 (post-cast for half-prec) + N, # length of input_row + pre_idx_row, # cute.Tensor [M] int32 + pre_idx_count, + pre_idx_offset, + smem_wmin_f32, # cute.Tensor [NUM_WARPS] float32 + smem_wmax_f32, # cute.Tensor [NUM_WARPS] float32 + smem_wsum_f32, # cute.Tensor [NUM_WARPS] float32 + smem_wcnt_i32, # cute.Tensor [NUM_WARPS] int32 + s_thr, # cute.Tensor [3] float32: [threshold, val_lo, val_hi] + s_iscalars, # cute.Tensor [5] int32: [cand_count, done, cnt_lo, cnt_hi, out_count] + tidx, + warp_id, + lane, + ): + """preIdx scan + warp reduce + block aggregate + initial threshold. + + Smem layout split: floats kept in fp32 buffers, ints kept in int32 + buffers (no bit-cast tricks needed — avoids ArithValue/ir_value + coupling and keeps types clean for the MLIR codegen). + """ + local_min = cutlass.Float32(self.FLT_MAX) + local_max = cutlass.Float32(self.NEG_FLT_MAX) + local_sum = cutlass.Float32(0.0) + local_cnt = cutlass.Int32(0) + + # Stride loop over preIdx with pre_idx_offset shift. + # pre_idx_count is compile-time constant (top_k from JIT key). + # Two cases: + # (a) pre_idx_count >= num_threads: every thread loads ≥1 preIdx; + # n_iters = pre_idx_count // num_threads ∈ {1, 2, 4, ...}. + # Fully unrolled (straight-line code). + # (b) pre_idx_count < num_threads (e.g. K=512, BS=1024): only first + # pre_idx_count threads have a preIdx; guard with `if tidx= self.num_threads): + n_iters = cutlass.const_expr(pre_idx_count // self.num_threads) + for u in cutlass.range_constexpr(n_iters): + i = tidx + cutlass.Int32(u * self.num_threads) + raw = pre_idx_row[i] + idx = raw + pre_idx_offset + if idx >= 0 and idx < N: + v = self._load_fp32(input_row, idx) + local_max = cute.arch.fmax(local_max, v) + local_min = _fmin_f32_inline(local_min, v) + local_sum = local_sum + v + local_cnt = local_cnt + 1 + else: + # K < num_threads — only first K threads load a preIdx. + # cute DSL requires variables to exist before dynamic `if` blocks, + # so predeclare `idx` with an out-of-range sentinel and update + # it conditionally; the downstream `if idx >= 0 and idx < N` + # gate handles the sentinel naturally. + idx = cutlass.Int32(-1) + if tidx < cutlass.Int32(pre_idx_count): + idx = pre_idx_row[tidx] + pre_idx_offset + if idx >= 0 and idx < N: + v = self._load_fp32(input_row, idx) + local_max = cute.arch.fmax(local_max, v) + local_min = _fmin_f32_inline(local_min, v) + local_sum = local_sum + v + local_cnt = local_cnt + 1 + + # Warp-level reductions + smem write. + # When pre_idx_count < num_threads (e.g. K=512 with num_threads=1024), + # only the first `active_preidx_warps` warps have real data; remaining + # warps would just reduce identity values + write identity to smem. + # Skip those dummy warps to save ~30 cy/warp. Full barrier below still + # required so all threads reach Phase 2 entry together. K ∈ {512, + # 1024, 2048} are all multiples of WARP_SIZE so no rounding needed. + # Clamp to num_warps so K > num_threads case (e.g. K=2048, threads=512) + # doesn't index past smem allocation (sized to num_warps). + active_preidx_warps = cutlass.const_expr( + min(pre_idx_count // self.WARP_SIZE, self.num_warps) + ) + if cutlass.const_expr(active_preidx_warps < self.num_warps): + if warp_id < cutlass.Int32(active_preidx_warps): + wmin = self.warp_reduce_min_f32(local_min) + wmax = self.warp_reduce_max_f32(local_max) + wsum = self.warp_reduce_sum_f32(local_sum) + wcnt = self.warp_reduce_sum_i32(local_cnt) + if lane == 0: + smem_wmin_f32[warp_id] = wmin + smem_wmax_f32[warp_id] = wmax + smem_wsum_f32[warp_id] = wsum + smem_wcnt_i32[warp_id] = wcnt + else: + wmin = self.warp_reduce_min_f32(local_min) + wmax = self.warp_reduce_max_f32(local_max) + wsum = self.warp_reduce_sum_f32(local_sum) + wcnt = self.warp_reduce_sum_i32(local_cnt) + if lane == 0: + smem_wmin_f32[warp_id] = wmin + smem_wmax_f32[warp_id] = wmax + smem_wsum_f32[warp_id] = wsum + smem_wcnt_i32[warp_id] = wcnt + cute.arch.barrier() + + # Block aggregate: 4 reductions over num_warps slots. Gated by + # self.enable_warp_parallel_reduce — True picks warp-parallel reduce + # (warp 0 lanes do 4 REDUX.SYNC instructions); False keeps tid==0 + # serial loop (default, since num_threads=512 measured a regression + # with the warp-parallel path; expected to win at num_threads=1024). + if cutlass.const_expr(self.enable_warp_parallel_reduce): + # NEW: warp-parallel 4-way reduce in warp 0. Read only the first + # `active_preidx_warps` slots (dummy warps above wrote nothing, + # so those smem slots are uninitialized). + if warp_id == cutlass.Int32(0): + v_min = cutlass.Float32(self.FLT_MAX) + v_max = cutlass.Float32(self.NEG_FLT_MAX) + v_sum = cutlass.Float32(0.0) + v_cnt = cutlass.Int32(0) + if lane < cutlass.Int32(active_preidx_warps): + v_min = smem_wmin_f32[lane] + v_max = smem_wmax_f32[lane] + v_sum = smem_wsum_f32[lane] + v_cnt = smem_wcnt_i32[lane] + pmin = self.warp_reduce_min_f32(v_min) + pmax = self.warp_reduce_max_f32(v_max) + psum = self.warp_reduce_sum_f32(v_sum) + pcnt = self.warp_reduce_sum_i32(v_cnt) + if lane == cutlass.Int32(0): + pmean = cutlass.Float32(0.0) + if pcnt > 0: + pmean = psum / cutlass.Float32(pcnt) + else: + pmean = (pmin + pmax) * cutlass.Float32(0.5) + cnt_lo_seed = pre_idx_count + (pre_idx_count >> 2) + s_thr[0] = pmean + s_thr[1] = pmin + s_thr[2] = pmax + s_iscalars[0] = cutlass.Int32(0) # cand_count + s_iscalars[1] = cutlass.Int32(0) # done + s_iscalars[2] = cutlass.Int32(cnt_lo_seed) # cnt_lo + s_iscalars[3] = cutlass.Int32(1) # cnt_hi + s_iscalars[4] = cutlass.Int32(0) # out_count + else: + # tid==0 serial loop. + if tidx == 0: + pmin = cutlass.Float32(self.FLT_MAX) + pmax = cutlass.Float32(self.NEG_FLT_MAX) + psum = cutlass.Float32(0.0) + pcnt = cutlass.Int32(0) + # Iterate over active_preidx_warps (= num_warps when K >= + # num_threads; smaller when K < num_threads since dummy warps + # above no longer write smem). + for w in cutlass.range_constexpr(active_preidx_warps): + v_min = smem_wmin_f32[w] + v_max = smem_wmax_f32[w] + v_sum = smem_wsum_f32[w] + v_cnt = smem_wcnt_i32[w] + pmax = cute.arch.fmax(pmax, v_max) + pmin = _fmin_f32_inline(pmin, v_min) + psum = psum + v_sum + pcnt = pcnt + v_cnt + + pmean = cutlass.Float32(0.0) + if pcnt > 0: + pmean = psum / cutlass.Float32(pcnt) + else: + pmean = (pmin + pmax) * cutlass.Float32(0.5) + + cnt_lo_seed = pre_idx_count + (pre_idx_count >> 2) + s_thr[0] = pmean + s_thr[1] = pmin + s_thr[2] = pmax + s_iscalars[0] = cutlass.Int32(0) + s_iscalars[1] = cutlass.Int32(0) + s_iscalars[2] = cutlass.Int32(cnt_lo_seed) + s_iscalars[3] = cutlass.Int32(1) + s_iscalars[4] = cutlass.Int32(0) + cute.arch.barrier() + + # ------------------------------------------------------------------ + # block_count_ge — Phase 2 / Phase 3 GE-count over global input + # Per-thread accumulate (4-element strided), cache to smem_ptcnt[tid] + # for P3 reuse, warp-reduce, block-reduce → s_iscalars[0] = cand_count. + # ------------------------------------------------------------------ + @cute.jit + def block_count_ge( + self, + input_row, # cute.Tensor [N] fp32 + N, # length of input_row + threshold, # cutlass.Float32 scalar + smem_ptcnt, # cute.Tensor [BLOCK_SIZE] int32 (P3 cache) + smem_wcnt, # cute.Tensor [NUM_WARPS] int32 (block reduce scratch) + s_iscalars, # cute.Tensor [5] int32 (writes [0] = cand_count) + tidx, + warp_id, + lane, + ): + """Count input[i] >= threshold across N elements. + + Vectorized: each thread loads vec_w-bit per iter (e.g., 128 bits loading 4 fp32 / 8 bf16 / 8 fp16) + via cute.copy + CopyUniversalOp. Falls back to scalar tail for the N-mod-vec_w remainder. + """ + num_threads = cutlass.const_expr(self.num_threads) + vec_w = cutlass.const_expr(self.vec_bits // self.dtype.width) + elem_bytes = cutlass.const_expr(self.dtype.width // 8) + vec_align = cutlass.const_expr(self.vec_align_bytes) + c = cutlass.Int32(0) + copy_atom = self._make_load_copy_atom() + + step_elem = cutlass.const_expr(num_threads * vec_w) + + row_addr = input_row.iterator.toint() + n_aligned = (N // cutlass.Int32(vec_w)) * cutlass.Int32(vec_w) + i = tidx * cutlass.Int32(vec_w) + step = cutlass.Int32(step_elem) + + # Fast path: 4-way unroll for LSU-pipelining ILP. + if self.enable_unroll_4: + # ================================================================= + # Each loop body loads 1 vec_w chunk; LLVM unrolls 4 iters at IR + # level. After unroll, GVN/CSE has one common base ptr in scope and + # MAY fold the 4 derived addresses into shared base + immediate + # offsets, e.g., loading from [base+0x2000/0x4000/0x6000]). + # ================================================================= + rng_frag = cute.make_fragment((vec_w,), self.dtype) + # Number of complete vec_w-aligned loads this thread can do: + # need: i + k*step_elem + (vec_w - 1) < N + # max k: floor((N - i - vec_w) / step_elem) + # N_iters = max_k + 1 + big_iters = cutlass.Int32(0) + if N > i + cutlass.Int32(vec_w - 1): + big_iters = (N - i - cutlass.Int32(vec_w)) // cutlass.Int32( + step_elem + ) + cutlass.Int32(1) + + for k in cutlass.range(big_iters, unroll=4): + i_local = i + k * cutlass.Int32(step_elem) + src_ptr_k = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(i_local) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src_k = cute.make_tensor(src_ptr_k, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src_k, rng_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = rng_frag[j] + else: + vj = cutlass.Float32(rng_frag[j]) + if vj >= threshold: + c = c + cutlass.Int32(1) + # Advance i past all consumed vec_w-aligned positions so the + # medium/tail loops below correctly skip (they check i + ... < N). + i = i + big_iters * cutlass.Int32(step_elem) + + # Tail vec loop: 1-way, handles remainder < 2*step (= remaining 1 + # full vec_w-stride or less). i is always vec_w-aligned here (it + # advanced by multiples of step_elem = num_threads*vec_w), so the + # same vec_align bytes hold. + tail_frag = cute.make_fragment((vec_w,), self.dtype) + while i + cutlass.Int32(vec_w - 1) < N: + src_ptr = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(i) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src = cute.make_tensor(src_ptr, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src, tail_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = tail_frag[j] + else: + vj = cutlass.Float32(tail_frag[j]) + if vj >= threshold: + c = c + cutlass.Int32(1) + i = i + step + + # Tail scalar loop + it = n_aligned + tidx + while it < N: + v = self._load_fp32(input_row, it) + if v >= threshold: + c = c + cutlass.Int32(1) + it = it + cutlass.Int32(num_threads) + + # Cache per-thread count for P3 retry-shrink reuse. + smem_ptcnt[tidx] = c + + # Warp reduce + lane-0 write + wc = self.warp_reduce_sum_i32(c) + if lane == 0: + smem_wcnt[warp_id] = wc + cute.arch.barrier() + + # Block aggregate (sum reduce over num_warps slots). No trailing + # barrier: caller is expected to insert its own __syncthreads after + # its post-processing of cand_count. + if cutlass.const_expr(self.enable_warp_parallel_reduce): + # NEW: warp-parallel sum reduce in warp 0. + if warp_id == cutlass.Int32(0): + v = cutlass.Int32(0) + if lane < cutlass.Int32(self.num_warps): + v = smem_wcnt[lane] + total = self.warp_reduce_sum_i32(v) + if lane == cutlass.Int32(0): + s_iscalars[0] = total + else: + # tid==0 serial sum. + if tidx == 0: + total = cutlass.Int32(0) + for w in cutlass.range_constexpr(self.num_warps): + total = total + smem_wcnt[w] + s_iscalars[0] = total + + # ------------------------------------------------------------------ + # Phase 2: Secant-interpolation threshold search + # Refines threshold to bring cand_count into [kK, kCC] using secant + # interpolation on (val_lo, cnt_lo) / (val_hi, cnt_hi). At most + # self.MAX_REFINE_ITERS iterations. + # ------------------------------------------------------------------ + @cute.jit + def phase2_secant_search( + self, + input_row, + N, + smem_ptcnt, + smem_wcnt, + s_thr, # [threshold, val_lo, val_hi] + s_iscalars, # [cand_count, done, cnt_lo, cnt_hi, out_count] + tidx, + warp_id, + lane, + ): + """Refine smem threshold to land cand_count in [kK, kCC] window. + + Calls block_count_ge to evaluate each candidate threshold and + updates the bracket (val_lo, val_hi, cnt_lo, cnt_hi). Marks + s_iscalars[1] (done) = 1 on convergence, 2 on bracket exhaustion. + """ + kK = cutlass.const_expr(self.top_k) + kCC = cutlass.const_expr(self.kC) + kFTarget = cutlass.const_expr(self.kFTarget) + + # ---- Initial count with the Phase-1 mean as threshold ---- + # TODO: smem_ptcnt is not always needed? only for the last block_count_ge. + # Do we have methods to reduce its write? + thr_init = s_thr[0] + self.block_count_ge( + input_row, + N, + thr_init, + smem_ptcnt, + smem_wcnt, + s_iscalars, + tidx, + warp_id, + lane, + ) + + # tid==0 classifies the initial count. + if tidx == 0: + c0 = s_iscalars[0] + t0 = s_thr[0] + if c0 >= cutlass.Int32(kK) and c0 <= cutlass.Int32(kCC): + s_iscalars[1] = cutlass.Int32(1) # done = 1 (converged) + elif c0 > cutlass.Int32(kCC): + # too many → threshold is the new lower bound (search HIGHER) + s_thr[1] = t0 + s_iscalars[2] = c0 + else: + # too few → threshold is the new upper bound (search LOWER) + s_thr[2] = t0 + s_iscalars[3] = c0 + cute.arch.barrier() + + # ---- Secant refinement loop ---- + it = cutlass.Int32(0) + while it < cutlass.Int32(self.MAX_REFINE_ITERS) and s_iscalars[1] == cutlass.Int32(0): + # tid==0 computes new threshold via secant interpolation. + if tidx == 0: + vlo = s_thr[1] + vhi = s_thr[2] + clo = s_iscalars[2] + chi = s_iscalars[3] + rng = vhi - vlo + nv = cutlass.Float32(0.0) + if clo > chi and rng > cutlass.Float32(1e-10): + f = cutlass.Float32(clo - cutlass.Int32(kFTarget)) / cutlass.Float32(clo - chi) + # clamp f to [0.05, 0.95] + f = cute.arch.fmax(cutlass.Float32(0.05), f) + f = _fmin_f32_inline(f, cutlass.Float32(0.95)) + if it == cutlass.Int32(0): + # iter 0: f = min(f, 0.5) — runtime compare (matches CUDA) + f = _fmin_f32_inline(f, cutlass.Float32(0.5)) + nv = vlo + rng * f + else: + nv = (vlo + vhi) * cutlass.Float32(0.5) + + # clamp nv into (vlo, vhi) range + if nv <= vlo: + nv = vlo + rng * cutlass.Float32(0.05) + if nv >= vhi: + nv = vhi - rng * cutlass.Float32(0.05) + + if nv == vlo or nv == vhi: + # Bracket exhausted — try midpoint, else give up. + nv = (vlo + vhi) * cutlass.Float32(0.5) + if nv == vlo or nv == vhi: + s_thr[0] = vlo + s_iscalars[1] = cutlass.Int32(2) # done = 2 (give up) + else: + s_thr[0] = nv + else: + s_thr[0] = nv + cute.arch.barrier() + + # Re-check done (tid==0 may have set it to 2) + if s_iscalars[1] == cutlass.Int32(0): + new_thr = s_thr[0] + self.block_count_ge( + input_row, + N, + new_thr, + smem_ptcnt, + smem_wcnt, + s_iscalars, + tidx, + warp_id, + lane, + ) + # tid==0 classifies the new count. + if tidx == 0: + c_new = s_iscalars[0] + t_new = s_thr[0] + if c_new >= cutlass.Int32(kK) and c_new <= cutlass.Int32(kCC): + s_iscalars[1] = cutlass.Int32(1) + elif c_new > cutlass.Int32(kCC): + s_thr[1] = t_new + s_iscalars[2] = c_new + else: + s_thr[2] = t_new + s_iscalars[3] = c_new + cute.arch.barrier() + it = it + cutlass.Int32(1) + + # ---- Post-loop fallback: if still not done, force threshold ---- + if tidx == 0: + if s_iscalars[1] == cutlass.Int32(0): + if s_iscalars[2] <= cutlass.Int32(kCC * 2): + s_thr[0] = s_thr[1] # threshold = val_lo + else: + s_thr[0] = s_thr[2] # threshold = val_hi + s_iscalars[1] = cutlass.Int32(2) + cute.arch.barrier() + + # ------------------------------------------------------------------ + # Phase 3: Ballot-free candidate collect + # If P2 ended with done=2 (bracket exhausted), first run a retry-shrink + # loop (≤10 iters) to bring cand_count <= kCC. + # Then reuse cached smem_ptcnt → warp prefix sum → block prefix sum + # → stream-write keys[]/vals[] for v >= threshold. + # ------------------------------------------------------------------ + @cute.jit + def phase3_collect_candidates( + self, + input_row, + N, + smem_keys, + smem_vals, + smem_ptcnt, + smem_wcnt, + s_thr, + s_iscalars, + tidx, + warp_id, + lane, + ): + """Retry-shrink (if done!=1) + warp/block prefix sum + stream-write. + + After this fn, smem_keys[0:cand_count] contains all v >= threshold + in some order (determined by tid's per-thread index within stream-write + loop), and smem_vals[0:cand_count] holds matching original indices. + smem_ptcnt is reused per-thread cached counts from the LAST + block_count_ge inside P2 (or inside the retry-shrink below). + """ + kK = cutlass.const_expr(self.top_k) + kCC = cutlass.const_expr(self.kC) + num_threads = cutlass.const_expr(self.num_threads) + + # ---- Retry-shrink loop (only if P2 didn't converge cleanly) ---- + if s_iscalars[1] != cutlass.Int32(1): + # Re-count with current threshold (may already have stale cand_count) + cur_thr = s_thr[0] + self.block_count_ge( + input_row, + N, + cur_thr, + smem_ptcnt, + smem_wcnt, + s_iscalars, + tidx, + warp_id, + lane, + ) + if tidx == 0: + if s_iscalars[0] > cutlass.Int32(kCC): + s_thr[1] = s_thr[0] # val_lo = threshold + cute.arch.barrier() + + # 10-iter retry-shrink. Runtime while with `cand_count > kCC` in the + # loop condition. + rs = cutlass.Int32(0) + while rs < cutlass.Int32(10) and s_iscalars[0] > cutlass.Int32(kCC): + if tidx == 0: + lo = s_thr[1] + hi = s_thr[2] + mid = (lo + hi) * cutlass.Float32(0.5) + if mid == lo: + mid = hi + s_thr[0] = mid + cute.arch.barrier() + new_thr = s_thr[0] + self.block_count_ge( + input_row, + N, + new_thr, + smem_ptcnt, + smem_wcnt, + s_iscalars, + tidx, + warp_id, + lane, + ) + if tidx == 0: + c_rs = s_iscalars[0] + if c_rs > cutlass.Int32(kCC): + s_thr[1] = s_thr[0] + elif c_rs < cutlass.Int32(kK): + s_thr[2] = s_thr[0] + cute.arch.barrier() + rs = rs + cutlass.Int32(1) + + # ---- Warp prefix sum over smem_ptcnt ---- + # my_total_qual = per-thread count cached by last block_count_ge. + my_total_qual = smem_ptcnt[tidx] + tp = my_total_qual + + # 5-level shfl_up_sync inclusive scan within warp. + for off_i in cutlass.range_constexpr(5): + off_v = cutlass.const_expr(1 << off_i) + other = cute.arch.shuffle_sync_up(tp, off_v, mask_and_clamp=0) + if lane >= cutlass.Int32(off_v): + tp = tp + other + + my_excl_offset = tp - my_total_qual + # Warp total = lane 31's tp; broadcast via shfl_sync_bfly (or + # cross-lane read: shuffle_sync_op with lane=31). + warp_total = cute.arch.shuffle_sync(tp, cutlass.Int32(self.WARP_SIZE - 1)) + + if lane == 0: + smem_wcnt[warp_id] = warp_total + cute.arch.barrier() + + # Exclusive prefix sum over num_warps warp totals. + if cutlass.const_expr(self.enable_warp_parallel_reduce): + # NEW: warp-parallel via block_scan.warp_scan (Hillis-Steele + # inclusive scan, log2(num_warps) shfl_up steps). Exclusive + # prefix = inclusive - val. Total = inclusive at last lane. + if warp_id == cutlass.Int32(0): + if lane < cutlass.Int32(self.num_warps): + val = smem_wcnt[lane] + inclusive = warp_scan(val, tidx, lane, num_threads_per_warp=self.num_warps) + smem_wcnt[lane] = inclusive - val # exclusive prefix + if lane == cutlass.Int32(self.num_warps - 1): + s_iscalars[0] = inclusive # cand_count (total) + else: + # tid==0 serial exclusive prefix. + if tidx == 0: + total = cutlass.Int32(0) + for w in cutlass.range_constexpr(self.num_warps): + cnt = smem_wcnt[w] + smem_wcnt[w] = total + total = total + cnt + s_iscalars[0] = total + cute.arch.barrier() + + # Each thread's write base = warp-prefix + intra-warp exclusive offset. + my_base = smem_wcnt[warp_id] + my_write_pos = my_base + my_excl_offset + + # ---- Stream-write loop ---- + thr_final = s_thr[0] + vec_w = cutlass.const_expr(self.vec_bits // self.dtype.width) + elem_bytes = cutlass.const_expr(self.dtype.width // 8) + vec_align = cutlass.const_expr(self.vec_align_bytes) + copy_atom = self._make_load_copy_atom() + row_addr = input_row.iterator.toint() + step_elem = cutlass.const_expr(num_threads * vec_w) + + n_aligned = (N // cutlass.Int32(vec_w)) * cutlass.Int32(vec_w) + wc = my_write_pos + ic = tidx * cutlass.Int32(vec_w) + step = cutlass.Int32(step_elem) + + # Phase3 unrolling: master gated by self.enable_phase3_unroll. + # When OFF, only the tail 1-way loop runs (matches the pre-unroll + # state of phase3_collect). When ON, the inner enable_unroll_4 + # controls the 4-way fast path. + if self.enable_phase3_unroll: + # Fast path: 4-way unrolled vec loop (4 loading instructions in flight). + if self.enable_unroll_4: + # ============================================================= + # unroll: cutlass.range(unroll=4). + # Each body loads 1 vec_w chunk; LLVM unrolls 4 iters at IR + # level. Same intent as the Phase-2 rewrite above. + # ============================================================= + rng_frag = cute.make_fragment((vec_w,), self.dtype) + big_iters = cutlass.Int32(0) + if N > ic + cutlass.Int32(vec_w - 1): + big_iters = (N - ic - cutlass.Int32(vec_w)) // cutlass.Int32( + step_elem + ) + cutlass.Int32(1) + + for k in cutlass.range(big_iters, unroll=4): + ic_local = ic + k * cutlass.Int32(step_elem) + src_ptr_k = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(ic_local) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src_k = cute.make_tensor(src_ptr_k, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src_k, rng_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = rng_frag[j] + else: + vj = cutlass.Float32(rng_frag[j]) + if vj >= thr_final and wc < cutlass.Int32(kCC): + smem_keys[wc] = vj + smem_vals[wc] = ic_local + cutlass.Int32(j) + wc = wc + cutlass.Int32(1) + # Advance ic past all consumed vec_w-aligned positions. + ic = ic + big_iters * cutlass.Int32(step_elem) + + # Tail vec loop: 1-way, handles remainder < 2*step. ic stays vec_w- + # aligned across the unroll loop (steps by num_threads*vec_w). + tail_frag = cute.make_fragment((vec_w,), self.dtype) + while ic + cutlass.Int32(vec_w - 1) < N: + src_ptr = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(ic) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src = cute.make_tensor(src_ptr, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src, tail_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = tail_frag[j] + else: + vj = cutlass.Float32(tail_frag[j]) + if vj >= thr_final and wc < cutlass.Int32(kCC): + smem_keys[wc] = vj + smem_vals[wc] = ic + cutlass.Int32(j) + wc = wc + cutlass.Int32(1) + ic = ic + step + + # Tail scalar loop (N % vec_w) + it = n_aligned + tidx + while it < N: + v = self._load_fp32(input_row, it) + if v >= thr_final and wc < cutlass.Int32(kCC): + smem_keys[wc] = v + smem_vals[wc] = it + wc = wc + cutlass.Int32(1) + it = it + cutlass.Int32(num_threads) + cute.arch.barrier() + + # ------------------------------------------------------------------ + # block_fused_snap_iter — P4 snap convergence inner step + # ------------------------------------------------------------------ + @cute.jit + def block_fused_snap_iter( + self, + smem_keys, + smem_wcnt, + smem_hist, # reused as scratch for s_up/s_down warp aggregates + s_thr, + s_iscalars, + count, + tidx, + warp_id, + lane, + ): + """One iteration of histogram snap. Updates s_iscalars[2]=cnt_lo (cge), + s_iscalars[3]=cnt_hi (cgt), and s_thr[0]=threshold (moves toward + the cnt-in-(kK_GT, kK_GE) bracket). + """ + kK = cutlass.const_expr(self.top_k) + num_threads = cutlass.const_expr(self.num_threads) + thr = s_thr[0] + + lge = cutlass.Int32(0) + lgt = cutlass.Int32(0) + s_up = cutlass.Float32(self.FLT_MAX) + s_down = cutlass.Float32(self.NEG_FLT_MAX) + + isi = tidx + while isi < count: + v = smem_keys[isi] + if v >= thr: + lge = lge + cutlass.Int32(1) + if v > thr: + lgt = lgt + cutlass.Int32(1) + # s_up = min(s_up, v) — hot path in block_fused_snap_iter (~10us) + s_up = _fmin_f32_inline(s_up, v) + if v < thr: + s_down = cute.arch.fmax(s_down, v) + isi = isi + cutlass.Int32(num_threads) + + # Pack lge/lgt into a single int32 so the warp reduction below + # sums both counts in one shuffle. Safe as long as each per-warp + # count stays < 2^16 = 65536 — which holds because lge/lgt are + # bounded by cand_count ≤ kC ≤ 6144 (see GvrParams). Bump kC + # past 65536 and this packing silently corrupts. + packed = (lge << cutlass.Int32(16)) | lgt + packed = self.warp_reduce_sum_i32(packed) + s_up = self.warp_reduce_min_f32(s_up) + s_down = self.warp_reduce_max_f32(s_down) + + # Lane 0 stages results into warp slots (smem_hist[0..NW-1] = s_up, + # smem_hist[NW..2*NW-1] = s_down stored as int32 bit-cast). + if lane == 0: + smem_wcnt[warp_id] = packed + smem_hist[warp_id] = float_as_uint32(s_up) + smem_hist[self.num_warps + warp_id] = float_as_uint32(s_down) + cute.arch.barrier() + + # 3-way block reduce + threshold bound update. + if cutlass.const_expr(self.enable_warp_parallel_reduce): + # NEW: warp-parallel 3-way reduce in warp 0. + if warp_id == cutlass.Int32(0): + v_tp = cutlass.Int32(0) + v_up = cutlass.Float32(self.FLT_MAX) + v_dn = cutlass.Float32(self.NEG_FLT_MAX) + if lane < cutlass.Int32(self.num_warps): + v_tp = smem_wcnt[lane] + vu_bits = smem_hist[lane] + vd_bits = smem_hist[self.num_warps + lane] + v_up = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vu_bits.ir_value()) + ) + v_dn = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vd_bits.ir_value()) + ) + tp = self.warp_reduce_sum_i32(v_tp) + total_up = self.warp_reduce_min_f32(v_up) + total_down = self.warp_reduce_max_f32(v_dn) + if lane == cutlass.Int32(0): + cge = tp >> cutlass.Int32(16) + cgt = tp & cutlass.Int32(0xFFFF) + s_iscalars[2] = cge + s_iscalars[3] = cgt + if cgt >= cutlass.Int32(kK): + if total_up < cutlass.Float32(self.FLT_MAX): + s_thr[0] = total_up + elif cge < cutlass.Int32(kK): + if total_down > cutlass.Float32(self.NEG_FLT_MAX): + s_thr[0] = total_down + else: + # tid==0 serial 3-way reduce. + if tidx == 0: + tp = cutlass.Int32(0) + total_up = cutlass.Float32(self.FLT_MAX) + total_down = cutlass.Float32(self.NEG_FLT_MAX) + for w in cutlass.range_constexpr(self.num_warps): + tp = tp + smem_wcnt[w] + vu = llvm.bitcast(cutlass.Float32.mlir_type, smem_hist[w].ir_value()) + vd = llvm.bitcast( + cutlass.Float32.mlir_type, smem_hist[self.num_warps + w].ir_value() + ) + vu_w = cutlass.Float32(vu) + vd_w = cutlass.Float32(vd) + total_up = _fmin_f32_inline(total_up, vu_w) + total_down = cute.arch.fmax(total_down, vd_w) + + cge = tp >> cutlass.Int32(16) + cgt = tp & cutlass.Int32(0xFFFF) + s_iscalars[2] = cge + s_iscalars[3] = cgt + if cgt >= cutlass.Int32(kK): + if total_up < cutlass.Float32(self.FLT_MAX): + s_thr[0] = total_up + elif cge < cutlass.Int32(kK): + if total_down > cutlass.Float32(self.NEG_FLT_MAX): + s_thr[0] = total_down + cute.arch.barrier() + + # ------------------------------------------------------------------ + # Phase 4: Histogram-based k-th selection + two-pass writeback + # ------------------------------------------------------------------ + @cute.jit + def phase4_histogram_snap( + self, + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count, + tidx, + warp_id, + lane, + ): + """Three branches by cand_count: + == kK : direct emit (fast path) + > kK : histogram k-th bin search + snap + 2-pass writeback + < kK : emit cand_count + pad with -self.FLT_MAX + """ + kK = cutlass.const_expr(self.top_k) + kBins = cutlass.const_expr(self.kNumBins) + num_threads = cutlass.const_expr(self.num_threads) + num_warps = cutlass.const_expr(self.num_warps) + bins_per_warp = cutlass.const_expr(kBins // self.num_warps) + + # ----- Branch A: cand_count == kK (fast path) ----- + if cand_count == cutlass.Int32(kK): + i4 = tidx + while i4 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[i4] = self.dtype(smem_keys[i4]) + output_indices_row[i4] = smem_vals[i4] + i4 = i4 + cutlass.Int32(num_threads) + elif cand_count > cutlass.Int32(kK): + # ----- Branch B: cand_count > kK → histogram snap ----- + + # Block min/max over keys[0:cand_count] + local_cmin = cutlass.Float32(self.FLT_MAX) + local_cmax = cutlass.Float32(self.NEG_FLT_MAX) + i5 = tidx + while i5 < cand_count: + v = smem_keys[i5] + local_cmin = _fmin_f32_inline(local_cmin, v) + local_cmax = cute.arch.fmax(local_cmax, v) + i5 = i5 + cutlass.Int32(num_threads) + cmin = self.warp_reduce_min_f32(local_cmin) + cmax = self.warp_reduce_max_f32(local_cmax) + # Stage warp results into smem_wcnt[w] (cmin) and smem_hist[w] (cmax) + # as bit-cast int32. cmax stored at smem_hist[0..NW-1]. + if lane == 0: + smem_wcnt[warp_id] = float_as_uint32(cmin) + smem_hist[warp_id] = float_as_uint32(cmax) + cute.arch.barrier() + + # Every thread independently recomputes block_min/block_max + # from the warp-staged smem slots (CUDA heuristic_topk.cuh:891-898 + # pattern). No tid==0 → s_thr broadcast → saves a block barrier. + bmin_r = cutlass.Float32(self.FLT_MAX) + bmax_r = cutlass.Float32(self.NEG_FLT_MAX) + # Unrolled num_warps times (16 or 32 — fixed at compile time). + for w in cutlass.range_constexpr(self.num_warps): + vmin_bits = smem_wcnt[w] + vmax_bits = smem_hist[w] + vmin = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmin_bits.ir_value()) + ) + vmax = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmax_bits.ir_value()) + ) + bmin_r = _fmin_f32_inline(bmin_r, vmin) + bmax_r = cute.arch.fmax(bmax_r, vmax) + if bmax_r <= bmin_r: + bmax_r = bmin_r + cutlass.Float32(1e-6) + # All threads must finish reading smem_hist[0..NW-1] (the + # warp-staged cmax slots above) before any thread starts + # zeroing smem_hist below — otherwise warp-0 thread-0 can + # zero smem_hist[0] while warp-N is still reading it, + # producing a 0 cmax → squashed bmax_r → all candidates land + # in bin 0 → wrong K-th threshold. Hit-rate-dependent race. + cute.arch.barrier() + + # Zero histogram (must zero ALL slots since smem_hist[0..NW-1] was + # used as cmax scratch above). + i6 = tidx + while i6 < cutlass.Int32(kBins): + smem_hist[i6] = cutlass.Int32(0) + i6 = i6 + cutlass.Int32(num_threads) + cute.arch.barrier() + + range1 = bmax_r - bmin_r + # inv1 = (kBins - 1 + 0.99) / range1 (range1 > 0 guaranteed by 1e-6 patch) + inv1 = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range1 + + # Build histogram by atomicAdd. + i7 = tidx + while i7 < cand_count: + vk = smem_keys[i7] + bin_f = (vk - bmin_r) * inv1 + bin_i = cutlass.Int32(bin_f) + if bin_i < cutlass.Int32(0): + bin_i = cutlass.Int32(0) + if bin_i > cutlass.Int32(kBins - 1): + bin_i = cutlass.Int32(kBins - 1) + atomicAdd(smem_hist.iterator + bin_i, cutlass.Int32(1)) + i7 = i7 + cutlass.Int32(num_threads) + cute.arch.barrier() + + # ---- Parallel k-th bin search (3-step) ---- + # Step 1: each warp sums BINS_PER_WARP bins (high→low slice) + warp_bin_sum = cutlass.Int32(0) + for jb in cutlass.range_constexpr(bins_per_warp): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb) + ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + if lane == 0: + smem_wcnt[warp_id] = warp_bin_sum + cute.arch.barrier() + + # Step 2: tid==0 finds target warp; stores prefix-count + warp index + # into s_iscalars[2] (=cnt_lo: prefix before target warp) + # and s_iscalars[3] (=cnt_hi: target warp index) + if tidx == 0: + cum = cutlass.Int32(0) + tw = cutlass.Int32(num_warps - 1) + found = cutlass.Int32(0) + for w2 in cutlass.range_constexpr(self.num_warps): + cum = cum + smem_wcnt[w2] + if cum >= cutlass.Int32(kK) and found == cutlass.Int32(0): + tw = cutlass.Int32(w2) + found = cutlass.Int32(1) + # Recompute prefix BEFORE target warp + cum2 = cutlass.Int32(0) + for w3 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w3) < tw: + cum2 = cum2 + smem_wcnt[w3] + s_iscalars[2] = cum2 # prefix + s_iscalars[3] = tw # target warp index + cute.arch.barrier() + + # Step 3: target warp's lane 0 scans BINS_PER_WARP bins → threshold + # NOTE: This loop runs in a single thread (warp 0 lane 0). Tried + # changing range_constexpr → cutlass.range(unroll=1) to mirror + # CUDA's `for+break` (gets nvcc to keep runtime branch). SASS + # I2FP dropped 64→1, total inst -544 for fp32, but perf was + # WORSE (-7pp on fp32 large N, -14pp on bf16 synth). The runtime + # branch/counter overhead in a 1-thread serial path exceeds the + # static I2FP/FMUL/FFMA waste — those 60+ fp ops are essentially + # free for one thread. Keeping range_constexpr. + target_warp = s_iscalars[3] + if warp_id == target_warp and lane == cutlass.Int32(0): + base_cum = s_iscalars[2] + thr_local = bmin_r + bmin_local = bmin_r + set_done = cutlass.Int32(0) + for jb2 in cutlass.range_constexpr(bins_per_warp): + bidx2 = ( + cutlass.Int32(kBins - 1) + - target_warp * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb2) + ) + base_cum = base_cum + smem_hist[bidx2] + if base_cum >= cutlass.Int32(kK) and set_done == cutlass.Int32(0): + thr_local = bmin_local + cutlass.Float32(bidx2) * range1 / cutlass.Float32( + kBins + ) + set_done = cutlass.Int32(1) + s_thr[0] = thr_local + cute.arch.barrier() + + # ---- Snap convergence loop ---- + # snap_limit = cand_count (matches CUDA heuristic_topk.cuh:985). + # The older `cand_count / 4` bound silently accepted a non- + # converged threshold in ~0.09% of adversarial distributions + # (Pass 1 then picked K from cgt > kK candidates in scan order, + # missing some true top-K members). Common path still converges + # in 1-3 iters; the higher upper bound only affects long-tail + # cells. + snap_limit = cand_count + + # Loop with runtime break-via-guard. We unroll up to a safe ceiling + # then loop the rest with a while. + # For simplicity: while loop with explicit break-flag. + si = cutlass.Int32(0) + done_snap = cutlass.Int32(0) + while si < snap_limit and done_snap == cutlass.Int32(0): + self.block_fused_snap_iter( + smem_keys, + smem_wcnt, + smem_hist, + s_thr, + s_iscalars, + cand_count, + tidx, + warp_id, + lane, + ) + # After block_fused_snap_iter, s_iscalars[2]=cge, s_iscalars[3]=cgt. + if s_iscalars[3] < cutlass.Int32(kK) and s_iscalars[2] >= cutlass.Int32(kK): + done_snap = cutlass.Int32(1) + si = si + cutlass.Int32(1) + + # ---- Two-pass output writeback (ballot+popc, CUDA-style) ---- + # Per-iter: ballot collects emit flags into a 32-bit mask, popc gives + # within-warp count, lane 0 atomicAdds to out_count, shuffle broadcasts + # the base offset. No per-iter barriers — only 1 barrier between passes. + sel_thr = s_thr[0] + if tidx == 0: + s_iscalars[4] = cutlass.Int32(0) # out_count + cute.arch.barrier() + + # Pass 1: v > sel_thr — strided over (warp_id*WARP_SIZE, ...) like CUDA. + # `if mask_gt != 0` mirrors CUDA heuristic_topk.cuh:1020 — when no + # lane in the warp emits, skip the popc + atomicAdd + shuffle round + # trip (the atomicAdd alone is ~10-30 cycles on a SMEM atomic unit). + base_w = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w < cand_count: + ix1 = base_w + lane + emit_gt = cutlass.Int32(0) + v_p1 = cutlass.Float32(self.NEG_FLT_MAX) + if ix1 < cand_count: + v_p1 = smem_keys[ix1] + if v_p1 > sel_thr: + emit_gt = cutlass.Int32(1) + mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) + if mask_gt != cutlass.Uint32(0): + cnt_gt = cutlass.Int32(cute.arch.popc(mask_gt)) + lane_mask_gt = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) + moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask_gt)) + bp_gt = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_gt = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_gt, + ) + bp_gt = cute.arch.shuffle_sync(bp_gt, cutlass.Int32(0)) + wpos_p1 = bp_gt + moff_gt + if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p1] = self.dtype(v_p1) + output_indices_row[wpos_p1] = smem_vals[ix1] + base_w = base_w + cutlass.Int32(num_threads) + cute.arch.barrier() + + # Pass 2: v == sel_thr (same pattern as Pass 1, same `if mask` guard). + # Empty-iter case is much more common here because only tie-values + # at the K-th rank emit. + base_w2 = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w2 < cand_count: + ix2 = base_w2 + lane + emit_eq = cutlass.Int32(0) + v_p2 = cutlass.Float32(self.NEG_FLT_MAX) + if ix2 < cand_count: + v_p2 = smem_keys[ix2] + if v_p2 == sel_thr: + emit_eq = cutlass.Int32(1) + mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) + if mask_eq != cutlass.Uint32(0): + cnt_eq = cutlass.Int32(cute.arch.popc(mask_eq)) + lane_mask_eq = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) + moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask_eq)) + bp_eq = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_eq = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_eq, + ) + bp_eq = cute.arch.shuffle_sync(bp_eq, cutlass.Int32(0)) + wpos_p2 = bp_eq + moff_eq + if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p2] = self.dtype(v_p2) + output_indices_row[wpos_p2] = smem_vals[ix2] + base_w2 = base_w2 + cutlass.Int32(num_threads) + cute.arch.barrier() + + # Pad remainder with -self.FLT_MAX / -1 + filled_par = s_iscalars[4] + if filled_par > cutlass.Int32(kK): + filled_par = cutlass.Int32(kK) + ipad = filled_par + tidx + while ipad < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[ipad] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[ipad] = cutlass.Int32(-1) + ipad = ipad + cutlass.Int32(num_threads) + else: + # ----- Branch C: cand_count < kK ----- + # Emit cand_count + pad + i10 = tidx + while i10 < cand_count: + if cutlass.const_expr(self.return_output_values): + output_values_row[i10] = self.dtype(smem_keys[i10]) + output_indices_row[i10] = smem_vals[i10] + i10 = i10 + cutlass.Int32(num_threads) + i11 = cand_count + tidx + while i11 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[i11] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[i11] = cutlass.Int32(-1) + i11 = i11 + cutlass.Int32(num_threads) + + # ------------------------------------------------------------------ + # Main kernel — one CTA per row + # CUDA source: heuristicTopKDecode.cu:49-93 (heuristicTopKMultiRowKernel) + # ------------------------------------------------------------------ + @cute.kernel + def gvr_topk_kernel( + self, + input_data: cute.Tensor, # [numRows, stride0] dtype + pre_idx: cute.Tensor, # [numRows / next_n, pre_idx_stride] int32 + seq_lens: cute.Tensor, # [numRows / next_n] int32 + output_values: cute.Tensor, # [numRows, top_k] dtype + output_indices: cute.Tensor, # [numRows, top_k] int32 + ): + """One CTA per row. Grid = (num_rows, 1, 1).""" + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + + next_n = cutlass.const_expr(self.next_n) + top_k = cutlass.const_expr(self.top_k) + num_threads = cutlass.const_expr(self.num_threads) + num_warps = cutlass.const_expr(self.num_warps) + kC = cutlass.const_expr(self.kC) + kNumBins = cutlass.const_expr(self.kNumBins) + + warp_id = tidx // self.WARP_SIZE + lane = tidx & (self.WARP_SIZE - 1) + + row_idx = bidx + pre_idx_row_idx = row_idx // next_n + # Temporal-shift offset, mirroring heuristicTopKDecode.cu PR #14219: + # cr == 1 (V3.2): (row % next_n) + 1 maps prev-step indices into this + # step's KV space (+1 for the newly appended token). + # cr > 1 (V4): 0 — in compressed-index space, new entries are + # appended at the end so prev indices remain valid as-is. + if cutlass.const_expr(self.compress_ratio == 1): + pre_idx_offset = cutlass.Int32(row_idx % next_n) + cutlass.Int32(1) + else: + pre_idx_offset = cutlass.Int32(0) + + # Per-row length. seq_lens is in uncompressed-token space; logits/preIdx + # live in compressed-token-index space when cr > 1 → divide by cr. + # (For cr == 1, the divide is a no-op, but the explicit form mirrors + # the CUDA branch and keeps the IR straightforward to read.) + seq_len = seq_lens[pre_idx_row_idx] + actual_kv_len = ( + seq_len - cutlass.Int32(next_n) + cutlass.Int32(row_idx % next_n) + cutlass.Int32(1) + ) + if cutlass.const_expr(self.compress_ratio == 1): + N = actual_kv_len + else: + N = actual_kv_len // cutlass.Int32(self.compress_ratio) + + # Slice per-row views. + input_row = input_data[row_idx, None] + pre_idx_row = pre_idx[pre_idx_row_idx, None] + # When return_output_values=False, ``output_values`` is None at + # launch and the gated writes below are compiled out; slicing into + # None would crash so we keep the view None as well. + if cutlass.const_expr(self.return_output_values): + output_values_row = output_values[row_idx, None] + else: + output_values_row = None + output_indices_row = output_indices[row_idx, None] + pre_idx_count = pre_idx.shape[1] + + griddepcontrol_wait() + + # ---- Shared memory allocation ---- + smem = SmemAllocator() + # keys[kC] fp32 (P3 candidate values; smem keys always fp32 even for half-prec) + # Use fp32 even for half-prec to make secant search algorithm keep the accuracy/precision and converge faster. + smem_keys = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((kC,), order=(0,)), + byte_alignment=128, + ) + # vals[kC] int32 (P3 candidate indices) + smem_vals = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((kC,), order=(0,)), + byte_alignment=128, + ) + # histogram[kNumBins] int32 (P4 only) + smem_hist = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((kNumBins,), order=(0,)), + byte_alignment=128, + ) + # per_thread_counts[BLOCK_SIZE] int32 (P2/P3 cached counts) + smem_ptcnt = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((num_threads,), order=(0,)), + byte_alignment=128, + ) + # warp_counts[NUM_WARPS] int32 (P3 prefix-sum scratch) + smem_wcnt = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=128, + ) + # Phase-1 warp aggregates (fp32 + int32; ~256 bytes total) + smem_wmin = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=64, + ) + smem_wmax = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=64, + ) + smem_wsum = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=64, + ) + smem_wcnt_p1 = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=64, + ) + # Float scalars: threshold, val_lo, val_hi + s_thr = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((3,), order=(0,)), + byte_alignment=16, + ) + # Int scalars: cand_count, done, cnt_lo, cnt_hi, out_count + s_iscalars = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((5,), order=(0,)), + byte_alignment=16, + ) + + # ---- Degenerate path: N <= top_k → copy input as-is ---- + if N <= cutlass.Int32(top_k): + jd = tidx + while jd < N: + if cutlass.const_expr(self.return_output_values): + output_values_row[jd] = input_row[jd] + output_indices_row[jd] = cutlass.Int32(jd) + jd = jd + cutlass.Int32(num_threads) + jp = N + cutlass.Int32(tidx) + while jp < cutlass.Int32(top_k): + if cutlass.const_expr(self.return_output_values): + output_values_row[jp] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[jp] = cutlass.Int32(-1) + jp = jp + cutlass.Int32(num_threads) + else: + # ================================================================= + # Phase 1 — preIdx Min/Max/Mean + # ================================================================= + self.phase1_preidx_stats( + input_row, + N, + pre_idx_row, + pre_idx_count, + pre_idx_offset, + smem_wmin, + smem_wmax, + smem_wsum, + smem_wcnt_p1, + s_thr, + s_iscalars, + tidx, + warp_id, + lane, + ) + + # Degenerate threshold init: val_hi <= -self.FLT_MAX or val_lo >= val_hi + v_lo = s_thr[1] + v_hi = s_thr[2] + if v_hi <= cutlass.Float32(self.NEG_FLT_MAX) or v_lo >= v_hi: + if tidx == 0: + # Emit identity output (first min(top_k, N) indices) + emit_count = cutlass.Int32(top_k) if cutlass.Int32(top_k) < N else N + je = cutlass.Int32(0) + while je < emit_count: + output_indices_row[je] = je + if cutlass.const_expr(self.return_output_values): + output_values_row[je] = input_row[je] + je = je + cutlass.Int32(1) + else: + # ============================================================= + # Phase 2 — Secant threshold search + # ============================================================= + self.phase2_secant_search( + input_row, + N, + smem_ptcnt, + smem_wcnt, + s_thr, + s_iscalars, + tidx, + warp_id, + lane, + ) + + # ============================================================= + # Phase 3 — Ballot-free candidate collect + # ============================================================= + self.phase3_collect_candidates( + input_row, + N, + smem_keys, + smem_vals, + smem_ptcnt, + smem_wcnt, + s_thr, + s_iscalars, + tidx, + warp_id, + lane, + ) + + # ============================================================= + # Phase 4 — Histogram snap + writeback (top-K from candidates) + # ============================================================= + # cand_count = min(s_iscalars[0], kCC) + cand_count_p4 = s_iscalars[0] + if cand_count_p4 > cutlass.Int32(self.kC): + cand_count_p4 = cutlass.Int32(self.kC) + + self.phase4_histogram_snap( + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count_p4, + tidx, + warp_id, + lane, + ) + + griddepcontrol_launch_dependents() + + # ------------------------------------------------------------------ + # Host-side launcher + # ------------------------------------------------------------------ + @cute.jit + def __call__( + self, + input_data: cute.Tensor, + pre_idx: cute.Tensor, + seq_lens: cute.Tensor, + output_values: cute.Tensor, # or None. + output_indices: cute.Tensor, + stream, + ): + num_rows = input_data.shape[0] + self.gvr_topk_kernel( + input_data, + pre_idx, + seq_lens, + output_values, + output_indices, + ).launch( + grid=(num_rows, 1, 1), + block=(self.num_threads, 1, 1), + stream=stream, + use_pdl=TRTLLM_ENABLE_PDL, + min_blocks_per_mp=self.min_blocks_per_mp, + ) + + +__all__ = ["GvrTopKKernel", "GvrParams"] diff --git a/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py b/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py new file mode 100644 index 000000000000..c5b5521b7982 --- /dev/null +++ b/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py @@ -0,0 +1,506 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Standalone driver + pytest sweep for the cuTe DSL GVR Top-K kernel. + +Compares the kernel output against ``torch.topk`` using tie-aware set +equality. This file exposes every knob +(T, V, ``min_blocks_per_mp``, warp-parallel reduce, both unroll switches, +``use_constant_hint``, ``compress_ratio``, ``max_seq_len`` hint) so bench +scripts can override the heuristic. + +**Not in CI** — this file imports the DSL kernel module directly +(no ``trtllm`` runtime dep), to enable knob-A/B development outside the +production op. + +Two usage modes: + +* `python -m pytest run_gvr_topk.py`` — exhaustive parameterized correctness sweep + (dtype × K × N × seed × next_n × T × V × warp-parallel-reduce). +* ``python run_gvr_topk.py --dtype bf16 --top_k 1024 --N 8192`` — + single-case correctness verification on user-specified shape; knob + overrides via ``--num_threads`` / ``--use_256bit_load`` / etc. +""" + +import argparse +import functools +import sys +from pathlib import Path +from typing import Optional + +import cutlass +import cutlass.cute as cute +import pytest +import torch + +try: + from tensorrt_llm._torch.cute_dsl_kernels.blackwell.top_k.gvr_topk_decode import GvrTopKKernel +except (ModuleNotFoundError, ImportError): + sys.path.insert(0, str(Path(__file__).parents[4] / "tensorrt_llm/_torch/cute_dsl_kernels")) + from blackwell.top_k.gvr_topk_decode import GvrTopKKernel # type: ignore[no-redef] + + +_DTYPE_TORCH_TO_CUTE = { + torch.float32: cutlass.Float32, + torch.bfloat16: cutlass.BFloat16, + torch.float16: cutlass.Float16, +} + + +@functools.cache +def _compile( + cute_dtype, + top_k: int, + next_n: int, + enable_unroll_4: bool, + enable_phase3_unroll: bool, + use_constant_hint: bool, + min_blocks_per_mp: int, + use_256bit_load: bool, + num_threads_per_block: int, + enable_warp_parallel_reduce: bool, + compress_ratio: int, + return_output_values: bool, +): + """JIT-compile the GVR kernel for a specific knob combination. + + ``functools.cache`` keys on all args so repeated calls in the same + process reuse the compiled kernel without an explicit module-level dict. + """ + n_rows = cute.sym_int() + n_cols = cute.sym_int() + n_batch = cute.sym_int() + in_align = 32 if use_256bit_load else 16 + input_fake = cute.runtime.make_fake_compact_tensor( + cute_dtype, + (n_rows, n_cols), + stride_order=(1, 0), + assumed_align=in_align, + ) + pre_idx_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_batch, top_k), + stride_order=(1, 0), + assumed_align=16, + ) + seq_lens_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_batch,), + stride_order=(0,), + ) + # When return_output_values=False the kernel skips all STG.value + # writes; pass None so cute.compile doesn't materialize the value + # output placeholder. + out_values_fake = ( + cute.runtime.make_fake_compact_tensor( + cute_dtype, + (n_rows, top_k), + stride_order=(1, 0), + assumed_align=16, + ) + if return_output_values + else None + ) + out_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, top_k), + stride_order=(1, 0), + assumed_align=16, + ) + fake_stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + kernel = GvrTopKKernel( + dtype=cute_dtype, + top_k=top_k, + next_n=next_n, + num_threads=num_threads_per_block, + enable_unroll_4=enable_unroll_4, + enable_phase3_unroll=enable_phase3_unroll, + use_constant_hint=use_constant_hint, + min_blocks_per_mp=min_blocks_per_mp, + use_256bit_load=use_256bit_load, + enable_warp_parallel_reduce=enable_warp_parallel_reduce, + compress_ratio=compress_ratio, + return_output_values=return_output_values, + ) + return cute.compile( + kernel, + input_fake, + pre_idx_fake, + seq_lens_fake, + out_values_fake, + out_indices_fake, + stream=fake_stream, + options="--enable-tvm-ffi", + ) + + +def gvr_topk_decode( + logits: torch.Tensor, + pre_idx: torch.Tensor, + seq_lens: torch.Tensor, + top_k: int, + next_n: int = 1, + out_values: Optional[torch.Tensor] = None, + out_indices: Optional[torch.Tensor] = None, + num_sms: int = 148, # default number of sms in a B200 + enable_unroll_4: Optional[bool] = None, + enable_phase3_unroll: Optional[bool] = None, + use_constant_hint: bool = False, + min_blocks_per_mp: Optional[int] = None, + use_256bit_load: Optional[bool] = None, + num_threads_per_block: Optional[int] = None, + enable_warp_parallel_reduce: Optional[bool] = None, + compress_ratio: int = 1, + max_seq_len: Optional[int] = None, + return_output_values: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + """CuTe DSL GVR Top-K wrapper with every tuning knob exposed. + + ``None``-valued knobs are resolved via the production auto-heuristic + (same rules as ``CuteDSLGvrTopKDecodeRunner.forward``); concrete values + override the heuristic for A/B testing. + + Args: + logits: ``[num_rows, max_S]`` float32 / bfloat16 / float16. + pre_idx: ``[num_rows // next_n, pre_idx_count]`` int32. + ``pre_idx[..., 0]`` must be the argmax index — indexer invariant. + seq_lens: ``[num_rows // next_n]`` int32 (uncompressed-token space). + top_k: K ∈ {512, 1024, 2048} — compile-time specialized. + next_n: Temporal stride for V3.2 ``preIdxOffset = (row % next_n) + 1``. + compress_ratio: KV-indexer compression factor (1 = DSv3.2, 4 = DSv4). + When != 1, logits/preIdx live in compressed-token-index space: + ``N`` is divided by ``compress_ratio`` and ``preIdxOffset`` + is forced to 0. Mirrors heuristicTopKDecode.cu PR #14219. + max_seq_len: Graph-safe hint for peak ``logits.shape[1]`` at replay + (same compressed-token-index space as ``logits``). + + Returns: + ``(out_values, out_indices)`` both shaped ``[num_rows, top_k]``. + """ + assert logits.is_cuda, "logits must be on CUDA" + assert logits.dim() == 2, f"logits must be 2D, got {logits.shape}" + assert pre_idx.dim() == 2 and pre_idx.dtype == torch.int32 + assert seq_lens.dim() == 1 and seq_lens.dtype == torch.int32 + + if logits.dtype not in _DTYPE_TORCH_TO_CUTE: + raise ValueError(f"Unsupported logits dtype: {logits.dtype}") + cute_dtype = _DTYPE_TORCH_TO_CUTE[logits.dtype] + + num_rows = logits.shape[0] + if return_output_values: + if out_values is None: + out_values = torch.empty((num_rows, top_k), dtype=logits.dtype, device=logits.device) + if out_indices is None: + out_indices = torch.empty((num_rows, top_k), dtype=torch.int32, device=logits.device) + + # Resolve None defaults via the same heuristic as the production Runner. + if enable_unroll_4 is None: + enable_unroll_4 = True + if enable_phase3_unroll is None: + enable_phase3_unroll = True + + N_cols = logits.shape[1] + N_dec = max_seq_len if max_seq_len is not None else N_cols + if num_threads_per_block is None: + if max_seq_len is not None and logits.dtype != torch.float32: + n_thresh_t = 131072 + else: + n_thresh_t = 65536 + num_threads_per_block = 1024 if (num_rows <= num_sms and N_dec >= n_thresh_t) else 512 + if use_256bit_load is None: + use_256bit_load = logits.dtype == torch.float32 and N_dec >= 16384 + if enable_warp_parallel_reduce is None: + enable_warp_parallel_reduce = num_threads_per_block == 1024 + + if min_blocks_per_mp is None: + vec_bits_host = 256 if use_256bit_load else 128 + vec_w_host = vec_bits_host // (32 if logits.dtype == torch.float32 else 16) + n_vec_iters = max(1, N_dec // (num_threads_per_block * vec_w_host)) + is_fp32 = logits.dtype == torch.float32 + if is_fp32: + if n_vec_iters < 4: + min_blocks_per_mp = 0 + elif num_rows <= num_sms: + min_blocks_per_mp = 1 + elif num_sms * 2 < num_rows <= num_sms * 3 and N_dec <= 32768: + # Wave-fit + latency-bound; at N>=65K mb=2 wins (bandwidth-bound). + min_blocks_per_mp = 3 + else: + min_blocks_per_mp = 2 + else: + if num_rows > num_sms: + min_blocks_per_mp = 3 + elif n_vec_iters < 4: + min_blocks_per_mp = 0 + else: + min_blocks_per_mp = 1 + + compiled = _compile( + cute_dtype, + top_k, + next_n, + enable_unroll_4, + enable_phase3_unroll, + use_constant_hint, + min_blocks_per_mp, + use_256bit_load, + num_threads_per_block, + enable_warp_parallel_reduce, + compress_ratio, + return_output_values, + ) + # When return_output_values=False the kernel was compiled to skip + # STG.value and accepts None for the value-output slot. + compiled( + logits, + pre_idx, + seq_lens, + out_values if return_output_values else None, + out_indices, + ) + if return_output_values: + return out_values, out_indices + else: + return None, out_indices + + +# ---- Correctness helpers ---------------------------------------------------- +def _make_inputs( + num_rows: int, + N: int, + top_k: int, + dtype: torch.dtype, + seed: int, + next_n: int = 1, + compress_ratio: int = 1, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build (logits, pre_idx, seq_lens) for a multi-row test. + + Shapes: + logits : [num_rows, N] — compressed-token-index space + pre_idx : [num_rows // next_n, top_k] — argmax in slot 0 (indexer invariant) + seq_lens: [num_rows // next_n] — UNCOMPRESSED-token space + + Kernel divides ``seq_lens`` by ``compress_ratio`` internally. Setting + ``seq_lens = N * cr`` makes the kernel's + ``N_kernel = (seq_lens - next_n + ofs + 1) // cr`` match the reference + ``N_eff = N - next_n + ofs + 1`` for next_n in {1, 2} (covers the + current sweep). For cr=1 this reduces to ``seq_lens = N``. + + ``pre_idx.shape[1] == top_k`` per CUDA invariant (heuristic_topk.cuh:810: + ``preIdxCount == topK`` is a dispatch precondition). + """ + torch.manual_seed(seed) + device = "cuda" + logits_f32 = torch.randn(num_rows, N, dtype=torch.float32, device=device) * 2.0 + logits = logits_f32.to(dtype) + num_groups = num_rows // next_n + # argmax must come from the effective scan range, not full N — for + # next_n>1 the kernel's row-0 N_eff is only (N - next_n + 1) cols. + effective_len = N - next_n + 1 + argmax_idx = logits[::next_n, :effective_len].argmax(dim=-1).int() + pre_idx = torch.zeros(num_groups, top_k, dtype=torch.int32, device=device) + pre_idx[:, 0] = argmax_idx + for j in range(1, top_k): + pre_idx[:, j] = j + # seq_lens is uncompressed; ``N * cr`` makes the kernel's + # ``N_kernel = (seq_lens - next_n + ofs + 1) // cr`` match ref's + # ``N_eff = N - next_n + ofs + 1`` for next_n in {1, 2}. (For cr=1 + # this reduces to seq_lens = N.) + seq_lens_val = N * compress_ratio + seq_lens = torch.full((num_groups,), seq_lens_val, dtype=torch.int32, device=device) + return logits, pre_idx, seq_lens + + +def _tie_aware_correct( + kernel_idxs: torch.Tensor, + logits: torch.Tensor, + seq_lens: torch.Tensor, + top_k: int, + next_n: int, + compress_ratio: int = 1, +) -> tuple[bool, str]: + """Multi-row tie-aware correctness check with strict sort+allclose. + + Per row r: scan range mirrors the kernel formula (see + ``GvrTopKKernel.gvr_topk_kernel``): + + actual_kv_len = seq_lens[r // next_n] - next_n + (r % next_n) + 1 + N_eff = actual_kv_len // compress_ratio # cr=1 is identity + + Reference ``torch.topk`` is masked to this range so reference and + kernel scan exactly the same columns under any (next_n, cr) combo. + + Returns ``(False, message)`` on the first failing row; ``(True, "ok")`` + when all rows pass. Sort+allclose catches the "drop-strictly-above + + add-tied-at-kth" bug that count-below-kth alone misses on ties. + """ + num_rows = kernel_idxs.shape[0] + logits_f32 = logits.to(torch.float32) + seq_lens_host = seq_lens.cpu().tolist() + for row in range(num_rows): + ofs = row % next_n + actual_kv_len = int(seq_lens_host[row // next_n]) - next_n + ofs + 1 + N_eff = actual_kv_len // compress_ratio + if N_eff < top_k: + # Degenerate path — skip; caller's main() guards against this. + continue + row_logits = logits_f32[row, :N_eff] + topk_vals, _ = torch.topk(row_logits, k=top_k, largest=True, sorted=True) + kth_value = topk_vals[-1].item() + sel = [int(i) for i in kernel_idxs[row].cpu().tolist() if i >= 0] + if any(i >= N_eff for i in sel): + return False, f"row={row}: out-of-range index" + if len(set(sel)) != len(sel): + return False, f"row={row}: duplicate indices" + if len(sel) != top_k: + return False, f"row={row}: returned {len(sel)} indices, expected {top_k}" + sel_vals = row_logits[torch.tensor(sel, device=logits.device, dtype=torch.long)] + n_below = int((sel_vals < kth_value).sum().item()) + if n_below > 0: + return False, (f"row={row}: {n_below} selected values < Kth-rank ({kth_value:.6f})") + # Strict: sorted-value multiset must match torch.topk. + sel_sorted, _ = sel_vals.sort(descending=True) + if not torch.allclose(sel_sorted, topk_vals, rtol=1e-5, atol=1e-5): + max_diff = (sel_sorted - topk_vals).abs().max().item() + return False, f"row={row}: sorted-value mismatch (max diff {max_diff:.4e})" + return True, "ok" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device required") +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("top_k", [512, 1024, 2048]) +@pytest.mark.parametrize("N", [4096, 65536]) +@pytest.mark.parametrize("next_n", [1]) +@pytest.mark.parametrize("batch_size", [1, 32]) +@pytest.mark.parametrize("use_256bit_load", [False, True]) +@pytest.mark.parametrize("num_threads_per_block", [512, 1024]) +@pytest.mark.parametrize("enable_warp_parallel_reduce", [False, True]) +def test_gvr_topk_decode( + dtype: torch.dtype, + top_k: int, + N: int, + next_n: int, + batch_size: int, + use_256bit_load: bool, + num_threads_per_block: int, + enable_warp_parallel_reduce: bool, +) -> None: + # Kernel scans `N_eff = seq_lens[0] - next_n + (row_idx % next_n) + 1` + # columns. Smallest row's N_eff = N - next_n + 1. Degenerate path + # (N_eff <= top_k) is a separate code branch — skip here. + if N - next_n + 1 < top_k: + pytest.skip("N_eff < top_k is degenerate; the kernel requires N_eff >= top_k") + seed = 42 + num_rows = batch_size * next_n + logits, pre_idx, seq_lens = _make_inputs( + num_rows, + N, + top_k, + dtype, + seed, + next_n=next_n, + compress_ratio=1, + ) + num_sms = torch.cuda.get_device_properties(0).multi_processor_count + _, out_idxs = gvr_topk_decode( + logits, + pre_idx, + seq_lens, + top_k, + next_n=next_n, + num_sms=num_sms, + use_256bit_load=use_256bit_load, + num_threads_per_block=num_threads_per_block, + enable_warp_parallel_reduce=enable_warp_parallel_reduce, + return_output_values=False, + ) + torch.cuda.synchronize() + ok, msg = _tie_aware_correct(out_idxs, logits, seq_lens, top_k, next_n) + assert ok, ( + f"dtype={dtype} K={top_k} N={N} seed={seed} next_n={next_n} " + f"batch_size={batch_size} use_256bit_load={use_256bit_load} " + f"num_threads_per_block={num_threads_per_block} " + f"enable_warp_parallel_reduce={enable_warp_parallel_reduce}: {msg}" + ) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--dtype", choices=["fp32", "bf16", "fp16"], default="bf16") + p.add_argument("--top_k", type=int, default=1024, choices=[512, 1024, 2048]) + p.add_argument("--N", type=int, default=8192) + p.add_argument("--batch_size", type=int, default=1) + p.add_argument("--next_n", type=int, default=1) + p.add_argument("--num_sms", type=int, default=148) + p.add_argument("--compress_ratio", type=int, default=1, choices=[1, 4]) + p.add_argument("--num_threads", type=int, default=512) + p.add_argument("--use_256bit_load", action="store_true") + p.add_argument("--min_blocks_per_mp", type=int, default=None) + p.add_argument("--enable_warp_parallel_reduce", action="store_true") + p.add_argument("--disable_unroll_4", action="store_true") + p.add_argument("--disable_phase3_unroll", action="store_true") + p.add_argument("--use_constant_hint", action="store_true") + p.add_argument("--max_seq_len", type=int, default=None) + args = p.parse_args() + + dtype = {"fp32": torch.float32, "bf16": torch.bfloat16, "fp16": torch.float16}[args.dtype] + + effective_len = args.N - args.next_n + 1 + if effective_len < args.top_k: + print(f"FAIL: N_eff={effective_len} < top_k={args.top_k} (degenerate path)") + sys.exit(1) + + seed = 42 + num_rows = args.batch_size * args.next_n + logits, pre_idx, seq_lens = _make_inputs( + num_rows, + args.N, + args.top_k, + dtype, + seed, + next_n=args.next_n, + compress_ratio=args.compress_ratio, + ) + knobs = dict( + num_threads_per_block=args.num_threads, + use_256bit_load=args.use_256bit_load, + min_blocks_per_mp=args.min_blocks_per_mp, + enable_warp_parallel_reduce=args.enable_warp_parallel_reduce, + enable_unroll_4=not args.disable_unroll_4, + enable_phase3_unroll=not args.disable_phase3_unroll, + use_constant_hint=args.use_constant_hint, + compress_ratio=args.compress_ratio, + max_seq_len=args.max_seq_len, + return_output_values=False, + ) + print( + f"config: dtype={args.dtype} top_k={args.top_k} N={args.N} " + f"batch_size={args.batch_size} next_n={args.next_n}, num_sms={args.num_sms}" + ) + print(f"knobs: {knobs}") + + _, out_idxs = gvr_topk_decode( + logits, + pre_idx, + seq_lens, + args.top_k, + next_n=args.next_n, + num_sms=args.num_sms, + **knobs, + ) + torch.cuda.synchronize() + + ok, msg = _tie_aware_correct( + out_idxs, + logits, + seq_lens, + args.top_k, + args.next_n, + compress_ratio=args.compress_ratio, + ) + print(f"correctness: {'PASS' if ok else f'FAIL ({msg})'}") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py b/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py new file mode 100644 index 000000000000..d0a5c2085344 --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py @@ -0,0 +1,240 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +import tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops # noqa: F401 +from tensorrt_llm._utils import get_sm_version + +skip_not_sm100 = pytest.mark.skipif( + get_sm_version() not in (100, 103), + reason=f"CuTe DSL GVR Top-K only supports SM 100/103, got SM {get_sm_version()}", +) + + +def _make_inputs( + num_rows: int, + N: int, + top_k: int, + dtype: torch.dtype, + next_n: int, + seed: int, + compress_ratio: int = 1, + preidx_hit_rate: float = 0.0, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build (logits, pre_idx, seq_lens) for the op. + + ``logits`` lives in compressed-token-index space (``N = N_uncompressed / + compress_ratio``). ``seq_lens`` is in UNCOMPRESSED space — kernel divides + by ``compress_ratio`` internally. ``pre_idx[..., 0]`` is the per-group + argmax (indexer invariant). + + ``preidx_hit_rate`` controls how many ``pre_idx[..., 1:]`` slots are + real ``torch.topk`` indices vs random fillers. 0.0 = current worst-case + (only slot 0 meaningful, rest = junk arange); 0.3-0.8 = realistic + production (V3.2 ~40%, V4 Pro ~75%) where the kernel's Guess phase + short-circuits. Always preserves the ``pre_idx[..., 0] = argmax`` + invariant on slot 0. + """ + torch.manual_seed(seed) + device = "cuda" + logits_f32 = torch.randn(num_rows, N, dtype=torch.float32, device=device) * 2.0 + logits = logits_f32.to(dtype) + + num_groups = num_rows // next_n + # argmax must come from the effective scan range, not full N — for + # next_n>1 the kernel's row-0 N_eff is only (N - next_n + 1) cols, so + # an argmax landing in the [N_eff, N) tail would violate the + # pre_idx[..., 0] invariant. Use N_eff = N - next_n + 1 (the + # tightest, row-0 boundary; always <= per-row N_eff for any ofs). + effective_len = N - next_n + 1 + group_logits = logits[::next_n, :effective_len] + argmax_idx = group_logits.argmax(dim=-1).int() + pre_idx = torch.zeros(num_groups, top_k, dtype=torch.int32, device=device) + pre_idx[:, 0] = argmax_idx + + if preidx_hit_rate <= 0.0: + # Worst-case: only slot 0 is meaningful, rest are junk arange. + # Tests kernel robustness to low-hit-rate preIdx. + for j in range(1, top_k): + pre_idx[:, j] = j + else: + # Realistic: mix ``preidx_hit_rate`` real torch.topk indices with + # random in-range fillers. Tests the Guess-phase short-circuit + # path (production V3.2 ~40%, V4 Pro ~75%). + ref_topk = group_logits.topk(top_k, dim=-1).indices.int() + keep_mask = torch.rand(ref_topk.shape, device=device) < preidx_hit_rate + random_fill = torch.randint( + 0, effective_len, ref_topk.shape, device=device, dtype=torch.int32 + ) + guess = torch.where(keep_mask, ref_topk, random_fill) + # Slot 0 must stay as argmax (kernel's strict invariant). + guess[:, 0] = argmax_idx + pre_idx[:, :] = guess + + # seq_lens is uncompressed; kernel divides by cr internally. + # ``seq_lens = N * cr`` makes the kernel's per-row effective scan + # length cover the full N columns of ``logits`` for the typical + # row 0 (ofs=0). Within a group different rows still see slightly + # different N_eff under cr>1 floor division — the reference check + # mirrors the kernel formula exactly, so all (next_n, cr) combos + # are exercisable. + seq_lens_val = N * compress_ratio + seq_lens = torch.full((num_groups,), seq_lens_val, dtype=torch.int32, device=device) + return logits, pre_idx, seq_lens + + +def _tie_aware_check( + out_indices: torch.Tensor, + logits: torch.Tensor, + seq_lens: torch.Tensor, + top_k: int, + next_n: int, + compress_ratio: int = 1, +) -> None: + """Vectorized multi-row tie-aware correctness check with strict sort+allclose. + + Per row r: scan range is ``logits[r, :N_eff(r)]`` where N_eff mirrors + the kernel's exact formula (see ``GvrTopKKernel.gvr_topk_kernel``): + + actual_kv_len = seq_lens[r // next_n] - next_n + (r % next_n) + 1 + N_eff = actual_kv_len // compress_ratio # cr=1 is identity + + Reference ``torch.topk`` is masked to this range so the reference and + kernel scan exactly the same columns under any (next_n, cr) combo + (including next_n>=3 + cr>=2 where the floor-division makes per-row + N_eff vary within a group). + + All checks (out-of-range, duplicates, n_below, sort+allclose) run as + batched GPU ops; only assertion-failure diagnostics fall back to host. + """ + num_rows, top_k_out = out_indices.shape + assert top_k_out == top_k + device = logits.device + logits_f32 = logits.to(torch.float32) + N = logits.shape[1] + + # Per-row N_eff mirroring the kernel formula. seq_lens is per-group + # (length num_rows // next_n); broadcast across the next_n rows of + # each group before computing actual_kv_len // cr. + row_idx = torch.arange(num_rows, device=device) + group_idx = row_idx // next_n + ofs = row_idx % next_n + seq_lens_per_row = seq_lens.to(device=device, dtype=torch.long)[group_idx] + actual_kv_len = seq_lens_per_row - next_n + ofs + 1 + N_eff = actual_kv_len // compress_ratio # [num_rows] + + # Mask logits beyond per-row N_eff to -inf so torch.topk ignores tails. + col_idx = torch.arange(N, device=device) + in_range_mask = col_idx[None, :] < N_eff[:, None] # [num_rows, N] + masked_logits = torch.where(in_range_mask, logits_f32, float("-inf")) + + # Reference per-row top-K, sorted descending. + ref_vals, _ = torch.topk(masked_logits, k=top_k, largest=True, sorted=True, dim=-1) + + # ---- 1. Out-of-range / -1 placeholder check (single fused mask) ---- + out_of_range = (out_indices < 0) | (out_indices >= N_eff[:, None]) + if bool(out_of_range.any().item()): + bad_row = int(out_of_range.any(dim=1).int().argmax().item()) + bad_indices = out_indices[bad_row].cpu().tolist() + raise AssertionError( + f"row={bad_row}: kernel returned out-of-range index " + f"(N_eff={int(N_eff[bad_row].item())}, indices={bad_indices})" + ) + + # ---- 2. Duplicate-index check (sort each row, scan consecutive eq) ---- + sorted_idx, _ = out_indices.sort(dim=-1) + has_dup = (sorted_idx[:, 1:] == sorted_idx[:, :-1]).any(dim=-1) + if bool(has_dup.any().item()): + bad_row = int(has_dup.int().argmax().item()) + raise AssertionError( + f"row={bad_row}: kernel returned duplicate indices: " + f"{out_indices[bad_row].cpu().tolist()}" + ) + + # ---- 3. Gather selected values (safe — already in range) ---- + sel_vals = torch.gather(logits_f32, dim=-1, index=out_indices.long()) + + # ---- 4. n_below check vs per-row K-th value ---- + kth_vals = ref_vals[:, -1:] # [num_rows, 1] + n_below_per_row = (sel_vals < kth_vals).sum(dim=-1) + if bool((n_below_per_row > 0).any().item()): + bad_row = int(n_below_per_row.argmax().item()) + n_below = int(n_below_per_row[bad_row].item()) + kth = float(kth_vals[bad_row, 0].item()) + raise AssertionError( + f"row={bad_row}: {n_below} selected values < Kth-rank value ({kth:.6f})" + ) + + # ---- 5. Strict: sorted-value multiset == torch.topk reference ---- + sel_sorted, _ = sel_vals.sort(dim=-1, descending=True) + diff = (sel_sorted - ref_vals).abs() + if not bool(torch.allclose(sel_sorted, ref_vals, rtol=1e-5, atol=1e-5)): + per_row_max = diff.max(dim=-1).values + bad_row = int(per_row_max.argmax().item()) + max_diff = float(per_row_max[bad_row].item()) + raise AssertionError(f"row={bad_row}: sorted-value mismatch — max diff {max_diff:.4e}") + + +@skip_not_sm100 +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("top_k", [512, 1024, 2048]) +@pytest.mark.parametrize("N", [4096, 65536]) +@pytest.mark.parametrize("next_n", [1, 2]) +@pytest.mark.parametrize("batch_size", [1, 32]) +@pytest.mark.parametrize("compress_ratio", [1, 4]) +@pytest.mark.parametrize("preidx_hit_rate", [0.0, 0.5]) +def test_cute_dsl_gvr_topk_decode( + dtype, top_k, N, next_n, batch_size, compress_ratio, preidx_hit_rate +): + """Compare custom op output against torch.topk reference (tie-aware). + + ``preidx_hit_rate=0.0`` exercises the worst-case (only argmax slot is + a real topK index); ``0.5`` matches realistic production preIdx + overlap with topK (V3.2 ~40%, V4 Pro ~75%) and exercises the + kernel's Guess-phase short-circuit path. + """ + if N - next_n + 1 < top_k: + pytest.skip( + f"N_eff < top_k ({N - next_n + 1} < {top_k}) is a degenerate path not exercised here" + ) + + num_rows = batch_size * next_n + logits, pre_idx, seq_lens = _make_inputs( + num_rows, + N, + top_k, + dtype, + next_n, + seed=42, + compress_ratio=compress_ratio, + preidx_hit_rate=preidx_hit_rate, + ) + + out_indices = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + + torch.ops.trtllm.cute_dsl_gvr_topk_decode( + logits, + pre_idx, + seq_lens, + out_indices, + top_k=top_k, + next_n=next_n, + compress_ratio=compress_ratio, + ) + torch.cuda.synchronize() + + _tie_aware_check(out_indices, logits, seq_lens, top_k, next_n, compress_ratio=compress_ratio) From f57f4aa85bd76bf134e835ab1b2b38dbb24eb71a Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:04:39 +0800 Subject: [PATCH 180/308] [https://nvbugs/6222480][test] fix stress test issue on H100 (#14721) Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- ...ig_ctxtp2_gentp2_gptoss_eagle_triton.yaml} | 4 +- ...fig_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml | 67 +++++++++++++++++++ ...g_config_ctxtp2_gentp2_gptoss_triton.yaml} | 4 +- .../defs/disaggregated/test_disaggregated.py | 34 +++++++--- .../test_lists/qa/llm_function_core.txt | 1 - .../test_lists/qa/llm_function_rtx6k.txt | 1 - .../test_lists/qa/llm_function_stress.txt | 7 +- .../test_lists/test-db/l0_l40s.yml | 1 - tests/integration/test_lists/waives.txt | 6 +- 9 files changed, 101 insertions(+), 24 deletions(-) rename tests/integration/defs/disaggregated/test_configs/{disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml => disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml} (97%) create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml rename tests/integration/defs/disaggregated/test_configs/{disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml => disagg_config_ctxtp2_gentp2_gptoss_triton.yaml} (96%) diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml similarity index 97% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml index d5fa9d45a931..b7e04b714d00 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml @@ -22,7 +22,7 @@ context_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -45,7 +45,7 @@ generation_servers: enable_block_reuse: false free_gpu_memory_fraction: 0.8 moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml new file mode 100644 index 000000000000..1ebbe92ac159 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml @@ -0,0 +1,67 @@ +model: gpt_oss/gpt-oss-120b +hostname: localhost +backend: pytorch +context_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 16640 + max_seq_len: 8232 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: &eagle_spec + decoding_type: Eagle + max_draft_len: 3 + eagle3_one_model: true + speculative_model: gpt_oss/gpt-oss-120b-Eagle3 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: null + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 +generation_servers: + num_instances: 1 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + moe_expert_parallel_size: 4 + enable_attention_dp: true + max_num_tokens: 10240 + max_seq_len: 10240 + max_batch_size: 128 + trust_remote_code: true + enable_chunked_prefill: true + disable_overlap_scheduler: true + speculative_config: *eagle_spec + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + moe_config: + backend: TRTLLM + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + print_iter_log: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml similarity index 96% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml index 47b619faf141..1d6f4d336e01 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml @@ -17,7 +17,7 @@ context_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: null print_iter_log: true cache_transceiver_config: @@ -39,7 +39,7 @@ generation_servers: free_gpu_memory_fraction: 0.8 disable_overlap_scheduler: true moe_config: - backend: CUTLASS + backend: TRITON cuda_graph_config: enable_padding: true batch_sizes: diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index f320690ad787..b8136359f972 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -298,12 +298,14 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm.yaml", "deepseek_r1_v2_fp4_mtp_stress": f"{test_configs_root}/disagg_config_ctxtp4_gentp4_deepseek_r1_v2_fp4_tllm_mtp.yaml", - "gpt_oss_120b_stress": + "gpt_oss_120b_trtllm_stress": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", - "gpt_oss_120b_eagle_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_eagle.yaml", - "gpt_oss_120b_cutlass_stress": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm_cutlass.yaml", + "gpt_oss_120b_eagle_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml", + "gpt_oss_120b_eagle_trtllm_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml", + "gpt_oss_120b_triton_stress": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml", "qwen3_5_4b_fp8_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp1_qwen3_5_4b_fp8_tllm.yaml", "gpt_oss_120b_harmony": @@ -2297,7 +2299,7 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, cancellation_delay=0.5), marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell)), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_stress', + test_desc='gpt_oss_120b_trtllm_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, @@ -2306,22 +2308,34 @@ def test_disaggregated_gpt_oss_120b_harmony(disaggregated_test_root, pytest.param( TestConfig( model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_eagle_stress', + test_desc='gpt_oss_120b_eagle_triton_stress', request_count=60000, accuracy_threshold=0.42, speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', cancellation_rate=10, cancellation_delay=0.5, ), - marks=(pytest.mark.skip_less_device(8), skip_pre_hopper), + marks=(pytest.mark.skip_less_device(8), skip_no_hopper), + ), + pytest.param( + TestConfig( + model_path='gpt_oss/gpt-oss-120b', + test_desc='gpt_oss_120b_eagle_trtllm_stress', + request_count=60000, + accuracy_threshold=0.42, + speculative_model_path='gpt_oss/gpt-oss-120b-Eagle3', + cancellation_rate=10, + cancellation_delay=0.5, + ), + marks=(pytest.mark.skip_less_device(8), skip_pre_blackwell), ), pytest.param(TestConfig(model_path='gpt_oss/gpt-oss-120b', - test_desc='gpt_oss_120b_cutlass_stress', + test_desc='gpt_oss_120b_triton_stress', request_count=60000, accuracy_threshold=0.42, cancellation_rate=10, cancellation_delay=0.5), - marks=(pytest.mark.skip_less_device(4), skip_pre_hopper)), + marks=(pytest.mark.skip_less_device(4), skip_no_hopper)), pytest.param(TestConfig(model_path='Qwen3.5-4B-FP8', test_desc='qwen3_5_4b_fp8_stress', request_count=3000, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index e6a923f5d0e9..31812cdab731 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -661,7 +661,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False] diff --git a/tests/integration/test_lists/qa/llm_function_rtx6k.txt b/tests/integration/test_lists/qa/llm_function_rtx6k.txt index 25e7eabd5da7..43ec270512d8 100644 --- a/tests/integration/test_lists/qa/llm_function_rtx6k.txt +++ b/tests/integration/test_lists/qa/llm_function_rtx6k.txt @@ -123,7 +123,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=True-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] diff --git a/tests/integration/test_lists/qa/llm_function_stress.txt b/tests/integration/test_lists/qa/llm_function_stress.txt index fa8a710359f8..1bf7c4f3f77b 100644 --- a/tests/integration/test_lists/qa/llm_function_stress.txt +++ b/tests/integration/test_lists/qa/llm_function_stress.txt @@ -1,8 +1,9 @@ disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_mtp_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] -disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_trtllm_stress] +disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_5_4b_fp8_stress] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_nvfp4_4gpus diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index c0fb7b7cbfba..19d4f222f4f6 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -60,7 +60,6 @@ l0_l40s: - examples/test_llama.py::test_llm_llama_v2_lora_1gpu[chinese-llama-2-lora-13b-llama-v2-13b-hf-lora_fp16-base_fp16] - examples/test_llama.py::test_llm_llama_v3_dora_1gpu[commonsense-llama-v3-8b-dora-r32-llama-v3-8b-hf-base_fp16] - examples/test_llama.py::test_llm_llama_v1_1gpu_kv_cache_reuse_with_prompt_table[llama-7b] - - examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] - examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] - llmapi/test_llm_examples.py::test_llmapi_quickstart - examples/test_visual_gen.py::test_visual_gen_quickstart diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e0f5e7bf2342..25c932aad85c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -116,7 +116,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa SKIP (ht accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype SKIP (https://nvbugs/5520319) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6162122) -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_eagle3_tp8[eagle3_one_model=False-torch_compile=True] SKIP (https://nvbugs/5775326) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) @@ -203,7 +202,6 @@ examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bf examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (https://nvbugs/4961624) examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) -examples/test_phi.py::test_llm_phi_lora_1gpu[Phi-3-mini-4k-instruct-ru-lora-Phi-3-mini-4k-instruct-lora_fp16-base_fp16] SKIP (https://nvbugs/5612313) examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502) examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-disable_quant-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5174573) @@ -232,8 +230,8 @@ full:GH200/examples/test_multimodal.py::test_llm_multimodal_general[video-neva-p full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:GH200/unittest/trt/model_api/test_model_quantization.py SKIP (https://nvbugs/4979955) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_cutlass_stress] SKIP (https://nvbugs/6222480) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_stress] SKIP (https://nvbugs/6222480) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_triton_stress] SKIP (https://nvbugs/6250439) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_triton_stress] SKIP (https://nvbugs/6222480) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) full:RTX/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5569696) From 06e3a776b25c6ad9c2296ba9287bb8147f3d4336 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:08:40 +0800 Subject: [PATCH 181/308] [None][test] Waive 6 failed cases for main in QA CI (#14787) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 25c932aad85c..5e38c2185d2d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -124,6 +124,7 @@ accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[laten accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype SKIP (https://nvbugs/6076767) +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_off] SKIP (https://nvbugs/6255417) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_on] SKIP (https://nvbugs/6094068) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1-cutlass] SKIP (https://nvbugs/6116088) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_on-trtllm] SKIP (https://nvbugs/6094068) From 125c0da3a2a9b0db44a2e132290772a1151d8a72 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:33:43 +0800 Subject: [PATCH 182/308] [None][test] Waive 1 failed cases for main in QA CI (#14783) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 5e38c2185d2d..eb958ebc3868 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -122,6 +122,7 @@ accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-c accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6163033) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) +accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 SKIP (https://nvbugs/6244727) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp SKIP (https://nvbugs/6211693) accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype SKIP (https://nvbugs/6076767) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tp4ep4_cudagraph_overlap_adp_off] SKIP (https://nvbugs/6255417) From 18724f7f60d63635d223e3343664f1ebd6201f72 Mon Sep 17 00:00:00 2001 From: Dhinesh Ponnarasan <160256912+DhineshPonnarasan@users.noreply.github.com> Date: Tue, 2 Jun 2026 02:54:20 -0400 Subject: [PATCH 183/308] [None][fix] synchronize MLA cache reuse fallback metadata (#14049) Signed-off-by: Dhinesh Ponnarasan --- .../_torch/pyexecutor/py_executor_creator.py | 4 + ...y_executor_creator_mla_cache_reuse_sync.py | 338 ++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 39de33f8f51a..1ff0f5c0bf28 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -679,6 +679,8 @@ def drafting_loop_wrapper(model): f"KV cache reuse for MLA can only be enabled on SM90/SM100/SM103/SM120, " f"disable enable_block_reuse for SM{sm_version}") kv_cache_config.enable_block_reuse = False + _set_model_engines_cache_reuse([model_engine, draft_model_engine], + False) kv_cache_quant_algo = model_engine.model.model_config.quant_config.kv_cache_quant_algo if kv_cache_config.enable_block_reuse and not ( @@ -689,6 +691,8 @@ def drafting_loop_wrapper(model): f"disable enable_block_reuse for KV cache quant algorithm: {kv_cache_quant_algo}" ) kv_cache_config.enable_block_reuse = False + _set_model_engines_cache_reuse([model_engine, draft_model_engine], + False) if enable_chunked_context and sm_version not in [90, 100, 103, 120]: logger.warning( "Chunked Prefill for MLA can only be enabled on SM90/SM100/SM103/SM120, " diff --git a/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py b/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py new file mode 100644 index 000000000000..5c8e3d62efd5 --- /dev/null +++ b/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace + +from tensorrt_llm._torch.pyexecutor import py_executor_creator +from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManagerType +from tensorrt_llm.quantization import QuantAlgo + + +class _DummyCalibrator: + """Mock calibrator for testing that bypasses actual calibration logic.""" + + def init(self, *args, **kwargs): + """Initialize calibrator (no-op).""" + return None + + def maybe_wrap_model(self, model): + """Wrap model (no-op, returns model unchanged).""" + return model + + +class _DummyResourceManager: + """Mock resource manager that stores and retrieves resource instances by type.""" + + def __init__(self, resources): + """Initialize with resource dictionary. + + Args: + resources: Dict mapping ResourceManagerType to resource instances. + """ + self._resources = resources + + def get_resource_manager(self, resource_type): + """Retrieve resource by type. + + Args: + resource_type: ResourceManagerType enum value. + + Returns: + Resource instance or None if not found. + """ + return self._resources.get(resource_type) + + +class _DummyPyExecutor: + """Mock PyExecutor that stores resource manager and model engine references.""" + + def __init__(self, resources, model_engine, peft_cache_config, execution_stream): + """Initialize mock executor. + + Args: + resources: Dict of resource managers. + model_engine: Model engine instance. + peft_cache_config: PEFT cache configuration. + execution_stream: CUDA stream for execution. + """ + self.resource_manager = _DummyResourceManager(resources) + self.model_engine = model_engine + self.peft_cache_config = peft_cache_config + self.execution_stream = execution_stream + self.started = False + + def start_worker(self): + """Mark executor as started.""" + self.started = True + + +class _DummyKvCacheCreator: + """Mock KV cache creator that builds a dummy KV cache manager with reuse settings.""" + + def __init__(self, **kwargs): + """Initialize with KV cache configuration. + + Args: + **kwargs: Keyword args including max_seq_len, kv_cache_config, execution_stream. + """ + self._max_seq_len = kwargs["max_seq_len"] + self._kv_cache_config = kwargs["kv_cache_config"] + self._execution_stream = kwargs["execution_stream"] + + def try_prepare_estimation(self): + """Skip estimation phase (no-op).""" + return False + + def build_managers(self, resources, estimating_kv_cache): + """Build KV cache manager with reuse configuration. + + Args: + resources: Dict to store created managers. + estimating_kv_cache: Whether in estimation mode (unused). + """ + del estimating_kv_cache + resources[ResourceManagerType.KV_CACHE_MANAGER] = SimpleNamespace( + enable_block_reuse=self._kv_cache_config.enable_block_reuse, + _stream=self._execution_stream, + ) + + +class _DummyModelEngine: + """Mock model engine that exposes attention runtime features and model configuration.""" + + def __init__(self, *, attn_runtime_features, kv_cache_quant_algo): + """Initialize with runtime features and quantization algorithm. + + Args: + attn_runtime_features: AttentionRuntimeFeatures instance. + kv_cache_quant_algo: Quantization algorithm for KV cache. + """ + self.attn_runtime_features = attn_runtime_features + self.max_seq_len = 128 + self.max_num_tokens = 128 + self.sparse_attention_config = None + self.attn_metadata = None + self.model = SimpleNamespace( + model_config=SimpleNamespace( + enable_flash_mla=False, + is_generation=True, + pretrained_config=SimpleNamespace(), + quant_config=SimpleNamespace(kv_cache_quant_algo=kv_cache_quant_algo), + ), + vocab_size_padded=32000, + ) + + +def _make_llm_args(): + """Create minimal LLM arguments for executor initialization. + + Returns: + SimpleNamespace with all required LLM configuration fields. + """ + kv_cache_config = SimpleNamespace( + enable_block_reuse=True, + enable_partial_reuse=False, + tokens_per_block=32, + max_attention_window=None, + mamba_state_cache_interval=1, + ) + scheduler_config = SimpleNamespace( + context_chunking_policy=None, + capacity_scheduler_policy=None, + ) + + return SimpleNamespace( + garbage_collection_gen0_threshold=0, + lora_config=None, + kv_connector_config=None, + scheduler_config=scheduler_config, + peft_cache_config=SimpleNamespace(), + kv_cache_config=kv_cache_config, + decoding_config=None, + guided_decoding_backend=None, + custom_tokenizer=None, + trust_remote_code=False, + mm_encoder_only=False, + enable_chunked_prefill=False, + attn_backend="TRTLLM", + speculative_config=None, + disable_overlap_scheduler=True, + sleep_config=None, + cache_transceiver_config=None, + dwdp_config=None, + layer_wise_benchmarks_config=SimpleNamespace( + calibration_mode=None, + calibration_file_path=None, + calibration_layer_indices=None, + ), + sampler_type=None, + disable_flashinfer_sampling=False, + cuda_graph_config=None, + parallel_config=SimpleNamespace(to_mapping=lambda: SimpleNamespace()), + get_runtime_sizes=lambda: (1, 128, 128, 4), + ) + + +def _run_create_py_executor(monkeypatch, *, sm_version, kv_cache_quant_algo): + """Execute create_py_executor with mocked dependencies and return cache reuse flags. + + Mocks all external dependencies (model engine, resource managers, etc.) to isolate + executor creation logic and verify that KV cache reuse configuration is synchronized + with attention runtime metadata across fallback paths. + + Args: + monkeypatch: pytest fixture for mocking. + sm_version: CUDA SM version to simulate (e.g., 89, 90). + kv_cache_quant_algo: Quantization algorithm to use (e.g., NO_QUANT, INT8). + + Returns: + Tuple of (kv_cache_reuse_flag, runtime_cache_reuse_flag) from created executor. + """ + llm_args = _make_llm_args() + fake_mapping = SimpleNamespace( + rank=0, + tp_size=1, + enable_attention_dp=False, + is_last_pp_rank=lambda: True, + ) + + monkeypatch.setattr( + py_executor_creator, + "_load_config_and_create_checkpoint_loader", + lambda llm_args, checkpoint_dir: (llm_args, None), + ) + monkeypatch.setattr(py_executor_creator, "_get_mapping", lambda _: fake_mapping) + monkeypatch.setattr( + py_executor_creator.Distributed, "get", staticmethod(lambda mapping: SimpleNamespace()) + ) + monkeypatch.setattr( + py_executor_creator, "validate_feature_combination", lambda *args, **kwargs: None + ) + monkeypatch.setattr(py_executor_creator, "get_calibrator", lambda: _DummyCalibrator()) + monkeypatch.setattr( + py_executor_creator, "instantiate_sampler", lambda *args, **kwargs: SimpleNamespace() + ) + monkeypatch.setattr( + py_executor_creator, "get_spec_resource_manager", lambda *args, **kwargs: None + ) + monkeypatch.setattr(py_executor_creator, "get_spec_drafter", lambda *args, **kwargs: None) + monkeypatch.setattr(py_executor_creator, "_adjust_torch_mem_fraction", lambda: None) + monkeypatch.setattr(py_executor_creator, "log_memory_usage", lambda *args, **kwargs: None) + monkeypatch.setattr(py_executor_creator, "is_mla", lambda _: True) + monkeypatch.setattr(py_executor_creator, "is_hybrid_linear", lambda _: False) + monkeypatch.setattr(py_executor_creator, "get_sm_version", lambda: sm_version) + monkeypatch.setattr(py_executor_creator, "KvCacheCreator", _DummyKvCacheCreator) + + monkeypatch.setattr(py_executor_creator.torch.cuda, "mem_get_info", lambda: (2 << 30, 4 << 30)) + monkeypatch.setattr(py_executor_creator.torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr(py_executor_creator.torch.cuda, "reset_peak_memory_stats", lambda: None) + monkeypatch.setattr( + py_executor_creator.torch.cuda, "memory_stats", lambda: {"allocated_bytes.all.current": 0} + ) + monkeypatch.setattr( + py_executor_creator.torch.cuda, "Stream", lambda: SimpleNamespace(cuda_stream=123) + ) + + def _create_model_engine(**kwargs): + return _DummyModelEngine( + attn_runtime_features=kwargs["attn_runtime_features"], + kv_cache_quant_algo=kv_cache_quant_algo, + ) + + monkeypatch.setattr(py_executor_creator, "PyTorchModelEngine", _create_model_engine) + + def _create_py_executor_instance(**kwargs): + return _DummyPyExecutor( + resources=kwargs["resources"], + model_engine=kwargs["model_engine"], + peft_cache_config=kwargs["peft_cache_config"], + execution_stream=kwargs["execution_stream"], + ) + + monkeypatch.setattr( + py_executor_creator, "create_py_executor_instance", _create_py_executor_instance + ) + + py_executor = py_executor_creator.create_py_executor( + llm_args=llm_args, + checkpoint_dir=None, + ) + kv_cache_manager = py_executor.resource_manager.get_resource_manager( + ResourceManagerType.KV_CACHE_MANAGER + ) + + return ( + kv_cache_manager.enable_block_reuse, + py_executor.model_engine.attn_runtime_features.cache_reuse, + ) + + +def test_mla_unsupported_sm_fallback_syncs_cache_reuse(monkeypatch): + """Verify MLA unsupported SM fallback disables cache reuse in both config and runtime. + + When MLA is enabled but SM version (89) is unsupported, the fallback should: + - Set kv_cache_config.enable_block_reuse to False + - Also set model_engine.attn_runtime_features.cache_reuse to False + + This test ensures invariant synchronization is maintained across the fallback. + """ + kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + monkeypatch, + sm_version=89, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + ) + + assert kv_cache_reuse is False + assert runtime_cache_reuse is False + + +def test_mla_unsupported_kv_quant_fallback_syncs_cache_reuse(monkeypatch): + """Verify MLA unsupported KV quant fallback disables cache reuse in both config and runtime. + + When MLA is enabled, SM version (90) is supported, but KV quantization (INT8) is unsupported, + the fallback should: + - Set kv_cache_config.enable_block_reuse to False + - Also set model_engine.attn_runtime_features.cache_reuse to False + + This test ensures invariant synchronization is maintained across the fallback. + """ + kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + monkeypatch, + sm_version=90, + kv_cache_quant_algo=QuantAlgo.INT8, + ) + + assert kv_cache_reuse is False + assert runtime_cache_reuse is False + + +def test_mla_supported_configuration_preserves_cache_reuse(monkeypatch): + """Verify MLA supported configuration preserves cache reuse in both config and runtime. + + When both SM version (90) and KV quantization (NO_QUANT) are supported for MLA, + no fallback occurs and: + - kv_cache_config.enable_block_reuse remains True + - model_engine.attn_runtime_features.cache_reuse remains True + + This positive test ensures the default path does not regress. + """ + kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + monkeypatch, + sm_version=90, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + ) + + assert kv_cache_reuse is True + assert runtime_cache_reuse is True From 460adc715d7c383cef195821e61b80bfb5a6385b Mon Sep 17 00:00:00 2001 From: Yao Yao Date: Tue, 2 Jun 2026 15:30:57 +0800 Subject: [PATCH 184/308] [None][feat] Add KV cache prefetch (#14748) Signed-off-by: Yao Yao --- .../runtime/kv_cache_manager_v2/__init__.pyi | 1 + .../kv_cache_manager_v2/_core/_kv_cache.py | 61 ++++++++++++++++++- .../kv_cache_manager_v2/_storage_manager.py | 39 ++++++++++++ .../test_kv_cache_manager_v2.py | 29 +++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi index ff36dc8f0b45..451778f7e754 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi @@ -226,6 +226,7 @@ class _KVCache: def stop_committing(self) -> None: ... def suspend(self) -> None: ... def resume(self, cuda_stream: CudaStream | None = None) -> bool: ... + def prefetch(self, target: CacheLevel) -> bool: ... def get_scratch_desc(self, layer_group_id: LayerGroupId) -> ScratchDesc | None: ... @property def has_scratch_slots(self) -> bool: ... diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py index 422f5d2794d6..ee8a7e24da0b 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py @@ -54,6 +54,7 @@ BatchedLockTarget, BlockPage, CommittedPage, + Page, ScratchSlotLock, UncommittedPage, _PageHolder, @@ -932,6 +933,49 @@ def resume(self, cuda_stream: CudaStream | None = None) -> bool: self._status = self.Status.ACTIVE return True + def prefetch(self, target: CacheLevel) -> bool: + """Best-effort prefetch active pages to the target cache level. + + The cache must be suspended. Prefetch is only a performance hint: a False + return value means the requested pages could not be recalled due to cache + pressure, but the cache remains functionally valid. + + Args: + target: Destination cache level for active pages in lower tiers. + + Returns: + True if the prefetch was dispatched, False if storage could not reserve enough pages. + """ + assert self.status == self.Status.SUSPENDED + manager = self.manager + storage = manager._storage + num_tiers = storage.num_cache_levels + assert CacheLevel(0) <= target < num_tiers + + num_pool_groups = storage.num_pool_groups + lc2pg = storage.get_pool_group_index + + all_pages = make_typed( + lambda _: make_typed(lambda _: list[Page](), num_tiers), num_pool_groups + ) + + for ordinal, beam_idx, lc_idx in self._active_pages(): + holder = self._page(ordinal, beam_idx, lc_idx) + if holder is None: + continue + page = expect_type(_PageHolder, holder).page + lvl = page.cache_level + if lvl < target: + continue + pg_idx = lc2pg(lc_idx) + all_pages[pg_idx][lvl].append(page) + + try: + storage.prefetch(target, all_pages) + except OutOfPagesError: + return False + return True + def _active_pages(self) -> Iterator[tuple[BlockOrdinal, BeamIndex, LifeCycleId]]: """Yields (ordinal, beam_idx, lc_idx) for all active pages. @@ -975,12 +1019,25 @@ def tokens_per_block(self) -> int: def _page( self, block_ordinal: BlockOrdinal, beam_index: BeamIndex, life_cycle: LifeCycleId ) -> BlockPage: - return self._blocks[block_ordinal].pages[beam_index][life_cycle] + """Return the page holder for an attention block or the SSM block.""" + is_ssm = block_ordinal == BAD_BLOCK_ORDINAL + assert (life_cycle == self.manager._life_cycles.ssm_life_cycle_id) == is_ssm + return ( + self._ssm_blocks[beam_index][life_cycle] + if is_ssm + else self._blocks[block_ordinal].pages[beam_index][life_cycle] + ) def _block( self, block_ordinal: BlockOrdinal, beam_index: BeamIndex ) -> TypedIndexList[LifeCycleId, BlockPage]: - return self._blocks[block_ordinal].pages[beam_index] + """Return the life-cycle page list for an attention block or the SSM block.""" + is_ssm = block_ordinal == BAD_BLOCK_ORDINAL + return ( + self._ssm_blocks[beam_index] + if is_ssm + else self._blocks[block_ordinal].pages[beam_index] + ) def _snapshot_ssm_to_tree_block( self, tree_block: Block, ssm_lc_id: LifeCycleId, beam_idx: BeamIndex diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py index 0d90c0e92aa3..90f822ba5539 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py @@ -898,3 +898,42 @@ def constrain_ratio( total = sum(num_bytes) assert total > 0 return typed_map(num_bytes, lambda x: x / total) + + def prefetch( + self, + dst_lvl: CacheLevel, + pages: TypedIndexList[PoolGroupIndex, TypedIndexList[CacheLevel, list[Page]]], + ) -> None: + """Dispatch page migration to the destination cache level. + + Args: + dst_lvl: Destination cache level for pages currently in lower tiers. + pages: Pages grouped by pool group and current cache level. + + Raises: + OutOfPagesError: If there are not enough pages available for the prefetch hint. + """ + num_slots = filled_list(0, self.num_pool_groups) + scheduled = list[Page]() + try: + for pg_idx, pg_pages in typed_enumerate(pages): + for lvl, lvl_pages in typed_enumerate(pg_pages): + assert lvl >= dst_lvl or not lvl_pages + for p in lvl_pages: + if p.scheduled_for_eviction: + self.exclude_from_eviction(p) + scheduled.append(p) + elif self.is_evictable(p, dst_lvl): + scheduled.append(p) + assert lvl >= dst_lvl + if lvl == dst_lvl: + continue + num_slots[pg_idx] += 1 + self.prepare_free_slots(dst_lvl, num_slots) + for pg_idx, pg_tasks in typed_enumerate(pages): + for lvl in typed_range(CacheLevel(dst_lvl + 1), self.num_cache_levels): + lvl_tasks = pg_tasks[lvl] + self._batched_migrate(pg_idx, dst_lvl, lvl, lvl_tasks, True) + finally: + for p in scheduled: + self.schedule_for_eviction(p) diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py index 65ccda597d36..649c7d54536c 100755 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py @@ -1275,6 +1275,22 @@ def test_resize_quota(self) -> None: stream_holder = CachedCudaStream() stream = cast(CudaStream, stream_holder.handle) + def count_active_pages_by_level(kv_cache: _KVCache) -> list[int]: + counts = [0] * self.manager._storage.num_cache_levels + for ordinal, beam_idx, lc_idx in kv_cache._active_pages(): + block_page = kv_cache._page(ordinal, beam_idx, lc_idx) + assert block_page is not None + counts[block_page.page.cache_level] += 1 + return counts + + def assert_prefetched_pages_are_evictable(kv_cache: _KVCache) -> None: + for ordinal, beam_idx, lc_idx in kv_cache._active_pages(): + block_page = kv_cache._page(ordinal, beam_idx, lc_idx) + assert block_page is not None + page = block_page.page + if page.cache_level == HOST_LEVEL and self.manager._storage.is_evictable(page): + self.assertTrue(page.scheduled_for_eviction) + # First commit some blocks to fill all levels of cache. This helps test the case where shrinking # the quota will drop some pages from the last-level cache. for _ in range(11): @@ -1329,6 +1345,19 @@ def test_resize_quota(self) -> None: assert success success = self.manager.resize(HOST_LEVEL, 128 << 20) assert success + prefetch_target = kv_cache_lst[1] + prefetch_counts_before = count_active_pages_by_level(prefetch_target) + self.assertGreater(prefetch_counts_before[DISK_LEVEL], 0) + success = prefetch_target.prefetch(HOST_LEVEL) + self.assertEqual(success, True) + prefetch_counts_after = count_active_pages_by_level(prefetch_target) + self.assertEqual(prefetch_counts_after[GPU_LEVEL], prefetch_counts_before[GPU_LEVEL]) + self.assertEqual(prefetch_counts_after[DISK_LEVEL], 0) + self.assertEqual( + prefetch_counts_after[HOST_LEVEL], + prefetch_counts_before[HOST_LEVEL] + prefetch_counts_before[DISK_LEVEL], + ) + assert_prefetched_pages_are_evictable(prefetch_target) # Now both requests can resume for kv_cache in kv_cache_lst: success = kv_cache.resume(stream) From 209f3717b099d6af3015312bfc16d94fbee244f4 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:55:56 +0800 Subject: [PATCH 185/308] [https://nvbugs/6191524][fix] In MLA.forward_context, also call the warmup when has_cached_kv_for_mla_context (#14536) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Signed-off-by: Zhen Huang <145532724+zhhuang-nv@users.noreply.github.com> Co-authored-by: Zhen Huang <145532724+zhhuang-nv@users.noreply.github.com> --- tensorrt_llm/_torch/attention_backend/trtllm.py | 7 +++---- tensorrt_llm/_torch/modules/attention.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index 53dcfbc5af2a..09ba91d52c4e 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -1743,14 +1743,13 @@ def is_chunked_prefill_for_mla_context( and metadata.num_ctx_cached_tokens > 0 and metadata.runtime_features.chunked_prefill) - def is_chunked_prefill_mla_context_for_warmup( + def has_cached_kv_for_mla_context_warmup( self, metadata: TrtllmAttentionMetadata, ) -> bool: - """Chunked prefill MLA context check for warmup; does not check num_ctx_cached_tokens.""" + """KV cache reuse / chunked prefill MLA context check for warmup, do not check num_ctx_cached_tokens.""" return (self.is_mla_enable and metadata.kv_cache_manager is not None - and metadata.enable_context_mla_with_cached_kv - and metadata.runtime_features.chunked_prefill) + and metadata.enable_context_mla_with_cached_kv) def load_paged_kv_cache_for_mla( self, diff --git a/tensorrt_llm/_torch/modules/attention.py b/tensorrt_llm/_torch/modules/attention.py index 616724550497..4d2cd971b4af 100644 --- a/tensorrt_llm/_torch/modules/attention.py +++ b/tensorrt_llm/_torch/modules/attention.py @@ -2359,9 +2359,11 @@ def forward_context_with_chunked_prefill( @staticmethod @functools.cache - def cached_warmup_forward_context_with_chunked_prefill( - num_heads_tp, qk_nope_head_dim, qk_rope_head_dim, kv_lora_rank, - v_head_dim, dtype, device): + def cached_warmup_forward_context_with_cached_kv(num_heads_tp, + qk_nope_head_dim, + qk_rope_head_dim, + kv_lora_rank, v_head_dim, + dtype, device): """Warmup torch.compile for cat operations with different tensor layouts. Tensors are marked with torch._dynamo.maybe_mark_dynamic(..., 0) on the @@ -2430,9 +2432,13 @@ def forward_context( if isinstance(self.mha, TrtllmAttention): assert isinstance(attn_metadata, TrtllmAttentionMetadata) trtllm_attention = cast(TrtllmAttention, self.mha) - if trtllm_attention.is_chunked_prefill_mla_context_for_warmup( + # Warm up maybe_compiled_cat for both the chunked-prefill path and + # the cached-kv path; without this, the cached-kv prefill + # (block-reuse without chunked_prefill) recompiles per shape and + # may stall inside inductor's compile worker. + if trtllm_attention.has_cached_kv_for_mla_context_warmup( attn_metadata): - self.cached_warmup_forward_context_with_chunked_prefill( + self.cached_warmup_forward_context_with_cached_kv( self.num_heads_tp, self.qk_nope_head_dim, self.qk_rope_head_dim, self.kv_lora_rank, self.v_head_dim, q.dtype, q.device) From a2996aed191ed38f58f9ad684087f96f0a7df220 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:39:36 +0800 Subject: [PATCH 186/308] [None][test] Waive 2 failed cases for main in QA CI (#14839) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Co-authored-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index eb958ebc3868..52a3740e0962 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -104,6 +104,7 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_sampler SKIP (https://nvbugs/6112497) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[llguidance] SKIP (https://nvbugs/6076767) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] SKIP (https://nvbugs/6256531) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] SKIP (https://nvbugs/6141653) From efb71c7447f86f0401a1efd0f94ddcf8897ea08d Mon Sep 17 00:00:00 2001 From: Yao Yao Date: Tue, 2 Jun 2026 16:56:16 +0800 Subject: [PATCH 187/308] [None][fix] Cherry-pick kv_cache_manager_v2 fixes to main (#14725) Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> Signed-off-by: Yao Yao Signed-off-by: Iman Tabrizian <10105175+tabrizian@users.noreply.github.com> Co-authored-by: Liao Lanyu <108499334+lancelly@users.noreply.github.com> Co-authored-by: Iman Tabrizian <10105175+Tabrizian@users.noreply.github.com> --- .../kv_cache_manager_v2/_core/_kv_cache.py | 17 +++++++++++++-- .../test_kv_cache_manager_v2.py | 21 +++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py index ee8a7e24da0b..a7fec6eea3e6 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py @@ -70,6 +70,7 @@ div_up, expect_type, filled_list, + intersect, make_typed, map_optional, stream_wait_events, @@ -551,8 +552,13 @@ def resize(self, capacity: int | None, history_length: int | None = None) -> boo continue stale_beg, stale_end = stale_ranges[lc] if enable_scratch: - num_scratch_blocks = len(scratch_ranges[lc]) - num_new_normal_blocks = (new_num_blocks - old_num_blocks) - num_scratch_blocks + # Only newly added blocks consume slots below; scratch range may + # extend before old_num_blocks when history_length < old_capacity. + new_block_range = HalfOpenRange(old_num_blocks, new_num_blocks) + num_new_blocks_using_scratch = len( + intersect(scratch_ranges[lc], new_block_range) + ) + num_new_normal_blocks = len(new_block_range) - num_new_blocks_using_scratch num_new_slots[lc] = num_new_normal_blocks * beam_width else: if old_num_blocks < stale_beg: @@ -1182,6 +1188,13 @@ def _commit_block(self, ordinal: BlockOrdinal, is_last: bool) -> None: # We can't commit and can't reuse existing block. Just stop committing. self._commit_state = self.CommitState.VIRTUAL_STOP + if seq_block.is_committed: + for lc_idx, lc in self.manager._life_cycles.attention_life_cycles(): + stale_range = _KVCache._get_stale_range(tokens_per_block, self.history_length, lc) + if ordinal in stale_range: + for beam_block in seq_block.pages: + beam_block[lc_idx] = None + if is_last or self._commit_state == self.CommitState.VIRTUAL_STOP: self._commit_state = self.CommitState.USER_STOP self._on_stop_committing() diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py index 649c7d54536c..65deac29381c 100755 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py @@ -343,7 +343,9 @@ def new_request( req_id, self.manager.create_kv_cache(reuse_scope, prompt), prompt, decode_len ) - def run_request(self, req: Request, interval: int, refcheck: bool) -> float: + def run_request( + self, req: Request, interval: int, refcheck: bool, delay_commit: bool = False + ) -> float: req_id, kv_cache, prompt, decode_len = req assert kv_cache.status == _KVCache.Status.ACTIVE stream = kv_cache.cuda_stream @@ -366,7 +368,8 @@ def run_request(self, req: Request, interval: int, refcheck: bool) -> float: for _ in range(decode_len): required_capacity = len(history) + 1 if required_capacity > capacity: - kv_cache.commit(history[kv_cache.history_length :]) + if not delay_commit: + kv_cache.commit(history[kv_cache.history_length :]) # workaround a mypyc bug: exception in property setter is not propagated # kv_cache.capacity = round_up(required_capacity, interval) if not kv_cache.resize(round_up(required_capacity, interval)): @@ -391,6 +394,7 @@ def run_naive( interval: int = 1, refcheck: bool = True, use_external_page_index_buf: bool = False, + delay_commit: bool = False, ) -> float: prompt_len = 1 decode_len = seq_len - prompt_len @@ -413,7 +417,7 @@ def run_naive( kv_cache = req0.kv_cache success = kv_cache.resume(stream) assert success - time_taken = self.run_request(req0, interval, refcheck) + time_taken = self.run_request(req0, interval, refcheck, delay_commit) s.take_finish_event().synchronize() kv_cache.close() @@ -576,9 +580,14 @@ def num_reused(reuse_scope: ReuseScope | None) -> int: self.assertGreater(num_reused(None), 0) self.assertGreater(num_reused(default_scope), 0) - @parameterized.expand(list(itertools.product([False, True], repeat=2))) + @parameterized.expand(list(itertools.product([False, True], repeat=3))) # @assert_no_ref_cycle - def test_naive(self, use_external_page_index_buf: bool, use_block_quant: bool) -> None: + def test_naive( + self, + use_external_page_index_buf: bool, + use_block_quant: bool, + delay_commit: bool, + ) -> None: self.prepare( 256 << 20, 256 << 20, @@ -588,7 +597,7 @@ def test_naive(self, use_external_page_index_buf: bool, use_block_quant: bool) - 48, block_quant_buf_size=(1024 if use_block_quant else None), ) - self.run_naive(512, 1, True, use_external_page_index_buf) + self.run_naive(512, 1, True, use_external_page_index_buf, delay_commit=delay_commit) @parameterized.expand([(2**i, False) for i in range(12)]) # @parameterized.expand([(32, True)]) From 4cc2d8a79693874884483edd9b7631470240e702 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:27:12 +0800 Subject: [PATCH 188/308] [None][test] Waive 11 failed cases for main in post-merge (#14854) Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 52a3740e0962..5836d3b48335 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -2,6 +2,7 @@ accuracy/test_cli_flow.py::TestGptNext::test_auto_dtype SKIP (https://nvbugs/616 accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6189918) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6189918) +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6075533) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python SKIP (https://nvbugs/6184575) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram SKIP (https://nvbugs/6245651) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) @@ -43,13 +44,16 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mt accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6050489) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6050489) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6211191) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6198785) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6071081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[tp4-cuda_graph=False] SKIP (https://nvbugs/6224636) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6185146) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6195110) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162115) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6211191) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6112497) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6162122) @@ -60,8 +64,10 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True] SKIP (https://nvbugs/5955773) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5945081) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6211191) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6162115) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6245394) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_2_model_mtp[2model_trtllm] SKIP (https://nvbugs/5981293) accuracy/test_llm_api_pytorch.py::TestGLM4_5Air::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/5981293) @@ -91,10 +97,13 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=False-sampler_async_worker=True] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=True-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6224637) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/5616182) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=TRTLLM] SKIP (https://nvbugs/5997547) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dflash SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6050489) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6211191) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=True] SKIP (https://nvbugs/6245389) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) @@ -117,6 +126,7 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa SKIP (ht accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype SKIP (https://nvbugs/5520319) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6162122) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6162122) +accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/5821415) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) @@ -176,6 +186,7 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) +disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6245317) disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6184906) disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp1-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6223556) disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6223556) From 33e0ee3aafa4281f0e76cb7b9c6f085eba20ae2d Mon Sep 17 00:00:00 2001 From: Guoming Zhang <137257613+nv-guomingz@users.noreply.github.com> Date: Tue, 2 Jun 2026 18:28:13 +0800 Subject: [PATCH 189/308] [None][feat] Enable flashifner gdn decoding kernel for qwen3.5 (#13645) Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- tensorrt_llm/_torch/modules/fla/chunk.py | 7 +- .../_torch/modules/fla/chunk_delta_h.py | 40 ++-- .../_torch/modules/fla/flashinfer_chunk.py | 41 ++-- .../_torch/modules/fla/fused_recurrent.py | 38 ++-- .../fla/fused_sigmoid_gating_recurrent.py | 156 ++++++++++++- .../_torch/modules/fla/fused_state_io.py | 212 +++++++----------- .../modules/mamba/test_fused_state_io.py | 150 ++++--------- 7 files changed, 332 insertions(+), 312 deletions(-) diff --git a/tensorrt_llm/_torch/modules/fla/chunk.py b/tensorrt_llm/_torch/modules/fla/chunk.py index ced64e46b0f7..6074f5ab9ed7 100644 --- a/tensorrt_llm/_torch/modules/fla/chunk.py +++ b/tensorrt_llm/_torch/modules/fla/chunk.py @@ -148,7 +148,8 @@ def chunk_gated_delta_rule( Scale factor for the RetNet attention scores. If not provided, it will default to `1 / sqrt(K)`. Default: `None`. initial_state (Optional[torch.Tensor]): - Initial state of shape `[N, H, K, V]` for `N` input sequences. + Initial state of shape `[N, H, V, K]` (K innermost, matching the + state pool) for `N` input sequences. For equal-length input sequences, `N` equals the batch size `B`. Default: `None`. initial_state_indices (Optional[torch.Tensor]): @@ -159,7 +160,7 @@ def chunk_gated_delta_rule( `initial_state` in-place. Callers are responsible for ensuring the selected slots are safe to update without aliasing races. output_final_state (Optional[bool]): - Whether to output the final state of shape `[N, H, K, V]`. Default: `False`. + Whether to output the final state of shape `[N, H, V, K]`. Default: `False`. cu_seqlens (torch.LongTensor): Cumulative sequence lengths of shape `[N+1]` used for variable-length training, consistent with the FlashAttention API. @@ -171,7 +172,7 @@ def chunk_gated_delta_rule( o (torch.Tensor): Outputs of shape `[B, T, H, V]` if `head_first=False` else `[B, H, T, V]`. final_state (torch.Tensor): - Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`. + Final state of shape `[N, H, V, K]` if `output_final_state=True` else `None`. Examples:: >>> import torch diff --git a/tensorrt_llm/_torch/modules/fla/chunk_delta_h.py b/tensorrt_llm/_torch/modules/fla/chunk_delta_h.py index b949cb7d927f..ba974106ced6 100644 --- a/tensorrt_llm/_torch/modules/fla/chunk_delta_h.py +++ b/tensorrt_llm/_torch/modules/fla/chunk_delta_h.py @@ -104,21 +104,23 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( ht = ht + i_h * K * V # load initial state + # Pool layout [slots, HV, V, K], K innermost: logical block (K, V) but K has + # stride 1, V has stride K, order=(0, 1) so Triton treats K as innermost. if USE_INITIAL_STATE: - p_h0_1 = tl.make_block_ptr(h0, (K, V), (V, 1), (0, i_v * BV), (64, BV), - (1, 0)) + p_h0_1 = tl.make_block_ptr(h0, (K, V), (1, K), (0, i_v * BV), (64, BV), + (0, 1)) b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) if K > 64: - p_h0_2 = tl.make_block_ptr(h0, (K, V), (V, 1), (64, i_v * BV), - (64, BV), (1, 0)) + p_h0_2 = tl.make_block_ptr(h0, (K, V), (1, K), (64, i_v * BV), + (64, BV), (0, 1)) b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) if K > 128: - p_h0_3 = tl.make_block_ptr(h0, (K, V), (V, 1), (128, i_v * BV), - (64, BV), (1, 0)) + p_h0_3 = tl.make_block_ptr(h0, (K, V), (1, K), (128, i_v * BV), + (64, BV), (0, 1)) b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) if K > 192: - p_h0_4 = tl.make_block_ptr(h0, (K, V), (V, 1), (192, i_v * BV), - (64, BV), (1, 0)) + p_h0_4 = tl.make_block_ptr(h0, (K, V), (1, K), (192, i_v * BV), + (64, BV), (0, 1)) b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) # main recurrence @@ -215,26 +217,26 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( b_k = tl.load(p_k, boundary_check=(0, 1)) b_h4 += tl.dot(b_k, b_v_new) - # epilogue + # epilogue — write final state back to pool-layout [slots, HV, V, K]. if STORE_FINAL_STATE or USE_INDEXED_STATE: - p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (0, i_v * BV), (64, BV), - (1, 0)) + p_ht = tl.make_block_ptr(ht, (K, V), (1, K), (0, i_v * BV), (64, BV), + (0, 1)) tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) if K > 64: - p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (64, i_v * BV), - (64, BV), (1, 0)) + p_ht = tl.make_block_ptr(ht, (K, V), (1, K), (64, i_v * BV), + (64, BV), (0, 1)) tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) if K > 128: - p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (128, i_v * BV), - (64, BV), (1, 0)) + p_ht = tl.make_block_ptr(ht, (K, V), (1, K), (128, i_v * BV), + (64, BV), (0, 1)) tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) if K > 192: - p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (192, i_v * BV), - (64, BV), (1, 0)) + p_ht = tl.make_block_ptr(ht, (K, V), (1, K), (192, i_v * BV), + (64, BV), (0, 1)) tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) @@ -277,7 +279,9 @@ def chunk_gated_delta_rule_fwd_h( "Indexed chunk state updates require inplace_indexed_state_update=True." ) store_final_state_in_kernel = output_final_state and not use_indexed_state - final_state = (k.new_empty(N, H, K, V, dtype=torch.float32) + # Kernel writes final state in [V, K] layout (K innermost) to match the + # pool layout. Allocate accordingly so the tensor's shape reflects memory. + final_state = (k.new_empty(N, H, V, K, dtype=torch.float32) if store_final_state_in_kernel else None) v_new = torch.empty_like(u) if save_new_value else None diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py index 0cef96bd3ad9..fbe0eb8f22ca 100644 --- a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -17,14 +17,9 @@ ``use_qk_l2norm_in_kernel`` parameter on ``flashinfer.chunk_gated_delta_rule`` is currently a dead arg, see ``flashinfer/gdn_prefill.py:317-356``). * Pre-gather and post-scatter of indexed SSM state (FlashInfer requires - packed ``[num_seqs, H, D, D]`` fp32 initial/output state). - * State layout: TRT-LLM's Triton uses ``[N, H, K, V]`` ordering of the last - two dims; FlashInfer's prefill kernel uses ``[N, H, V, K]`` (per docstring - in ``flashinfer/gdn_prefill.py``: "The final state layout is ``[N, H, V, K]``"; - confirmed by ``flashinfer/tests/gdn/test_prefill_delta_rule.py`` which - transposes the last two dims to compare with the reference implementation). - The wrapper transposes initial_state on the way in and output_state on the - way out so callers see TRT-LLM's ``[N, H, K, V]`` everywhere. + packed ``[num_seqs, H, V, K]`` fp32 initial/output state). TRT-LLM's GDN + state pool uses the same ``[N, H, V, K]`` logical layout, so the adapter + casts/gathers/scatters without transposing the last two dims. This module is only imported when ``TLLM_USE_FLASHINFER_GDN_PREFILL=1`` is set at process start; do not import it lazily inside hot paths. @@ -35,8 +30,8 @@ import torch from tensorrt_llm._torch.modules.fla.fused_state_io import ( - gather_cast_transpose_kv_to_fp32_vk, - transpose_cast_scatter_fp32_vk_to_kv, + cast_scatter_fp32_vk_to_vk, + gather_cast_vk_to_fp32_vk, ) from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd @@ -107,12 +102,10 @@ def chunk_gated_delta_rule( q3 = l2norm_fwd(q3) k3 = l2norm_fwd(k3) - # --- Step 4: gather initial state and convert layout ----------------- - # TRT-LLM stores SSM state with last two dims as (K, V); FlashInfer expects - # (V, K). Fuse gather + cast-to-fp32 + transpose + contiguous into a single - # Triton kernel so we get one HBM pass and one launch instead of three - # (``[indices]`` gather, ``.to(fp32)`` cast, ``.transpose.contiguous()`` copy). - gathered_init = gather_cast_transpose_kv_to_fp32_vk(initial_state, initial_state_indices) + # --- Step 4: gather initial state and cast dtype --------------------- + # TRT-LLM's GDN kernels and FlashInfer both use [N, H, V, K] state layout. + # Fuse gather + cast-to-fp32 + contiguous into a single Triton kernel. + gathered_init = gather_cast_vk_to_fp32_vk(initial_state, initial_state_indices) # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its @@ -160,26 +153,24 @@ def chunk_gated_delta_rule( ) out_state = None - # --- Step 7: convert state layout back, scatter / return ----------- - # Fuse transpose + cast (fp32 → initial_state.dtype) + optional indexed - # scatter into a single Triton pass, mirroring Step 4. The non-inplace - # branch allocates a fresh (K, V) buffer and writes every row; the inplace - # branch writes only the slots named by ``initial_state_indices`` and - # leaves the rest of ``initial_state`` untouched. + # --- Step 7: cast state back, scatter / return --------------------- + # Fuse cast (fp32 -> initial_state.dtype) + optional indexed scatter into a + # single Triton pass, mirroring Step 4. The inplace branch writes only the + # slots named by ``initial_state_indices`` and leaves the rest untouched. if inplace_indexed_state_update: - transpose_cast_scatter_fp32_vk_to_kv(out_state, initial_state, initial_state_indices) + cast_scatter_fp32_vk_to_vk(out_state, initial_state, initial_state_indices) final_to_return: Optional[torch.Tensor] = None elif output_final_state: num_seqs_out, num_h_out, v_out, k_out = out_state.shape final_to_return = torch.empty( num_seqs_out, num_h_out, - k_out, v_out, + k_out, dtype=initial_state.dtype, device=out_state.device, ) - transpose_cast_scatter_fp32_vk_to_kv(out_state, final_to_return, None) + cast_scatter_fp32_vk_to_vk(out_state, final_to_return, None) else: final_to_return = None diff --git a/tensorrt_llm/_torch/modules/fla/fused_recurrent.py b/tensorrt_llm/_torch/modules/fla/fused_recurrent.py index c93d9e512bd0..5bfdd85f4245 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_recurrent.py +++ b/tensorrt_llm/_torch/modules/fla/fused_recurrent.py @@ -74,7 +74,8 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( b_h = tl.zeros([BK, BV], dtype=tl.float32) if USE_INITIAL_STATE: - p_h0 = h0 + i_nh * K * V + o_k[:, None] * V + o_v[None, :] + # Pool layout [N, HV, V, K] with K innermost: offset v*K + k. + p_h0 = h0 + i_nh * V * K + o_k[:, None] + o_v[None, :] * K b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) for _ in range(0, T): @@ -110,7 +111,8 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( p_beta += HV * (V if IS_BETA_HEADWISE else 1) if STORE_FINAL_STATE: - p_ht = ht + i_nh * K * V + o_k[:, None] * V + o_v[None, :] + # Pool layout [N, HV, V, K]. + p_ht = ht + i_nh * V * K + o_k[:, None] + o_v[None, :] * K tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) @@ -137,7 +139,8 @@ def fused_recurrent_gated_delta_rule_fwd( o = q.new_empty(NK, *v.shape) if output_final_state: - final_state = q.new_empty(N, HV, K, V, dtype=torch.float32) + # Pool layout: [N, HV, V, K] (K innermost). + final_state = q.new_empty(N, HV, V, K, dtype=torch.float32) else: final_state = None @@ -240,11 +243,11 @@ def fused_recurrent_gated_delta_rule( Scale factor for the RetNet attention scores. If not provided, it will default to `1 / sqrt(K)`. Default: `None`. initial_state (Optional[torch.Tensor]): - Initial state of shape `[N, HV, K, V]` for `N` input sequences. + Initial state of shape `[N, HV, V, K]` for `N` input sequences. For equal-length input sequences, `N` equals the batch size `B`. Default: `None`. output_final_state (Optional[bool]): - Whether to output the final state of shape `[N, HV, K, V]`. Default: `False`. + Whether to output the final state of shape `[N, HV, V, K]`. Default: `False`. cu_seqlens (torch.LongTensor): Cumulative sequence lengths of shape `[N+1]` used for variable-length training, consistent with the FlashAttention API. @@ -252,7 +255,7 @@ def fused_recurrent_gated_delta_rule( o (torch.Tensor): Outputs of shape `[B, T, HV, V]`. final_state (torch.Tensor): - Final state of shape `[N, HV, K, V]` if `output_final_state=True` else `None`. + Final state of shape `[N, HV, V, K]` if `output_final_state=True` else `None`. Examples:: >>> import torch >>> import torch.nn.functional as F @@ -265,7 +268,7 @@ def fused_recurrent_gated_delta_rule( >>> v = torch.randn(B, T, HV, V, device='cuda') >>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda')) >>> beta = torch.rand(B, T, HV, device='cuda').sigmoid() - >>> h0 = torch.randn(B, HV, K, V, device='cuda') + >>> h0 = torch.randn(B, HV, V, K, device='cuda') >>> o, ht = fused_gated_recurrent_delta_rule( q, k, v, g, beta, initial_state=h0, @@ -387,8 +390,9 @@ def fused_recurrent_gated_delta_rule_update_fwd_kernel( idx = tl.load(h0_indices + i_n) # Add bounds checking for idx if idx >= 0: # Assuming negative indices are invalid - p_h0 = (h0_source + idx * HV * K * V + i_hv * K * V + - o_k[:, None] * V + o_v[None, :]) + # Pool layout [slots, HV, V, K], K innermost (stride 1). + p_h0 = (h0_source + idx * HV * V * K + i_hv * V * K + o_k[:, None] + + o_v[None, :] * K) b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) # Prepare intermediate state cache variables if enabled @@ -427,12 +431,12 @@ def fused_recurrent_gated_delta_rule_update_fwd_kernel( # store intermediate states if enabled if CACHE_INTERMEDIATE_STATES: if cache_idx >= 0: - # Compute cache pointer for this step - step_offset = step_idx * HV * K * V + # Match the SSM pool layout [slots, HV, V, K] with K innermost. + step_offset = step_idx * HV * V * K cache_ptr = (intermediate_states_buffer + - cache_idx * cache_steps * HV * K * V + - step_offset + i_hv * K * V + o_k[:, None] * V + - o_v[None, :]) + cache_idx * cache_steps * HV * V * K + + step_offset + i_hv * V * K + o_k[:, None] + + o_v[None, :] * K) tl.store(cache_ptr, b_h.to(cache_ptr.dtype.element_ty), mask=mask_h) @@ -447,12 +451,12 @@ def fused_recurrent_gated_delta_rule_update_fwd_kernel( p_beta += HV * (V if IS_BETA_HEADWISE else 1) # Store final state back to h0_source with bounds checking - # ssm states + # ssm states (pool layout [slots, HV, V, K], K innermost). if not DISABLE_STATE_UPDATE: idx = tl.load(h0_indices + i_n) if idx >= 0: # Add bounds checking - p_h0 = (h0_source + idx * HV * K * V + i_hv * K * V + - o_k[:, None] * V + o_v[None, :]) + p_h0 = (h0_source + idx * HV * V * K + i_hv * V * K + o_k[:, None] + + o_v[None, :] * K) tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) diff --git a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py index ffe1fafc649c..7337f7798425 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py +++ b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py @@ -1,5 +1,6 @@ # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py +import os from typing import Optional import torch @@ -7,6 +8,16 @@ import triton.language as tl from tensorrt_llm._torch.modules.fla.utils import custom_device_ctx +from tensorrt_llm.logger import logger + +try: + # A missing build raises ImportError; a CuTe/CUTLASS mismatch raises + # RuntimeError (mirror FlashInfer's own guard) -> Triton fallback. + from flashinfer.gdn_kernels.gdn_decode_bf16_state import \ + gated_delta_rule as _fi_gdn_decode_bf16_state_t1 + _FLASHINFER_GDN_BF16_STATE_AVAILABLE = True +except (ImportError, RuntimeError): + _FLASHINFER_GDN_BF16_STATE_AVAILABLE = False @triton.heuristics({ @@ -98,8 +109,11 @@ def fused_sigmoid_gating_delta_rule_update_kernel( if idx >= 0: tl.device_assert(idx < h0_dim0, "idx out of bounds in h0_source load") - p_h0 = (h0_source + idx * s_h0_0 + i_hv * K * V + - o_k[:, None] * V + o_v[None, :]) + # Pool layout [slots, HV, V, K] with K innermost (stride 1). + # b_h is logically [BK, BV]; element [k, v] lives at + # offset v*K + k within a (V, K) tile. + p_h0 = (h0_source + idx * s_h0_0 + i_hv * V * K + o_k[:, None] + + o_v[None, :] * K) b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) for _ in range(0, seq_T): @@ -166,13 +180,104 @@ def fused_sigmoid_gating_delta_rule_update_kernel( if idx >= 0: tl.device_assert(idx < h0_dim0, "idx out of bounds in h0_source store") - p_h0 = (h0_source + idx * s_h0_0 + i_hv * K * V + - o_k[:, None] * V + o_v[None, :]) + # Pool layout [slots, HV, V, K] with K innermost (stride 1). + p_h0 = (h0_source + idx * s_h0_0 + i_hv * V * K + o_k[:, None] + + o_v[None, :] * K) tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) i_nh += grid_stride_nh +def _can_use_flashinfer_gdn_decode( + initial_state_source: Optional[torch.Tensor], + K: int, + V: int, + T: int, + N: int, +) -> bool: + """Check whether FlashInfer GDN bf16-state decode kernel can be used.""" + # Env-var escape hatch for A/B comparison against the Triton fallback. + if os.environ.get("TRTLLM_FLA_DISABLE_FLASHINFER_GDN", "0") == "1": + return False + if not _FLASHINFER_GDN_BF16_STATE_AVAILABLE: + return False + if initial_state_source is None: + return False + if initial_state_source.dtype != torch.bfloat16: + return False + if K != 128 or V != 128: + return False + if N == 0: + return False + # Standard decode only: T is the flattened token total, so T == N forces + # exactly 1 token/sequence (making the [N, 1, ...] reshape valid). Varlen or + # multi-token batches (T != N) can't be reshaped from T alone -> Triton. + if T != N: + return False + + return True + + +def _flashinfer_gdn_decode( + A_log: torch.Tensor, + a: torch.Tensor, + dt_bias: torch.Tensor, + softplus_beta: float, + softplus_threshold: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + b: torch.Tensor, + initial_state_source: torch.Tensor, + initial_state_indices: torch.Tensor, + scale: float, + use_qk_l2norm_in_kernel: bool, + cu_seqlens: torch.Tensor, +) -> torch.Tensor: + """GDN standard decode via the FlashInfer CuTe-DSL bf16-state kernel. + + Guarded to ``T_per_seq == 1`` (uses ``gated_delta_rule``); the state pool + + indices are passed directly, no caller-side gather/scatter. + """ + N = len(cu_seqlens) - 1 + T_total = q.shape[1] + T_per_seq = T_total // N + HV = v.shape[2] + V = v.shape[3] + + # Reshape from packed varlen [1, N*T, ...] to batched [N, T, ...]. + q_bat = q.view(N, T_per_seq, q.shape[2], q.shape[3]) + k_bat = k.view(N, T_per_seq, k.shape[2], k.shape[3]) + v_bat = v.view(N, T_per_seq, v.shape[2], v.shape[3]) + a_bat = a.view(N, T_per_seq, -1) + b_bat = b.view(N, T_per_seq, -1) + + output = q.new_empty(N, T_per_seq, HV, V) + + assert T_per_seq == 1, ( + f"_flashinfer_gdn_decode expects standard decode (T_per_seq == 1), got " + f"{T_per_seq}; _can_use_flashinfer_gdn_decode should keep T == N") + _fi_gdn_decode_bf16_state_t1( + A_log=A_log, + a=a_bat, + dt_bias=dt_bias, + softplus_beta=softplus_beta, + softplus_threshold=softplus_threshold, + q=q_bat, + k=k_bat, + v=v_bat, + b=b_bat, + initial_state_source=initial_state_source, + initial_state_indices=initial_state_indices.int(), + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + scale=scale, + output=output, + ) + + # Reshape output from [N, T, HV, V] back to [1, N*T, HV, V]. + return output.reshape(1, T_total, HV, -1) + + def fused_sigmoid_gating_delta_rule_update( A_log: torch.Tensor, a: torch.Tensor, @@ -193,11 +298,44 @@ def fused_sigmoid_gating_delta_rule_update( Fused triton implementation of sigmoid gating delta rule update. This function uses a single fused kernel that combines both sigmoid gating computation and the recurrent delta rule update for better performance. + + When FlashInfer's CuTe-DSL GDN decode kernel is available and the state + dtype is bfloat16, dispatches to the faster FlashInfer path automatically. """ B, T, H, K, V = *k.shape, v.shape[-1] HV = v.shape[2] N = B if cu_seqlens is None else len(cu_seqlens) - 1 + if scale is None: + scale = k.shape[-1]**-0.5 + else: + assert scale > 0, "scale must be positive" + + # Dispatch to FlashInfer CuTe-DSL kernel when available and conditions met. + if (cu_seqlens is not None and _can_use_flashinfer_gdn_decode( + initial_state_source, K, V, T, N)): + logger.info_once( + "Using FlashInfer CuTe-DSL kernel for GDN decode " + "(bf16 state, K=V=128)", + key="flashinfer_gdn_decode") + return _flashinfer_gdn_decode( + A_log=A_log, + a=a, + dt_bias=dt_bias, + softplus_beta=softplus_beta, + softplus_threshold=softplus_threshold, + q=q, + k=k, + v=v, + b=b, + initial_state_source=initial_state_source, + initial_state_indices=initial_state_indices, + scale=scale, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + cu_seqlens=cu_seqlens, + ) + + # Fallback: Triton kernel path. # Accept native view layouts from forward_decode rather than forcing packed # copies through input_guard. stride_q = q.stride(1) @@ -211,11 +349,6 @@ def fused_sigmoid_gating_delta_rule_update( num_stages = 3 num_warps = 1 - if scale is None: - scale = k.shape[-1]**-0.5 - else: - assert scale > 0, "scale must be positive" - o = q.new_empty(NK, *v.shape) # (NK, NV, N * HV) is found faster than (N * HV, NV, NK) # As max of grid.z is 65535, we cap grid.z and let each Triton program @@ -225,9 +358,10 @@ def fused_sigmoid_gating_delta_rule_update( if initial_state_source is not None: s_h0_0, s_h0_1, s_h0_2, s_h0_3 = initial_state_source.stride() slot_num = initial_state_source.shape[0] + # Pool layout is [slots, HV, V, K] with K innermost (stride 1). assert s_h0_3 == 1, f"s_h0_3: {s_h0_3} is not 1" - assert s_h0_2 == V, f"s_h0_2: {s_h0_2} is not {V}" - assert s_h0_1 == K * V, f"s_h0_1: {s_h0_1} is not {K * V}" + assert s_h0_2 == K, f"s_h0_2: {s_h0_2} is not {K}" + assert s_h0_1 == V * K, f"s_h0_1: {s_h0_1} is not {V * K}" else: s_h0_0 = 0 slot_num = 0 diff --git a/tensorrt_llm/_torch/modules/fla/fused_state_io.py b/tensorrt_llm/_torch/modules/fla/fused_state_io.py index 05aa06f4a607..c5885f5b6011 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_state_io.py +++ b/tensorrt_llm/_torch/modules/fla/fused_state_io.py @@ -2,18 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 """Fused state I/O kernels for the FlashInfer GDN prefill adapter. -The FlashInfer prefill kernel consumes the SSM state in fp32 with the last -two dims transposed (V, K) relative to TRT-LLM's native (K, V) layout, and -optionally requires gathering a subset of the SSM pool by index. The naive -PyTorch chain is - - gathered_init = initial_state[indices].to(torch.float32) # gather + cast - gathered_init = gathered_init.transpose(-1, -2).contiguous() # transpose copy - -which produces three separate kernel launches and two large intermediate -fp32 buffers (~``num_seqs * H * K * V * 4`` bytes each) per FlashInfer -call. ``gather_cast_transpose_kv_to_fp32_vk`` fuses all three steps into a -single Triton kernel. +The SSM state pool and FlashInfer share the ``[slots, HV, V, K]`` layout +(K innermost), so no transpose is needed. These kernels fuse the gather + +dtype cast (and the reverse cast + scatter) that bridge the bf16 pool and +FlashInfer's fp32 state, avoiding the naive multi-launch PyTorch chain +(``pool[indices].to(fp32)`` / ``.to(bf16)`` + indexed scatter) and its large +intermediate buffers: + +- ``gather_cast_vk_to_fp32_vk``: gather pool slots by index + cast bf16->fp32. +- ``cast_scatter_fp32_vk_to_vk``: cast fp32->bf16 + scatter back to pool slots. """ from typing import Optional @@ -24,223 +21,177 @@ @triton.jit -def _gather_cast_transpose_kv_to_fp32_vk_kernel( +def _gather_cast_vk_to_fp32_vk_kernel( src_ptr, dst_ptr, indices_ptr, HAS_INDICES: tl.constexpr, H: tl.constexpr, - K: tl.constexpr, V: tl.constexpr, + K: tl.constexpr, src_stride_n, src_stride_h, - src_stride_k, src_stride_v, + src_stride_k, dst_stride_n, dst_stride_h, dst_stride_v, dst_stride_k, - BLOCK_K: tl.constexpr, BLOCK_V: tl.constexpr, + BLOCK_K: tl.constexpr, ): - """Read src[indices[seq], h, k, v] (any dtype) → write dst[seq, h, v, k] (fp32). - - Each program instance handles one (seq, head, K-block, V-block) tile. - """ pid_seq = tl.program_id(0) pid_h = tl.program_id(1) - pid_kv = tl.program_id(2) + pid_vk = tl.program_id(2) - num_v_blocks: tl.constexpr = (V + BLOCK_V - 1) // BLOCK_V - pid_kb = pid_kv // num_v_blocks - pid_vb = pid_kv % num_v_blocks + num_k_blocks: tl.constexpr = (K + BLOCK_K - 1) // BLOCK_K + pid_vb = pid_vk // num_k_blocks + pid_kb = pid_vk % num_k_blocks if HAS_INDICES: src_seq = tl.load(indices_ptr + pid_seq).to(tl.int64) else: src_seq = pid_seq.to(tl.int64) - k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) v_offs = pid_vb * BLOCK_V + tl.arange(0, BLOCK_V) - k_mask = k_offs < K + k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) v_mask = v_offs < V - mask = k_mask[:, None] & v_mask[None, :] + k_mask = k_offs < K + mask = v_mask[:, None] & k_mask[None, :] src_addrs = ( src_ptr + src_seq * src_stride_n + pid_h * src_stride_h - + k_offs[:, None] * src_stride_k - + v_offs[None, :] * src_stride_v + + v_offs[:, None] * src_stride_v + + k_offs[None, :] * src_stride_k ) - src_data = tl.load(src_addrs, mask=mask, other=0.0) - src_data_fp32 = src_data.to(tl.float32) - dst_addrs = ( dst_ptr + pid_seq * dst_stride_n + pid_h * dst_stride_h - + v_offs[None, :] * dst_stride_v - + k_offs[:, None] * dst_stride_k + + v_offs[:, None] * dst_stride_v + + k_offs[None, :] * dst_stride_k ) - tl.store(dst_addrs, src_data_fp32, mask=mask) + tl.store(dst_addrs, tl.load(src_addrs, mask=mask, other=0.0).to(tl.float32), mask=mask) -def gather_cast_transpose_kv_to_fp32_vk( +def gather_cast_vk_to_fp32_vk( initial_state: torch.Tensor, initial_state_indices: Optional[torch.Tensor], ) -> torch.Tensor: - """Fused ``initial_state[indices].to(fp32).transpose(-1, -2).contiguous()``. - - Args: - initial_state: ``[N_pool, H, K, V]`` of any float dtype. - initial_state_indices: optional ``[num_seqs]`` int tensor. ``None`` means - "use the whole pool" (equivalent to identity gather). - - Returns: - ``[num_seqs, H, V, K]`` fp32 contiguous, where ``num_seqs == len(indices)`` - when ``indices`` is provided, otherwise ``num_seqs == N_pool``. - """ + """Fused ``initial_state[indices].to(fp32).contiguous()`` for ``[N, H, V, K]`` state.""" assert initial_state.dim() == 4, f"initial_state must be 4D, got {initial_state.shape}" - N_pool, H, K, V = initial_state.shape - + n_pool, h, v, k = initial_state.shape if initial_state_indices is not None: num_seqs = initial_state_indices.shape[0] has_indices = True else: - num_seqs = N_pool + num_seqs = n_pool has_indices = False - output = torch.empty(num_seqs, H, V, K, dtype=torch.float32, device=initial_state.device) - - # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the + # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the full K and V dimensions. # entire (K, V) plane per (seq, head). Larger tiles save grid overhead; # smaller tiles improve occupancy at small num_seqs * H. - BLOCK_K = min(K, 128) - BLOCK_V = min(V, 128) - num_k_blocks = (K + BLOCK_K - 1) // BLOCK_K - num_v_blocks = (V + BLOCK_V - 1) // BLOCK_V - - grid = (num_seqs, H, num_k_blocks * num_v_blocks) - - _gather_cast_transpose_kv_to_fp32_vk_kernel[grid]( + output = torch.empty(num_seqs, h, v, k, dtype=torch.float32, device=initial_state.device) + block_v = min(v, 128) + block_k = min(k, 128) + num_v_blocks = triton.cdiv(v, block_v) + num_k_blocks = triton.cdiv(k, block_k) + grid = (num_seqs, h, num_v_blocks * num_k_blocks) + + _gather_cast_vk_to_fp32_vk_kernel[grid]( initial_state, output, initial_state_indices, HAS_INDICES=has_indices, - H=H, - K=K, - V=V, + H=h, + V=v, + K=k, src_stride_n=initial_state.stride(0), src_stride_h=initial_state.stride(1), - src_stride_k=initial_state.stride(2), - src_stride_v=initial_state.stride(3), + src_stride_v=initial_state.stride(2), + src_stride_k=initial_state.stride(3), dst_stride_n=output.stride(0), dst_stride_h=output.stride(1), dst_stride_v=output.stride(2), dst_stride_k=output.stride(3), - BLOCK_K=BLOCK_K, - BLOCK_V=BLOCK_V, + BLOCK_V=block_v, + BLOCK_K=block_k, ) return output @triton.jit -def _transpose_cast_scatter_fp32_vk_to_kv_kernel( +def _cast_scatter_fp32_vk_to_vk_kernel( src_ptr, dst_ptr, indices_ptr, HAS_INDICES: tl.constexpr, H: tl.constexpr, - K: tl.constexpr, V: tl.constexpr, + K: tl.constexpr, src_stride_n, src_stride_h, src_stride_v, src_stride_k, dst_stride_n, dst_stride_h, - dst_stride_k, dst_stride_v, - BLOCK_K: tl.constexpr, + dst_stride_k, BLOCK_V: tl.constexpr, + BLOCK_K: tl.constexpr, ): - """Read src[seq, h, v, k] (fp32 V,K layout) → write dst[dst_seq, h, k, v] (dst dtype, K,V layout). - - ``dst_seq = indices[seq] if HAS_INDICES else seq``. Casts to ``dst_ptr``'s element dtype. - """ pid_seq = tl.program_id(0) pid_h = tl.program_id(1) - pid_kv = tl.program_id(2) + pid_vk = tl.program_id(2) - num_v_blocks: tl.constexpr = (V + BLOCK_V - 1) // BLOCK_V - pid_kb = pid_kv // num_v_blocks - pid_vb = pid_kv % num_v_blocks + num_k_blocks: tl.constexpr = (K + BLOCK_K - 1) // BLOCK_K + pid_vb = pid_vk // num_k_blocks + pid_kb = pid_vk % num_k_blocks if HAS_INDICES: dst_seq = tl.load(indices_ptr + pid_seq).to(tl.int64) else: dst_seq = pid_seq.to(tl.int64) - k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) v_offs = pid_vb * BLOCK_V + tl.arange(0, BLOCK_V) - k_mask = k_offs < K + k_offs = pid_kb * BLOCK_K + tl.arange(0, BLOCK_K) v_mask = v_offs < V - mask = k_mask[:, None] & v_mask[None, :] + k_mask = k_offs < K + mask = v_mask[:, None] & k_mask[None, :] src_addrs = ( src_ptr + pid_seq * src_stride_n + pid_h * src_stride_h - + v_offs[None, :] * src_stride_v - + k_offs[:, None] * src_stride_k + + v_offs[:, None] * src_stride_v + + k_offs[None, :] * src_stride_k ) - src_data = tl.load(src_addrs, mask=mask, other=0.0) - src_data_cast = src_data.to(dst_ptr.dtype.element_ty) - dst_addrs = ( dst_ptr + dst_seq * dst_stride_n + pid_h * dst_stride_h - + k_offs[:, None] * dst_stride_k - + v_offs[None, :] * dst_stride_v + + v_offs[:, None] * dst_stride_v + + k_offs[None, :] * dst_stride_k ) - tl.store(dst_addrs, src_data_cast, mask=mask) + src_data = tl.load(src_addrs, mask=mask, other=0.0) + tl.store(dst_addrs, src_data.to(dst_ptr.dtype.element_ty), mask=mask) -def transpose_cast_scatter_fp32_vk_to_kv( +def cast_scatter_fp32_vk_to_vk( src_vk: torch.Tensor, dst: torch.Tensor, scatter_indices: Optional[torch.Tensor] = None, ) -> None: - """Fused ``src.transpose(-1, -2).contiguous().to(dst.dtype)`` + optional indexed scatter. - - Reverse of :func:`gather_cast_transpose_kv_to_fp32_vk`. Writes ``src_vk`` into - ``dst`` in TRT-LLM's (K, V) last-two-dims layout, casting to ``dst.dtype``, - in a single Triton pass. - - Args: - src_vk: ``[num_seqs, H, V, K]`` fp32 (FlashInfer output_state layout). - dst: target tensor with last two dims ``(K, V)``. Two cases: - - * ``scatter_indices is None``: ``dst.shape == [num_seqs, H, K, V]`` and - every row is written. - * ``scatter_indices is not None``: ``dst.shape == [N_pool, H, K, V]`` - and ``dst[scatter_indices[i]]`` gets row ``i`` of ``src_vk``; - other rows of ``dst`` are untouched. - - scatter_indices: optional ``[num_seqs]`` int tensor. - - Returns: - ``None``; writes happen in-place into ``dst``. - """ + """Fused fp32-to-dst cast plus optional indexed scatter for ``[N, H, V, K]`` state.""" assert src_vk.dim() == 4, f"src_vk must be 4D, got {src_vk.shape}" assert dst.dim() == 4, f"dst must be 4D, got {dst.shape}" - num_seqs, H, V, K = src_vk.shape + num_seqs, h, v, k = src_vk.shape assert dst.shape[1:] == ( - H, - K, - V, + h, + v, + k, ), f"dst shape {tuple(dst.shape)} incompatible with src {tuple(src_vk.shape)}" if scatter_indices is not None: assert scatter_indices.shape == (num_seqs,), ( @@ -248,29 +199,28 @@ def transpose_cast_scatter_fp32_vk_to_kv( ) has_indices = scatter_indices is not None - BLOCK_K = min(K, 128) - BLOCK_V = min(V, 128) - num_k_blocks = (K + BLOCK_K - 1) // BLOCK_K - num_v_blocks = (V + BLOCK_V - 1) // BLOCK_V - - grid = (num_seqs, H, num_k_blocks * num_v_blocks) + block_v = min(v, 128) + block_k = min(k, 128) + num_v_blocks = triton.cdiv(v, block_v) + num_k_blocks = triton.cdiv(k, block_k) + grid = (num_seqs, h, num_v_blocks * num_k_blocks) - _transpose_cast_scatter_fp32_vk_to_kv_kernel[grid]( + _cast_scatter_fp32_vk_to_vk_kernel[grid]( src_vk, dst, scatter_indices, HAS_INDICES=has_indices, - H=H, - K=K, - V=V, + H=h, + V=v, + K=k, src_stride_n=src_vk.stride(0), src_stride_h=src_vk.stride(1), src_stride_v=src_vk.stride(2), src_stride_k=src_vk.stride(3), dst_stride_n=dst.stride(0), dst_stride_h=dst.stride(1), - dst_stride_k=dst.stride(2), - dst_stride_v=dst.stride(3), - BLOCK_K=BLOCK_K, - BLOCK_V=BLOCK_V, + dst_stride_v=dst.stride(2), + dst_stride_k=dst.stride(3), + BLOCK_V=block_v, + BLOCK_K=block_k, ) diff --git a/tests/unittest/_torch/modules/mamba/test_fused_state_io.py b/tests/unittest/_torch/modules/mamba/test_fused_state_io.py index 1de2d8e91ef0..91bb28e4c893 100644 --- a/tests/unittest/_torch/modules/mamba/test_fused_state_io.py +++ b/tests/unittest/_torch/modules/mamba/test_fused_state_io.py @@ -2,13 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 """Operator-level tests for the fused state I/O Triton kernels. -Covers ``gather_cast_transpose_kv_to_fp32_vk`` and -``transpose_cast_scatter_fp32_vk_to_kv`` from -``tensorrt_llm._torch.modules.fla.fused_state_io``, comparing each against -the equivalent PyTorch chain (``[indices].to(fp32).transpose(-1,-2).contiguous()`` -and its inverse). Both kernels are pure layout-transform + dtype cast, so -the expected tolerance is **bit-exact** when input precision is preserved -through the chain. +Covers ``gather_cast_vk_to_fp32_vk`` and ``cast_scatter_fp32_vk_to_vk`` from +``tensorrt_llm._torch.modules.fla.fused_state_io``, comparing each against the +equivalent PyTorch chain (``pool[indices].to(fp32)`` and its inverse +``.to(dtype)`` + indexed scatter). Both kernels are pure gather/scatter + dtype +cast in the shared ``[slots, HV, V, K]`` layout (no transpose), so the expected +tolerance is **bit-exact** when input precision is preserved through the chain. """ import pytest @@ -32,28 +31,21 @@ def _supported_arch() -> bool: # Reference (un-fused) implementations ------------------------------------- -def _ref_gather_cast_transpose(initial_state, indices): - """PyTorch reference: ``init[idx].to(fp32).transpose(-1, -2).contiguous()``.""" +def _ref_gather_cast(initial_state, indices): if indices is not None: - gathered = initial_state[indices].to(torch.float32) - else: - gathered = initial_state.to(torch.float32) - return gathered.transpose(-1, -2).contiguous() - + return initial_state[indices].to(torch.float32).contiguous() + return initial_state.to(torch.float32).contiguous() -def _ref_transpose_cast_scatter(src_vk, dst, scatter_indices): - """PyTorch reference: ``src.transpose(-1, -2).contiguous().to(dst.dtype)`` + scatter. - Mutates ``dst`` in place. Returns nothing (matches the fused kernel API). - """ - out_kv = src_vk.transpose(-1, -2).contiguous().to(dst.dtype, copy=False) +def _ref_cast_scatter(src_vk, dst, scatter_indices): + out = src_vk.to(dst.dtype, copy=False) if scatter_indices is not None: - dst[scatter_indices] = out_kv + dst[scatter_indices] = out else: - dst.copy_(out_kv) + dst.copy_(out) -# Forward kernel: gather + cast + transpose -------------------------------- +# Forward kernel: gather + cast (vk -> fp32 vk) ---------------------------- @skip_unsupported @@ -62,23 +54,20 @@ def _ref_transpose_cast_scatter(src_vk, dst, scatter_indices): "shape,num_seqs", [ ((16, 4, 128, 128), 2), - ((1, 4, 128, 128), 1), - ((32, 16, 128, 128), 8), - ((8, 4, 64, 128), 3), # K != V - ((8, 4, 128, 64), 3), # K != V + ((8, 4, 64, 128), 3), + ((8, 4, 128, 64), 3), ], ) -def test_gather_cast_transpose_matches_torch_ref(dtype, shape, num_seqs): - """Indexed gather path — output must be bit-exact vs the PyTorch chain.""" - from tensorrt_llm._torch.modules.fla.fused_state_io import gather_cast_transpose_kv_to_fp32_vk +def test_gather_cast_vk_matches_torch_ref(dtype, shape, num_seqs): + from tensorrt_llm._torch.modules.fla.fused_state_io import gather_cast_vk_to_fp32_vk - N_pool, H, K, V = shape - torch.manual_seed(0) - src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) + N_pool, H, V, K = shape + torch.manual_seed(5) + src = (torch.randn(N_pool, H, V, K, device="cuda") * 0.1).to(dtype) indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] - ref = _ref_gather_cast_transpose(src, indices) - out = gather_cast_transpose_kv_to_fp32_vk(src, indices) + ref = _ref_gather_cast(src, indices) + out = gather_cast_vk_to_fp32_vk(src, indices) assert out.shape == (num_seqs, H, V, K) assert out.dtype == torch.float32 @@ -86,112 +75,59 @@ def test_gather_cast_transpose_matches_torch_ref(dtype, shape, num_seqs): torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0) -@skip_unsupported -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) -def test_gather_cast_transpose_no_indices_full_pool(dtype): - """``indices=None`` must process the entire pool, output shape == input shape (with last two transposed).""" - from tensorrt_llm._torch.modules.fla.fused_state_io import gather_cast_transpose_kv_to_fp32_vk - - N_pool, H, K, V = 12, 4, 128, 128 - torch.manual_seed(1) - src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) - - ref = _ref_gather_cast_transpose(src, None) - out = gather_cast_transpose_kv_to_fp32_vk(src, None) - - assert out.shape == (N_pool, H, V, K) - torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0) - - -# Reverse kernel: transpose + cast + scatter ------------------------------- +# Reverse kernel: cast + scatter (fp32 vk -> vk) --------------------------- @skip_unsupported -@pytest.mark.parametrize("dst_dtype", [torch.bfloat16, torch.float16]) -@pytest.mark.parametrize( - "N_pool,num_seqs,H,K,V", - [ - (16, 2, 4, 128, 128), - (32, 8, 16, 128, 128), - (8, 3, 4, 64, 128), # K != V - ], -) -def test_transpose_cast_scatter_inplace_preserves_untouched(dst_dtype, N_pool, num_seqs, H, K, V): - """Scattered slots match torch ref; untouched slots are bit-exact unchanged.""" - from tensorrt_llm._torch.modules.fla.fused_state_io import transpose_cast_scatter_fp32_vk_to_kv +@pytest.mark.parametrize("dst_dtype", [torch.bfloat16, torch.float16, torch.float32]) +def test_cast_scatter_vk_preserves_untouched(dst_dtype): + from tensorrt_llm._torch.modules.fla.fused_state_io import cast_scatter_fp32_vk_to_vk - torch.manual_seed(2) + N_pool, num_seqs, H, V, K = 16, 4, 4, 128, 128 + torch.manual_seed(6) src_vk = torch.randn(num_seqs, H, V, K, device="cuda", dtype=torch.float32) * 0.1 - pool_init = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dst_dtype) + pool_init = (torch.randn(N_pool, H, V, K, device="cuda") * 0.1).to(dst_dtype) indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] pool_ref = pool_init.clone() - _ref_transpose_cast_scatter(src_vk, pool_ref, indices) + _ref_cast_scatter(src_vk, pool_ref, indices) pool_out = pool_init.clone() - transpose_cast_scatter_fp32_vk_to_kv(src_vk, pool_out, indices) + cast_scatter_fp32_vk_to_vk(src_vk, pool_out, indices) - # The touched slots must match the torch ref bit-exact (RNE cast is deterministic). torch.testing.assert_close(pool_out[indices], pool_ref[indices], atol=0.0, rtol=0.0) - # Untouched slots must remain bit-exact to the initial pool. untouched = [i for i in range(N_pool) if i not in indices.tolist()] if untouched: torch.testing.assert_close(pool_out[untouched], pool_init[untouched], atol=0.0, rtol=0.0) -@skip_unsupported -@pytest.mark.parametrize("dst_dtype", [torch.bfloat16, torch.float16, torch.float32]) -def test_transpose_cast_scatter_full_write(dst_dtype): - """``scatter_indices=None`` must write every row; output equals torch ref.""" - from tensorrt_llm._torch.modules.fla.fused_state_io import transpose_cast_scatter_fp32_vk_to_kv - - num_seqs, H, K, V = 5, 4, 128, 128 - torch.manual_seed(3) - src_vk = torch.randn(num_seqs, H, V, K, device="cuda", dtype=torch.float32) * 0.1 - dst_init = torch.randn(num_seqs, H, K, V, device="cuda").to(dst_dtype) - - dst_ref = dst_init.clone() - _ref_transpose_cast_scatter(src_vk, dst_ref, None) - - dst_out = dst_init.clone() - transpose_cast_scatter_fp32_vk_to_kv(src_vk, dst_out, None) - - torch.testing.assert_close(dst_out, dst_ref, atol=0.0, rtol=0.0) - - # Roundtrip ---------------------------------------------------------------- @skip_unsupported @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) def test_roundtrip_gather_then_scatter_bitexact(dtype): - """``gather(src, idx) → scatter back to same idx`` must restore src bit-exact. + """``gather(src, idx) -> scatter back to same idx`` must restore src bit-exact. - Justification: bf16/fp16 → fp32 is lossless (precision strictly extends); - fp32 → bf16/fp16 via RNE is deterministic and recovers the original value - when the fp32 representation came from an unmodified bf16/fp16 source. + bf16/fp16 -> fp32 is lossless; fp32 -> bf16/fp16 via RNE is deterministic + and recovers the original when no math was performed in fp32. """ from tensorrt_llm._torch.modules.fla.fused_state_io import ( - gather_cast_transpose_kv_to_fp32_vk, - transpose_cast_scatter_fp32_vk_to_kv, + cast_scatter_fp32_vk_to_vk, + gather_cast_vk_to_fp32_vk, ) - N_pool, H, K, V = 16, 4, 128, 128 + N_pool, H, V, K = 16, 4, 128, 128 num_seqs = 5 torch.manual_seed(4) - src = (torch.randn(N_pool, H, K, V, device="cuda") * 0.1).to(dtype) + src = (torch.randn(N_pool, H, V, K, device="cuda") * 0.1).to(dtype) indices = torch.randperm(N_pool, device="cuda", dtype=torch.int32)[:num_seqs] - # gather → vk fp32 - vk = gather_cast_transpose_kv_to_fp32_vk(src, indices) - # scatter back to a fresh pool, same indices + vk = gather_cast_vk_to_fp32_vk(src, indices) restored = src.clone() - transpose_cast_scatter_fp32_vk_to_kv(vk, restored, indices) + cast_scatter_fp32_vk_to_vk(vk, restored, indices) - # The selected slots should be identical to the original (RNE roundtrip is lossless - # when no math was performed in fp32). torch.testing.assert_close(restored[indices], src[indices], atol=0.0, rtol=0.0) - # Other slots must be unchanged (we passed in restored = src.clone()). untouched = [i for i in range(N_pool) if i not in indices.tolist()] if untouched: torch.testing.assert_close(restored[untouched], src[untouched], atol=0.0, rtol=0.0) @@ -202,6 +138,6 @@ def test_roundtrip_gather_then_scatter_bitexact(dtype): def test_module_importable(): from tensorrt_llm._torch.modules.fla.fused_state_io import ( # noqa: F401 - gather_cast_transpose_kv_to_fp32_vk, - transpose_cast_scatter_fp32_vk_to_kv, + cast_scatter_fp32_vk_to_vk, + gather_cast_vk_to_fp32_vk, ) From e58e7587c650580586fd316bc07a867e7d8bdc9d Mon Sep 17 00:00:00 2001 From: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Date: Tue, 2 Jun 2026 06:53:10 -0400 Subject: [PATCH 190/308] [https://nvbugs/5940460][fix] Harden FP8 quant fusion matching after PyTorch 26.02 update (#14697) Signed-off-by: Dan Hansen <1+dhansen-nvidia@users.noreply.github.com> Signed-off-by: Pietro Cicotti <5833013+pcicotti@users.noreply.github.com> Co-authored-by: Dan Hansen <1+dhansen-nvidia@users.noreply.github.com> --- tensorrt_llm/_torch/compilation/backend.py | 70 +++-- .../_torch/compilation/patterns/__init__.py | 16 + .../compilation/patterns/ar_residual_norm.py | 163 +++++++++-- tests/integration/test_lists/waives.txt | 16 - .../_torch/multi_gpu/test_user_buffers.py | 274 ++++++++++++++++-- 5 files changed, 437 insertions(+), 102 deletions(-) diff --git a/tensorrt_llm/_torch/compilation/backend.py b/tensorrt_llm/_torch/compilation/backend.py index f300b158e106..b952236e3916 100644 --- a/tensorrt_llm/_torch/compilation/backend.py +++ b/tensorrt_llm/_torch/compilation/backend.py @@ -1,4 +1,5 @@ import os +from collections import OrderedDict from typing import List, Optional import torch @@ -14,6 +15,7 @@ from tensorrt_llm.mapping import Mapping from .multi_stream.auto_multi_stream import multi_stream_schedule +from .patterns import MATCHER_SUBSYSTEM from .patterns.ar_residual_norm import register_ar_fusions from .patterns.residual_add_norm import (register_add_norm, register_add_norm_quant) @@ -24,7 +26,6 @@ class Backend: - _custom_pass_instances: List[PatternMatcherPass] = None _graph_pool_handle: tuple[int, int] = None # Following classes are used to let weakref ref the stream and eventlist objects. @@ -49,8 +50,8 @@ def __init__( self.module_inference_time = 0 self.call_count = 0 self.mapping = mapping - self.custom_passes = Backend.get_custom_pass(enable_userbuffers, - mapping) + self.custom_passes = Backend.build_custom_passes( + enable_userbuffers, mapping) self.rank = tensorrt_llm.mpi_rank() self.enable_inductor = enable_inductor self.capture_num_tokens = sorted(capture_num_tokens or []) @@ -64,30 +65,35 @@ def __init__( Backend._graph_pool_handle = torch.cuda.graph_pool_handle() self.match_count = [] + self.match_count_by_pass = OrderedDict() @classmethod - def get_custom_pass(cls, enable_userbuffers, mapping: Mapping): + def build_custom_passes(cls, enable_userbuffers, mapping: Mapping): world_size = tensorrt_llm.mpi_world_size() - if not cls._custom_pass_instances: - # Really naive pass manager here - cls._custom_pass_instances = [PatternMatcherPass()] - if world_size > 1: - # Currently torch compile cannot work properly with lamport fusion kernel - # TO-DO: Fix this issue - os.environ["DISABLE_LAMPORT_REDUCE_NORM_FUSION"] = "1" - ub_enabled = enable_userbuffers and tensorrt_llm.bindings.internal.userbuffers.ub_supported( - ) - register_ar_fusions(cls._custom_pass_instances, mapping, - ub_enabled) - # Fallback: fuse remaining add+rmsnorm not preceded by allreduce - cls._custom_pass_instances.append(PatternMatcherPass()) - register_add_norm(cls._custom_pass_instances[-1]) - else: - # Add fp8 quant pattern before fp16/bf16 pattern - register_add_norm_quant(cls._custom_pass_instances[-1]) - cls._custom_pass_instances.append(PatternMatcherPass()) - register_add_norm(cls._custom_pass_instances[-1]) - return cls._custom_pass_instances + # Really naive pass manager here + custom_passes = [PatternMatcherPass("add_norm", MATCHER_SUBSYSTEM)] + if world_size > 1: + # Currently torch compile cannot work properly with lamport fusion kernel + # TO-DO: Fix this issue + os.environ["DISABLE_LAMPORT_REDUCE_NORM_FUSION"] = "1" + ub_enabled = enable_userbuffers and tensorrt_llm.bindings.internal.userbuffers.ub_supported( + ) + custom_passes[-1] = PatternMatcherPass("ar_residual_norm", + MATCHER_SUBSYSTEM) + register_ar_fusions(custom_passes, mapping, ub_enabled) + # Fallback: fuse remaining add+rmsnorm not preceded by allreduce + custom_passes.append( + PatternMatcherPass("add_norm_fallback", MATCHER_SUBSYSTEM)) + register_add_norm(custom_passes[-1]) + else: + # Add fp8 quant pattern before fp16/bf16 pattern + custom_passes[-1] = PatternMatcherPass("add_norm_quant", + MATCHER_SUBSYSTEM) + register_add_norm_quant(custom_passes[-1]) + custom_passes.append( + PatternMatcherPass("add_norm_fallback", MATCHER_SUBSYSTEM)) + register_add_norm(custom_passes[-1]) + return custom_passes def bypass_optimization(self): self.no_optimization = True @@ -107,10 +113,20 @@ def optimize( example_inputs: List[torch.Tensor], ): graph = gm.graph + self.match_count = [] + self.match_count_by_pass = OrderedDict() for custom_pass in self.custom_passes: - self.match_count.append(custom_pass.apply(graph)) - while self.match_count[-1]: - self.match_count.append(custom_pass.apply(graph)) + total_match_count = 0 + match_count = custom_pass.apply(graph) + self.match_count.append(match_count) + total_match_count += match_count + while match_count: + match_count = custom_pass.apply(graph) + self.match_count.append(match_count) + total_match_count += match_count + pass_name = custom_pass.pass_name or ( + f"unnamed_pass_{len(self.match_count_by_pass)}") + self.match_count_by_pass[pass_name] = total_match_count graph.eliminate_dead_code() # After this pass, cannot run any dce!!! remove_copy_for_mutates_args(graph) diff --git a/tensorrt_llm/_torch/compilation/patterns/__init__.py b/tensorrt_llm/_torch/compilation/patterns/__init__.py index e69de29bb2d1..bd3c588297ec 100644 --- a/tensorrt_llm/_torch/compilation/patterns/__init__.py +++ b/tensorrt_llm/_torch/compilation/patterns/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MATCHER_SUBSYSTEM = "torch_compile" diff --git a/tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py b/tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py index fb2f8829ef47..dde2512ba05c 100644 --- a/tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py +++ b/tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py @@ -8,11 +8,50 @@ PatternMatcherPass, fwd_only, register_replacement) +from tensorrt_llm.mapping import Mapping + from ...custom_ops.torch_custom_ops import BufferKind from ...distributed import AllReduceFusionOp, AllReduceStrategy +from . import MATCHER_SUBSYSTEM aten = torch.ops.aten -from tensorrt_llm.mapping import Mapping + + +def _append_named_pass(custom_passes: List[PatternMatcherPass], pass_name: str): + custom_passes.append(PatternMatcherPass(pass_name, MATCHER_SUBSYSTEM)) + + +def _check_getitem_only_users(match: Match, pattern_node) -> bool: + node = match.ctx.pattern_to_node[pattern_node] + if not isinstance(node, torch.fx.graph.Node): + return False + for user in node.users: + if user.op != "call_function" or user.target is not getitem: + return False + return True + + +def _has_getitem_user(match: Match, pattern_node, index: int) -> bool: + node = match.ctx.pattern_to_node[pattern_node] + if not isinstance(node, torch.fx.graph.Node): + return False + for user in node.users: + if (user.op == "call_function" and user.target is getitem + and user.args[1] == index): + return True + return False + + +def _make_fp8_quant_extra_check(input_node, strategy_node, quant_node, + require_scale_output: bool): + + def extra_check(match: Match) -> bool: + return (check_f16_bf16_input(match, input_node) + and check_non_ub_strategy(match, strategy_node) + and _check_getitem_only_users(match, quant_node) and + _has_getitem_user(match, quant_node, 1) == require_scale_output) + + return extra_check def register_ar_residual_norm(custom_pass: PatternMatcherPass, mapping: Mapping, @@ -135,15 +174,16 @@ def register_ar_residual_norm_out_fp8_quant(custom_pass: PatternMatcherPass, torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor.default, getitem_0, KeywordArg("scale"), - _users=2) - getitem_2 = CallFunction(getitem, - static_quantize_e4m3_per_tensor_default, - 0, - _users=2) + _users=MULTIPLE) + getitem_2 = CallFunction(getitem, static_quantize_e4m3_per_tensor_default, + 0) getitem_3 = CallFunction(getitem, static_quantize_e4m3_per_tensor_default, 1) - pattern = MultiOutputPattern([getitem_0, getitem_1, getitem_2, getitem_3 - ]) # norm_out, residual_out, quant_out, scale + pattern_with_scale = MultiOutputPattern( + [getitem_0, getitem_1, getitem_2, + getitem_3]) # norm_out, residual_out, quant_out, scale + pattern_without_scale = MultiOutputPattern( + [getitem_0, getitem_1, getitem_2]) # norm_out, residual_out, quant_out def empty_pattern( input: torch.Tensor, @@ -174,9 +214,29 @@ def target_pattern( trigger_completion_at_end) return allreduce[0], allreduce[2], allreduce[1], scale - def extra_check(match: Match) -> bool: - return check_f16_bf16_input( - match, input_node) and check_non_ub_strategy(match, strategy_node) + def target_pattern_without_scale( + input: torch.Tensor, + residual: torch.Tensor, + gamma: torch.Tensor, + workspace: torch.LongTensor, + strategy: int, + eps: float, + scale: torch.Tensor, + trigger_completion_at_end: bool, + ): + allreduce = allreduce_func( + input, residual, gamma, scale, None, workspace, mapping.tp_group, + int(strategy), + int(AllReduceFusionOp.RESIDUAL_RMS_NORM_OUT_QUANT_FP8), float(eps), + trigger_completion_at_end) + return allreduce[0], allreduce[2], allreduce[1] + + extra_check_with_scale = _make_fp8_quant_extra_check( + input_node, strategy_node, static_quantize_e4m3_per_tensor_default, + True) + extra_check_without_scale = _make_fp8_quant_extra_check( + input_node, strategy_node, static_quantize_e4m3_per_tensor_default, + False) register_replacement( empty_pattern, @@ -184,8 +244,18 @@ def extra_check(match: Match) -> bool: [], fwd_only, custom_pass, - search_fn_pattern=pattern, - extra_check=extra_check, + search_fn_pattern=pattern_with_scale, + extra_check=extra_check_with_scale, + ) + + register_replacement( + empty_pattern, + target_pattern_without_scale, + [], + fwd_only, + custom_pass, + search_fn_pattern=pattern_without_scale, + extra_check=extra_check_without_scale, ) @@ -213,15 +283,15 @@ def register_ar_residual_norm_fp8_quant(custom_pass: PatternMatcherPass, torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor.default, getitem_0, KeywordArg("scale"), - _users=2) - getitem_2 = CallFunction(getitem, - static_quantize_e4m3_per_tensor_default, - 0, - _users=2) + _users=MULTIPLE) + getitem_2 = CallFunction(getitem, static_quantize_e4m3_per_tensor_default, + 0) getitem_3 = CallFunction(getitem, static_quantize_e4m3_per_tensor_default, 1) - pattern = MultiOutputPattern([getitem_1, getitem_2, - getitem_3]) # residual_out, quant_out, scale + pattern_with_scale = MultiOutputPattern( + [getitem_1, getitem_2, getitem_3]) # residual_out, quant_out, scale + pattern_without_scale = MultiOutputPattern([getitem_1, getitem_2 + ]) # residual_out, quant_out def empty_pattern( input: torch.Tensor, @@ -251,9 +321,28 @@ def target_pattern( float(eps), trigger_completion_at_end) return allreduce[1], allreduce[0], scale - def extra_check(match: Match) -> bool: - return check_f16_bf16_input( - match, input_node) and check_non_ub_strategy(match, strategy_node) + def target_pattern_without_scale( + input: torch.Tensor, + residual: torch.Tensor, + gamma: torch.Tensor, + workspace: torch.LongTensor, + strategy: int, + eps: float, + scale: torch.Tensor, + trigger_completion_at_end: bool, + ): + allreduce = allreduce_func( + input, residual, gamma, scale, None, workspace, mapping.tp_group, + int(strategy), int(AllReduceFusionOp.RESIDUAL_RMS_NORM_QUANT_FP8), + float(eps), trigger_completion_at_end) + return allreduce[1], allreduce[0] + + extra_check_with_scale = _make_fp8_quant_extra_check( + input_node, strategy_node, static_quantize_e4m3_per_tensor_default, + True) + extra_check_without_scale = _make_fp8_quant_extra_check( + input_node, strategy_node, static_quantize_e4m3_per_tensor_default, + False) register_replacement( empty_pattern, @@ -261,8 +350,18 @@ def extra_check(match: Match) -> bool: [], fwd_only, custom_pass, - search_fn_pattern=pattern, - extra_check=extra_check, + search_fn_pattern=pattern_with_scale, + extra_check=extra_check_with_scale, + ) + + register_replacement( + empty_pattern, + target_pattern_without_scale, + [], + fwd_only, + custom_pass, + search_fn_pattern=pattern_without_scale, + extra_check=extra_check_without_scale, ) @@ -773,16 +872,20 @@ def extra_check(match: Match) -> bool: extra_check=extra_check, ) - custom_passes.append(PatternMatcherPass()) + _append_named_pass( + custom_passes, + f"ub_convert_supported_ar_to_ub:{allreduce_func.__name__}") register_convert_supported_ar_to_ub(custom_passes[-1]) - custom_passes.append(PatternMatcherPass()) + _append_named_pass(custom_passes, f"ub_prologue:{allreduce_func.__name__}") register_ub_prologue_patterns(custom_passes[-1]) - custom_passes.append(PatternMatcherPass()) + _append_named_pass(custom_passes, f"ub_finalize:{allreduce_func.__name__}") register_ub_finalize_patterns(custom_passes[-1]) - custom_passes.append(PatternMatcherPass()) + _append_named_pass( + custom_passes, + f"insert_copy_for_graph_output:{allreduce_func.__name__}") insert_copy_for_graph_output(custom_passes[-1]) @@ -793,7 +896,7 @@ def register_ar_fusions(custom_passes: List[PatternMatcherPass], register_ar_residual_norm(custom_passes[-1], mapping, torch.ops.trtllm.tunable_allreduce) - custom_passes.append(PatternMatcherPass()) + _append_named_pass(custom_passes, "ar_residual_norm_quant") for allreduce_func in [ torch.ops.trtllm.allreduce, torch.ops.trtllm.tunable_allreduce ]: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 5836d3b48335..6d762e93e4c6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -354,22 +354,6 @@ unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[ unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb -k "MEGAMOE_DEEPGEMM" SKIP (https://nvbugs/6175060) unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e256_k8_h7168_i2048-seq=1-dtype=torch.bfloat16-backend=MEGAMOE_DEEPGEMM-quant=W4A8_MXFP4_MXFP8-routing=DeepSeekV3] SKIP (https://nvbugs/6175060) unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py::TestLoraAttentionPytorchFlowVsTRT::test_lora_attention SKIP (https://nvbugs/5701421) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-bf16-_tokens16-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-bf16-_tokens16-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-bf16-_tokens256-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-bf16-_tokens256-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-fp16-_tokens16-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-fp16-_tokens16-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-fp16-_tokens256-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_mm_add_prologue[2-fp16-_tokens256-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-bf16-_tokens16-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-bf16-_tokens16-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-bf16-_tokens256-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-bf16-_tokens256-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens16-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens16-_hidden512] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens256-_hidden32] SKIP (https://nvbugs/5940460) -unittest/_torch/multi_gpu/test_user_buffers.py::test_user_buffers_pass[2-fp16-_tokens256-_hidden512] SKIP (https://nvbugs/5940460) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingDSv3-swiglu-1024-1024-1] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_qwen_next-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_topk_4-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) diff --git a/tests/unittest/_torch/multi_gpu/test_user_buffers.py b/tests/unittest/_torch/multi_gpu/test_user_buffers.py index eb5051afcb52..6162907c06c4 100644 --- a/tests/unittest/_torch/multi_gpu/test_user_buffers.py +++ b/tests/unittest/_torch/multi_gpu/test_user_buffers.py @@ -33,6 +33,15 @@ pytestmark = pytest.mark.threadleak(enabled=False) +def _assert_match_counts(backend, expected_match_count_by_pass): + # Key format: + # - "" for passes that are registered once + # - ":" for UB passes that are registered + # per allreduce backend, where is "allreduce" or + # "tunable_allreduce" + assert dict(backend.match_count_by_pass) == expected_match_count_by_pass + + def create_tp_mapping(tp_size, rank): return Mapping( world_size=tp_size, @@ -428,6 +437,149 @@ def forward(self, input): return res +def run_single_rank_ar_rms_norm_fp8_live_scale_compile(tensor_parallel_size, a, + b, c, gamma, scale): + rank = tensorrt_llm.mpi_rank() + torch.cuda.set_device(rank) + try: + if not ub.ub_supported(): + return True + eps = 1e-6 + dtype = a.dtype + + a = a.cuda() + c = c.cuda() + gamma = gamma.cuda() + scale = scale.cuda() + + ub_size = c.nelement() * c.element_size() + init_userbuffers_allocator(tensor_parallel_size, rank, ub_size) + + b_partial = torch.chunk(b, tensor_parallel_size, 0) + weight = b_partial[rank].cuda() + mapping = create_tp_mapping(tensor_parallel_size, rank) + + class Model(nn.Module): + + def __init__(self): + super().__init__() + self.allreduce = AllReduce(mapping=mapping, + strategy=AllReduceStrategy.AUTO, + dtype=dtype) + + def forward(self, input, residual): + local = torch.chunk(input, tensor_parallel_size, + 1)[rank].contiguous() + hidden = torch.matmul(local, weight) + norm, fused_residual = self.allreduce( + hidden, + all_reduce_params=AllReduceParams( + strategy=AllReduceStrategy.AUTO, + fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, + residual=residual, + norm_weight=gamma, + eps=eps, + )) + q_norm, q_scale = torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor( + norm, scale) + # static_quantize_e4m3_per_tensor leaves its 2nd output + # uninitialized (see fp8Op.cpp:112). Keep q_scale live so the + # live-scale fusion pattern still matches, but override its + # value with the input scale (in input.dtype) so eager and + # fused outputs are comparable. + q_scale = q_scale.to(input.dtype) * 0 + scale.to(input.dtype) + dequantized = dequant(q_norm, q_scale, input.dtype) + return dequantized, fused_residual, q_scale + + model = Model() + backend = Backend(enable_inductor=False, + enable_userbuffers=True, + mapping=mapping) + model_opt = torch.compile(model, backend=backend, fullgraph=True) + + with torch.inference_mode(): + ref_dequantized, ref_residual, ref_scale = model(a, c) + fused_dequantized, fused_residual, fused_scale = model_opt(a, c) + + _assert_match_counts( + backend, { + "ar_residual_norm": 0, + "ar_residual_norm_quant": 1, + "ub_convert_supported_ar_to_ub:allreduce": 0, + "ub_prologue:allreduce": 0, + "ub_finalize:allreduce": 0, + "insert_copy_for_graph_output:allreduce": 0, + "ub_convert_supported_ar_to_ub:tunable_allreduce": 1, + "ub_prologue:tunable_allreduce": 1, + "ub_finalize:tunable_allreduce": 0, + "insert_copy_for_graph_output:tunable_allreduce": 0, + "add_norm_fallback": 0, + }) + + torch.cuda.synchronize() + if rank == 0: + torch.testing.assert_close(fused_dequantized, + ref_dequantized, + atol=5e-1, + rtol=1e-2) + torch.testing.assert_close(fused_residual, + ref_residual, + atol=5e-1, + rtol=1e-2) + torch.testing.assert_close(fused_scale, ref_scale) + except Exception: + traceback.print_exc() + raise + return True + + +def run_single_rank_backend_passes_are_per_instance(tensor_parallel_size): + rank = tensorrt_llm.mpi_rank() + torch.cuda.set_device(rank) + support = ub.ub_supported() + if not support: + return True + + mapping = create_tp_mapping(tensor_parallel_size, rank) + backend_no_ub = Backend(enable_inductor=False, + enable_userbuffers=False, + mapping=mapping) + backend_ub = Backend(enable_inductor=False, + enable_userbuffers=True, + mapping=mapping) + + assert [ + custom_pass.pass_name for custom_pass in backend_no_ub.custom_passes + ] == ["ar_residual_norm", "ar_residual_norm_quant", "add_norm_fallback"] + assert [custom_pass.pass_name + for custom_pass in backend_ub.custom_passes] == [ + "ar_residual_norm", + "ar_residual_norm_quant", + "ub_convert_supported_ar_to_ub:allreduce", + "ub_prologue:allreduce", + "ub_finalize:allreduce", + "insert_copy_for_graph_output:allreduce", + "ub_convert_supported_ar_to_ub:tunable_allreduce", + "ub_prologue:tunable_allreduce", + "ub_finalize:tunable_allreduce", + "insert_copy_for_graph_output:tunable_allreduce", + "add_norm_fallback", + ] + + # A second Backend with the same config must build a distinct pass list — + # guards against any future reintroduction of class-level pass caching. + backend_ub2 = Backend(enable_inductor=False, + enable_userbuffers=True, + mapping=mapping) + for p1, p2 in zip(backend_ub.custom_passes, backend_ub2.custom_passes): + assert p1 is not p2 + for p_no_ub, p_ub in zip(backend_no_ub.custom_passes, + backend_ub.custom_passes): + assert p_no_ub is not p_ub + + return True + + def run_single_rank_ub_pass( tensor_parallel_size, input, l0_weight, l0_input_scale, l0_weight_scale, l1_weight, l1_input_scale, l1_weight_scale, l2_weight, l2_input_scale, @@ -460,15 +612,24 @@ def run_single_rank_ub_pass( model_opt = torch.compile(model, backend=backend, fullgraph=True) with torch.inference_mode(): output_fused = model_opt(input) - # 3 AR_NORM fusion happens first - # 2 AR_NORM fused with Quant - # 3 AR_NORM replacement - # 3 Scaled MM Prologue - # 2 UB Finalize Removal - # 1 Insert copy for graph output - assert backend.match_count == [ - 3, 0, 2, 0, 0, 0, 0, 0, 3, 0, 3, 0, 2, 0, 1, 0 - ] + # Assert the exact named pass totals rather than the raw fixed-point + # trace in backend.match_count. This is still an intentional tripwire + # for optimizer changes, but on semantic pass names instead of + # pass-manager bookkeeping zeros. + _assert_match_counts( + backend, { + "ar_residual_norm": 3, + "ar_residual_norm_quant": 2, + "ub_convert_supported_ar_to_ub:allreduce": 0, + "ub_prologue:allreduce": 0, + "ub_finalize:allreduce": 0, + "insert_copy_for_graph_output:allreduce": 0, + "ub_convert_supported_ar_to_ub:tunable_allreduce": 3, + "ub_prologue:tunable_allreduce": 3, + "ub_finalize:tunable_allreduce": 2, + "insert_copy_for_graph_output:tunable_allreduce": 1, + "add_norm_fallback": 0, + }) torch.cuda.synchronize() if rank == 0: @@ -533,8 +694,7 @@ def ref_scaled_mm_col(x, w, in_s, w_s): rtol=1e-2) except Exception: traceback.print_exc() - - return False + raise return True @@ -581,6 +741,44 @@ def test_user_buffers_pass(hidden, tokens, dtype, mpi_pool_executor): assert r is True +@pytest.mark.skipif(torch.cuda.device_count() < 2, + reason='needs 2 GPUs to run this test') +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16], + ids=lambda x: "fp16" if x == torch.float16 else "bf16") +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_user_buffers_ar_rms_norm_fp8_live_scale_compile( + dtype, mpi_pool_executor): + torch.manual_seed(44) + tensor_parallel_size = 2 + m = 16 + n = 32 + k = 16 + a = torch.randn((m, k), dtype=dtype) + b = torch.randn((k, n), dtype=dtype) + c = torch.randn((m, n), dtype=dtype) + gamma = torch.randn((n), dtype=dtype) + scale = torch.full((1, ), 0.1, dtype=torch.float32) + + results = mpi_pool_executor.map( + run_single_rank_ar_rms_norm_fp8_live_scale_compile, + *zip(*[(tensor_parallel_size, a, b, c, gamma, scale)] * + tensor_parallel_size)) + for r in results: + assert r is True + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, + reason='needs 2 GPUs to run this test') +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_backend_passes_are_per_instance(mpi_pool_executor): + tensor_parallel_size = 2 + results = mpi_pool_executor.map( + run_single_rank_backend_passes_are_per_instance, + *zip(*[(tensor_parallel_size, )] * tensor_parallel_size)) + for r in results: + assert r is True + + def run_single_rank_ar_rms_norm_fp4(tensor_parallel_size, a, b, c, gamma): rank = tensorrt_llm.mpi_rank() torch.cuda.set_device(rank) @@ -762,15 +960,24 @@ def run_single_rank_ub_mm_add_pass(tensor_parallel_size, num_tokens, torch.cuda.synchronize() output_ref = model(mm0_input_0, mm0_input_1, mm1_input_0, mm1_input_1, residual_0, residual_1) - # 3 AR_NORM fusion happens first - # 0 AR_NORM fused with Quant - # 3 AR_NORM replacement - # 3 Prologue - # 1 UB Finalize Removal - # 1 Insert copy for graph output - assert backend.match_count == [ - 3, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 1, 0, 1, 0 - ] + # Assert the exact named pass totals rather than the raw fixed-point + # trace in backend.match_count. This is still an intentional tripwire + # for optimizer changes, but on semantic pass names instead of + # pass-manager bookkeeping zeros. + _assert_match_counts( + backend, { + "ar_residual_norm": 3, + "ar_residual_norm_quant": 0, + "ub_convert_supported_ar_to_ub:allreduce": 0, + "ub_prologue:allreduce": 0, + "ub_finalize:allreduce": 0, + "insert_copy_for_graph_output:allreduce": 0, + "ub_convert_supported_ar_to_ub:tunable_allreduce": 3, + "ub_prologue:tunable_allreduce": 3, + "ub_finalize:tunable_allreduce": 1, + "insert_copy_for_graph_output:tunable_allreduce": 1, + "add_norm_fallback": 0, + }) torch.cuda.synchronize() if rank == 0: @@ -1004,15 +1211,24 @@ def block_scale_unswizzled(scale): output_ref = model(input) output_fused = model_opt(input) - # 3 AR_NORM fusion happens first - # 2 AR_NORM fused with Quant - # 3 AR_NORM replacement - # 3 Scaled MM Prologue - # 2 UB Finalize Removal - # 1 Insert copy for graph output - assert backend.match_count == [ - 3, 0, 2, 0, 0, 0, 0, 0, 3, 0, 3, 0, 2, 0, 1, 0 - ] + # Assert the exact named pass totals rather than the raw fixed-point + # trace in backend.match_count. This is still an intentional tripwire + # for optimizer changes, but on semantic pass names instead of + # pass-manager bookkeeping zeros. + _assert_match_counts( + backend, { + "ar_residual_norm": 3, + "ar_residual_norm_quant": 2, + "ub_convert_supported_ar_to_ub:allreduce": 0, + "ub_prologue:allreduce": 0, + "ub_finalize:allreduce": 0, + "insert_copy_for_graph_output:allreduce": 0, + "ub_convert_supported_ar_to_ub:tunable_allreduce": 3, + "ub_prologue:tunable_allreduce": 3, + "ub_finalize:tunable_allreduce": 2, + "insert_copy_for_graph_output:tunable_allreduce": 1, + "add_norm_fallback": 0, + }) torch.cuda.synchronize() torch.testing.assert_close(output_fused, output_ref, From 6ef7b387c6d93a8d9ca617e15bbce41900572f07 Mon Sep 17 00:00:00 2001 From: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:04:32 -0700 Subject: [PATCH 191/308] [https://nvbugs/6221450][fix] AutoDeploy: Qwen3.5 400B NVFP4 accuracy regression fix (#14667) Signed-off-by: Taylor Yeonbok Lee <249374542+taylor-yb-lee@users.noreply.github.com> --- .../configs/qwen3.5_moe_400b.yaml | 29 +-------- .../auto_deploy/custom_ops/linear/swiglu.py | 6 ++ .../models/custom/modeling_qwen3_5_moe.py | 11 ++-- .../transform/library/fuse_swiglu.py | 60 +++++++++++++++++-- tests/integration/test_lists/waives.txt | 1 - 5 files changed, 72 insertions(+), 35 deletions(-) diff --git a/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml b/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml index 70d0d7f41a0f..adef5dc45183 100644 --- a/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml +++ b/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml @@ -29,33 +29,10 @@ transforms: enabled: true fuse_nvfp4_moe: backend: trtllm_gen - detect_sharding: - # for long input, tp8ep1 gives better performance - # dist_mapping: {moe_tp: 8, moe_ep: 1} + apply_sharding_hints: allreduce_strategy: SYMM_MEM - shard_all_unprocessed: true - simple_shard_filter: "lm_head" - sharding_dims: ['tp','ep', 'bmm'] - # use only manual config for TP sharding - sharding_source: ['manual'] - manual_config: - tp_plan: - # GDN layer - "in_proj_qkv": "delta" - # attention layer - "q_proj": "colwise" - "k_proj": "colwise" - "v_proj": "colwise" - "o_proj": "rowwise" - # lm_head: "gather" = column split + all_gather (not "colwise" which - # requires a LayerSubgraph and crashes for standalone unprocessed nodes) - "lm_head": "gather" - # replicating shared experts (keep them commented out) - # "shared_expert_gate_proj": "colwise" - # "shared_expert_up_proj": "colwise" - # "shared_expert_down_proj": "rowwise" - # gating layer should be replicated as well - # "gate": "gather" + # Shared expert is excluded from sharding for performance purpose + shard_layers: ["moe", "delta", "mha"] multi_stream_moe: stage: compile enabled: true diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/linear/swiglu.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/linear/swiglu.py index d28845d2e30b..92bdd51e6aba 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/linear/swiglu.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/linear/swiglu.py @@ -51,6 +51,7 @@ def torch_swiglu_mlp( gate_bias: Optional[torch.Tensor], up_bias: Optional[torch.Tensor], down_bias: Optional[torch.Tensor], + layer_type: str = "unknown", ) -> torch.Tensor: """Standardized SwiGLU MLP operation. @@ -86,6 +87,7 @@ def _( gate_bias: Optional[torch.Tensor], up_bias: Optional[torch.Tensor], down_bias: Optional[torch.Tensor], + layer_type: str = "unknown", ) -> torch.Tensor: """Fake implementation for tracing.""" # Output shape is [..., hidden_size] where hidden_size = down_weight.shape[0] @@ -159,6 +161,7 @@ def torch_nvfp4_swiglu_mlp( down_input_scale: torch.Tensor, down_weight_scale: torch.Tensor, down_alpha: torch.Tensor, + layer_type: str = "unknown", ) -> torch.Tensor: """NVFP4 quantized SwiGLU MLP operation (intermediate representation). @@ -230,6 +233,7 @@ def _( down_input_scale: torch.Tensor, down_weight_scale: torch.Tensor, down_alpha: torch.Tensor, + layer_type: str = "unknown", ) -> torch.Tensor: """Fake implementation for tracing.""" # Output shape: [..., hidden_size] where hidden_size = down_weight.shape[0] @@ -323,6 +327,7 @@ def torch_finegrained_fp8_swiglu_mlp( gate_weight_scale: torch.Tensor, up_weight_scale: torch.Tensor, down_weight_scale: torch.Tensor, + layer_type: str = "unknown", ) -> torch.Tensor: """FineGrained FP8 quantized SwiGLU MLP operation (intermediate representation). @@ -382,6 +387,7 @@ def _( gate_weight_scale: torch.Tensor, up_weight_scale: torch.Tensor, down_weight_scale: torch.Tensor, + layer_type: str = "unknown", ) -> torch.Tensor: """Fake implementation for tracing.""" # Output shape: [..., hidden_size] where hidden_size = down_weight.shape[0] diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py index b53170f46470..dcb530f0f433 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py @@ -630,21 +630,21 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: self.gate_proj.weight, self.gate_proj.bias, tp_mode="colwise", - layer_type="moe", + layer_type="shared_expert", ) up = torch.ops.auto_deploy.torch_linear_simple( x, self.up_proj.weight, self.up_proj.bias, tp_mode="colwise", - layer_type="moe", + layer_type="shared_expert", ) return torch.ops.auto_deploy.torch_linear_simple( self.act_fn(gate) * up, self.down_proj.weight, self.down_proj.bias, tp_mode="rowwise", - layer_type="moe", + layer_type="shared_expert", ) @@ -765,8 +765,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: layer_type="moe", ) - expert_output = expert_output + shared_expert_output + # The shared expert is replicated (excluded from TP sharding), so all-reduce + # the sharded routed-expert output first, then add the replicated shared + # output; adding before would scale it by the TP world size. expert_output = torch.ops.auto_deploy.all_reduce(expert_output, layer_type="moe") + expert_output = expert_output + shared_expert_output expert_output = expert_output.reshape(batch_size, sequence_length, hidden_dim) return expert_output diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_swiglu.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_swiglu.py index 700cb3693b27..781910fa8928 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_swiglu.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_swiglu.py @@ -22,6 +22,7 @@ The SwiGLU pattern is: silu(x @ gate.T) * (x @ up.T) @ down.T """ +from contextlib import contextmanager from typing import Tuple, Type import torch @@ -47,7 +48,7 @@ eliminate_dead_code, get_attr_by_name, ) -from ...utils.node_utils import is_op +from ...utils.node_utils import extract_op_args, is_op, set_op_args from ...utils.pattern_matcher import ADPatternMatcherPass, register_ad_pattern from ...utils.quantization_utils import ensure_tma_col_major from ..interface import ( @@ -59,6 +60,42 @@ ) +def _weight_key(node: Node): + """Stable key for a weight arg: the get_attr target FQN (survives node re-creation + during pattern replacement), falling back to node identity. ``args[1]`` is the + weight for both linear and SwiGLU ops.""" + w = node.args[1] if len(node.args) > 1 else None + if not isinstance(w, Node): + return None + return w.target if w.op == "get_attr" else w + + +@contextmanager +def preserve_layer_types(gm: GraphModule, linear_op, fused_op): + """Carry the ``layer_type`` hint across a fusion that consumes ``linear_op`` nodes + and emits ``fused_op`` nodes (which would otherwise drop the hint). + + Snapshots each source weight's ``layer_type`` before the rewrite, then re-applies + it to the fused node keyed by weight, so hint-driven sharding (``shard_layers``) + can still classify the fused node. Wrap the matcher's ``patterns.apply`` call. + """ + wmap = {} + for n in gm.graph.nodes: + if is_op(n, linear_op): + [lt] = extract_op_args(n, "layer_type") + key = _weight_key(n) + if lt is not None and key is not None: + wmap[key] = lt + yield + if not wmap: + return + for n in gm.graph.nodes: + if is_op(n, fused_op): + key = _weight_key(n) + if key is not None and key in wmap: + set_op_args(n, layer_type=wmap[key]) + + def _maybe_to_deepgemm_layout( weight: torch.Tensor, scale: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -242,7 +279,12 @@ def _apply( dummy_args=dummy_args_with_bias, ) - num_matches = patterns.apply(gm.graph) + with preserve_layer_types( + gm, + torch.ops.auto_deploy.torch_linear_simple.default, + torch.ops.auto_deploy.torch_swiglu_mlp.default, + ): + num_matches = patterns.apply(gm.graph) if num_matches > 0: gm.recompile() @@ -554,7 +596,12 @@ def _apply( dummy_args=dummy_args, ) - num_matches = patterns.apply(gm.graph) + with preserve_layer_types( + gm, + torch.ops.auto_deploy.torch_fake_quant_nvfp4_linear.default, + torch.ops.auto_deploy.torch_nvfp4_swiglu_mlp.default, + ): + num_matches = patterns.apply(gm.graph) if num_matches > 0: gm.recompile() @@ -835,7 +882,12 @@ def _apply( dummy_args=dummy_args, ) - num_matches = patterns.apply(gm.graph) + with preserve_layer_types( + gm, + torch.ops.auto_deploy.torch_fake_quant_finegrained_fp8_linear.default, + torch.ops.auto_deploy.torch_finegrained_fp8_swiglu_mlp.default, + ): + num_matches = patterns.apply(gm.graph) if num_matches > 0: gm.recompile() diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6d762e93e4c6..46e627cc428d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -19,7 +19,6 @@ accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_ accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-trtllm] SKIP (https://nvbugs/6200112) accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8] SKIP (https://nvbugs/6248757) accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6158397) -accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp_trtllm] SKIP (https://nvbugs/6191524) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] SKIP (https://nvbugs/6084775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] SKIP (https://nvbugs/6029882) From 66262cfe8924da1ddc01e67f649e06f84948efbc Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:05:56 -0700 Subject: [PATCH 192/308] [TRTLLM-12648][test] implement disagg cancel stress metrics_thread (#14807) Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- .../stress_test/disagg_cancel/_testing.py | 92 +++++ .../defs/stress_test/disagg_cancel/harness.py | 153 +++++++- .../disagg_cancel/test_log_scanner.py | 79 +--- .../disagg_cancel/test_metrics_thread.py | 358 ++++++++++++++++++ 4 files changed, 606 insertions(+), 76 deletions(-) create mode 100644 tests/integration/defs/stress_test/disagg_cancel/_testing.py create mode 100644 tests/integration/defs/stress_test/disagg_cancel/test_metrics_thread.py diff --git a/tests/integration/defs/stress_test/disagg_cancel/_testing.py b/tests/integration/defs/stress_test/disagg_cancel/_testing.py new file mode 100644 index 000000000000..c78df131db32 --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/_testing.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared helpers for ``disagg_cancel`` thread-body unit tests. + +Each thread (``log_scanner``, ``metrics``, ``injector``, ``canary``, +``load``) is tested in isolation against a harness wired with +placeholder ``WorkerLaunchSpec`` entries — no live disagg cluster. +Per-thread tests only need to override the spec fields relevant to +their thread, so this module exposes a single ``make_spec`` factory +that fills in the rest, plus a minimal valid marathon YAML for the +``DisaggCancellationStressHarness`` constructor. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from typing import Optional + +from .harness import WorkerLaunchSpec + +# Minimal valid marathon YAML for tests that only need the harness to +# construct. Includes ``log_scan`` so the log-scanner thread has +# patterns to compile; threads that don't read ``log_scan`` ignore it. +DUMMY_YAML = textwrap.dedent( + """\ + hostname: localhost + model: dummy + backend: pytorch + context_servers: {} + generation_servers: {} + stress_config: + duration_min: 1 + kv_cache_manager: v1 + transceiver: cpp + log_scan: + hard_zero_patterns: + - "Broken promise" + - "Segfault" + - "Poisoned .* cache transfer buffer" + """ +) + + +def make_spec( + role: str, + index: int, + *, + port: int = 18000, + host: str = "localhost", + log_path: Optional[Path] = None, +) -> WorkerLaunchSpec: + """``WorkerLaunchSpec`` factory for unit tests. + + Callers override only the fields their thread reads (``log_path`` + for the log scanner, ``host`` / ``port`` for the metrics scraper, + etc.); the rest are placeholders that ``setup_disagg_cluster`` + would normally populate. + + Args: + role: ``"ctx"`` or ``"gen"``. Surfaces in failure reasons. + index: Per-role index (``ctx_0``, ``gen_0``, ...). + port: TCP port for HTTP scraping. Default ``18000``; tests + that bind a real socket pass an OS-allocated port. + host: HTTP hostname; default ``"localhost"``. + log_path: File path the log scanner should tail, or ``None`` + to simulate a worker launched with ``save_log=False``. + """ + return WorkerLaunchSpec( + role=role, + index=index, + model_name="dummy", + worker_config={}, + work_dir="/tmp", + port=port, + device="0", + env={}, + log_path=str(log_path) if log_path is not None else None, + host=host, + ) diff --git a/tests/integration/defs/stress_test/disagg_cancel/harness.py b/tests/integration/defs/stress_test/disagg_cancel/harness.py index 0b324a993037..68fbab35c95a 100644 --- a/tests/integration/defs/stress_test/disagg_cancel/harness.py +++ b/tests/integration/defs/stress_test/disagg_cancel/harness.py @@ -32,6 +32,8 @@ import re import threading import time +import urllib.error +import urllib.request from dataclasses import dataclass, field from pathlib import Path from typing import IO, Any, Callable, Optional @@ -167,6 +169,9 @@ class WorkerLaunchSpec: # ``None`` when launched with ``save_log=False`` (output inherits # pytest stdout); log_scanner skips ``None`` paths with a warning. log_path: Optional[str] = None + # Host for HTTP scraping (Prometheus / health). Multi-host setups + # set this from the YAML ``hostname:`` field at cluster-setup time. + host: str = "localhost" # --------------------------------------------------------------------------- @@ -305,6 +310,79 @@ def _compile_patterns(raw_patterns: list[Any]) -> list[tuple[str, re.Pattern[str return compiled +# --------------------------------------------------------------------------- +# Metrics-thread helpers +# --------------------------------------------------------------------------- + + +# Tolerates optional ``{labels}`` and trailing timestamp; captures value in group 1. +_KV_CACHE_UTIL_RE = re.compile( + r"^trtllm_kv_cache_utilization(?:\{[^}]*\})?\s+([+-]?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)" +) + + +def _parse_kv_cache_utilization(metrics_text: str) -> Optional[float]: + """Extract ``trtllm_kv_cache_utilization`` from a Prometheus exposition. + + Skips ``# HELP`` / ``# TYPE`` lines. Returns the first matching + sample's value as a float, or ``None`` if no sample is present. + + Args: + metrics_text: Body of an HTTP response from + ``/prometheus/metrics``. + + Returns: + Float utilization in the range the worker exposes (typically + ``[0.0, 1.0]``), or ``None`` if the metric is absent. + """ + for line in metrics_text.splitlines(): + if not line or line.startswith("#"): + continue + m = _KV_CACHE_UTIL_RE.match(line) + if m is not None: + try: + return float(m.group(1)) + except ValueError: + return None + return None + + +def _fetch_kv_cache_utilization( + host: str, port: int, timeout_s: float +) -> tuple[Optional[float], Optional[str]]: + """HTTP GET ``/prometheus/metrics`` and parse the KV-cache utilization. + + All failures (connection refused, timeout, HTTP error, parse miss) + are folded into ``(None, error_string)`` rather than raising. The + metrics thread must not fail-fast on transient scrape misses — the + worker may be mid-restart, the metrics endpoint may not be wired + up on every backend, etc. + + Args: + host: Worker hostname or IP. + port: Worker HTTP port. + timeout_s: Per-request timeout in seconds. + + Returns: + Tuple ``(utilization, error)``. Exactly one is ``None``: on + success ``utilization`` is the gauge value and ``error`` is + ``None``; on failure ``utilization`` is ``None`` and ``error`` + is a short string explaining why (for the timeseries record). + """ + url = f"http://{host}:{port}/prometheus/metrics" + try: + with urllib.request.urlopen(url, timeout=timeout_s) as response: + body = response.read().decode("utf-8", errors="replace") + except urllib.error.URLError as exc: + return None, f"url_error: {exc.reason}" + except (TimeoutError, OSError) as exc: + return None, f"io_error: {exc}" + util = _parse_kv_cache_utilization(body) + if util is None: + return None, "metric_absent" + return util, None + + # --------------------------------------------------------------------------- # Harness # --------------------------------------------------------------------------- @@ -337,6 +415,8 @@ def __init__( yaml_path: Path, *, log_scanner_poll_interval_s: float = 0.5, + metrics_scrape_interval_s: float = 30.0, + metrics_scrape_timeout_s: float = 5.0, ) -> None: """Construct a marathon harness. @@ -349,6 +429,14 @@ def __init__( measurable load source on its own; tests pass a smaller value (e.g. 0.02 s) to keep real-clock latency bounded. + metrics_scrape_interval_s: Cadence (seconds) between + consecutive Prometheus scrapes of every worker. + Default 30 s matches the spec's KV-cache utilization + sampling rate; tests pass a smaller value. + metrics_scrape_timeout_s: Per-request HTTP timeout for + the metrics scrape. Short by design — a slow scrape + is recorded as a miss rather than blocking the + metrics thread past its next scheduled scrape. Raises: ValueError: If the YAML is malformed or its @@ -364,12 +452,9 @@ def __init__( self._failure_reason: Optional[str] = None self._failure_lock = threading.Lock() - # Per-thread tunables. The default cadence is reactive enough - # for human-scale debugging (~0.5 s lag from log line to - # fail-fast) without becoming a measurable load source on its - # own; tests pass a smaller value via the constructor to keep - # real-clock latency bounded. self._log_scanner_poll_interval_s: float = log_scanner_poll_interval_s + self._metrics_scrape_interval_s: float = metrics_scrape_interval_s + self._metrics_scrape_timeout_s: float = metrics_scrape_timeout_s # Cluster + worker tracking (populated by setup()). self._cluster: Any = None # tuple returned by setup_disagg_cluster @@ -658,13 +743,59 @@ def _log_scanner_thread_body(self) -> None: logger.debug("[log_scanner] exiting; closed %d source(s)", len(sources)) def _metrics_thread_body(self) -> None: - """Scrape ``/prometheus/metrics`` for KV-cache utilization at ~30 s cadence. - - Stub: no-op. Real implementation parses - ``trtllm_kv_cache_utilization`` and appends timestamped - samples to ``self._kv_utilization_samples``. + """Scrape Prometheus KV-cache utilization from every worker. + + Polls each ``WorkerLaunchSpec`` at + ``_metrics_scrape_interval_s`` cadence, parses + ``trtllm_kv_cache_utilization`` out of + ``/prometheus/metrics``, and appends a timestamped sample to + ``self._kv_utilization_samples``. Each sample records + ``timestamp``, ``role``, ``index``, ``host``, ``port``, + ``utilization`` (``float`` on success, ``None`` on scrape + miss), and ``error`` (``None`` on success, short string on + miss). Scrape misses are recorded — not fail-fast — because + the worker may be mid-restart from a SIGKILL injection, mid- + SIGSTOP pause, or the Prometheus endpoint may not be wired + for the active backend. + + Exits when ``stop_event`` or ``failed_event`` is set. Honors + either signal between scrapes via ``stop_event.wait`` rather + than ``time.sleep`` to keep teardown latency bounded. """ - logger.debug("[metrics_thread] stub — exiting immediately") + if not self._worker_specs: + logger.warning("[metrics_thread] no worker specs; exiting") + return + + logger.info( + "[metrics_thread] scraping %d worker(s) every %.1fs", + len(self._worker_specs), + self._metrics_scrape_interval_s, + ) + + while not self.stop_event.is_set() and not self.failed_event.is_set(): + scrape_start = time.monotonic() + for spec in self._worker_specs: + if self.stop_event.is_set() or self.failed_event.is_set(): + break + util, err = _fetch_kv_cache_utilization( + spec.host, spec.port, self._metrics_scrape_timeout_s + ) + self._kv_utilization_samples.append( + { + "timestamp": time.time(), + "role": spec.role, + "index": spec.index, + "host": spec.host, + "port": spec.port, + "utilization": util, + "error": err, + } + ) + remaining = self._metrics_scrape_interval_s - (time.monotonic() - scrape_start) + if remaining > 0.0: + self.stop_event.wait(timeout=remaining) + + logger.debug("[metrics_thread] exiting; %d sample(s)", len(self._kv_utilization_samples)) # ------------------------------------------------------------------ # Internal helpers diff --git a/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py b/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py index e874377f2fed..a0454b6328f3 100644 --- a/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py +++ b/tests/integration/defs/stress_test/disagg_cancel/test_log_scanner.py @@ -36,65 +36,14 @@ import pytest -from .harness import DisaggCancellationStressHarness, WorkerLaunchSpec, _LogSource +from ._testing import DUMMY_YAML, make_spec +from .harness import DisaggCancellationStressHarness, _LogSource # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- -_DUMMY_YAML = textwrap.dedent( - """\ - hostname: localhost - model: dummy - backend: pytorch - context_servers: {} - generation_servers: {} - stress_config: - duration_min: 1 - kv_cache_manager: v1 - transceiver: cpp - log_scan: - hard_zero_patterns: - - "Broken promise" - - "Segfault" - - "Poisoned .* cache transfer buffer" - """ -) - - -def _make_spec(role: str, index: int, log_path: Path | None) -> WorkerLaunchSpec: - """Tiny ``WorkerLaunchSpec`` factory for unit tests. - - Only ``role`` / ``index`` / ``log_path`` are scanner-relevant — - the other fields are placeholders that would normally be set by - ``setup_disagg_cluster``. - - Args: - role: Worker role tag, ``"ctx"`` or ``"gen"``. Surfaces in - failure reasons via ``_LogSource``. - index: Per-role index (``ctx_0``, ``gen_0``, ...). Surfaces - in failure reasons. - log_path: Path the scanner should tail, or ``None`` to - simulate ``save_log=False`` workers. - - Returns: - A ``WorkerLaunchSpec`` with placeholder values for the - scanner-irrelevant fields. - """ - return WorkerLaunchSpec( - role=role, - index=index, - model_name="dummy", - worker_config={}, - work_dir="/tmp", - port=18000 + index, - device="0", - env={}, - log_path=str(log_path) if log_path is not None else None, - ) - - @pytest.fixture def harness_with_two_workers(tmp_path: Path): """Harness wired to two temp-file log sources (ctx_0 + gen_0). @@ -109,7 +58,7 @@ def harness_with_two_workers(tmp_path: Path): so test wall-clock times stay bounded. """ yaml_path = tmp_path / "stress.yaml" - yaml_path.write_text(_DUMMY_YAML) + yaml_path.write_text(DUMMY_YAML) # Tighten the scanner cadence so the tests finish quickly. The # default 0.5 s makes tests slow; 20 ms keeps real-clock latency @@ -122,8 +71,8 @@ def harness_with_two_workers(tmp_path: Path): gen_log.touch() h._worker_specs = [ - _make_spec("ctx", 0, ctx_log), - _make_spec("gen", 0, gen_log), + make_spec("ctx", 0, ctx_log), + make_spec("gen", 0, gen_log), ] return h, ctx_log, gen_log @@ -360,7 +309,7 @@ def test_log_file_created_after_scanner_starts(harness_with_two_workers): def test_no_worker_specs_exits_immediately(tmp_path: Path): yaml_path = tmp_path / "stress.yaml" - yaml_path.write_text(_DUMMY_YAML) + yaml_path.write_text(DUMMY_YAML) h = DisaggCancellationStressHarness(yaml_path) # No specs at all. @@ -374,11 +323,11 @@ def test_no_worker_specs_exits_immediately(tmp_path: Path): def test_specs_with_none_log_path_skip_gracefully(tmp_path: Path): yaml_path = tmp_path / "stress.yaml" - yaml_path.write_text(_DUMMY_YAML) + yaml_path.write_text(DUMMY_YAML) h = DisaggCancellationStressHarness(yaml_path) h._worker_specs = [ - _make_spec("ctx", 0, None), - _make_spec("gen", 0, None), + make_spec("ctx", 0, None), + make_spec("gen", 0, None), ] t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) @@ -411,7 +360,7 @@ def test_no_hard_zero_patterns_exits_immediately(tmp_path: Path): h = DisaggCancellationStressHarness(yaml_path) ctx_log = tmp_path / "worker_ctx_18000.log" ctx_log.write_text("Broken promise: A\n") # would match in normal config - h._worker_specs = [_make_spec("ctx", 0, ctx_log)] + h._worker_specs = [make_spec("ctx", 0, ctx_log)] t = threading.Thread(target=h._log_scanner_thread_body, daemon=True) t.start() @@ -452,7 +401,7 @@ def test_invalid_regex_is_skipped_with_warning(tmp_path: Path, caplog): h = DisaggCancellationStressHarness(yaml_path, log_scanner_poll_interval_s=0.02) ctx_log = tmp_path / "worker_ctx_18000.log" ctx_log.write_text("Broken promise: A\n") - h._worker_specs = [_make_spec("ctx", 0, ctx_log)] + h._worker_specs = [make_spec("ctx", 0, ctx_log)] with caplog.at_level("ERROR"): fired = _run_scanner_until_failure_or_timeout(h) @@ -494,7 +443,7 @@ def _mark(reason: str) -> None: def test_log_source_poll_returns_false_when_file_absent(tmp_path: Path): - spec = _make_spec("ctx", 0, tmp_path / "missing.log") + spec = make_spec("ctx", 0, tmp_path / "missing.log") src = _LogSource(spec=spec, path=Path(spec.log_path)) # type: ignore[arg-type] seen: list[str] = [] assert src.poll([("X", re.compile("X"))], _consume_marks(seen)) is False @@ -504,7 +453,7 @@ def test_log_source_poll_returns_false_when_file_absent(tmp_path: Path): def test_log_source_poll_returns_false_on_empty_read(tmp_path: Path): log = tmp_path / "empty.log" log.touch() - spec = _make_spec("gen", 0, log) + spec = make_spec("gen", 0, log) src = _LogSource(spec=spec, path=log) seen: list[str] = [] assert src.poll([("X", re.compile("X"))], _consume_marks(seen)) is False @@ -515,7 +464,7 @@ def test_log_source_poll_returns_false_on_empty_read(tmp_path: Path): def test_log_source_close_is_idempotent(tmp_path: Path): log = tmp_path / "x.log" log.write_text("hello\n") - spec = _make_spec("ctx", 0, log) + spec = make_spec("ctx", 0, log) src = _LogSource(spec=spec, path=log) src.poll([("X", re.compile("X"))], lambda r: None) src.close() diff --git a/tests/integration/defs/stress_test/disagg_cancel/test_metrics_thread.py b/tests/integration/defs/stress_test/disagg_cancel/test_metrics_thread.py new file mode 100644 index 000000000000..11d4cf952b4b --- /dev/null +++ b/tests/integration/defs/stress_test/disagg_cancel/test_metrics_thread.py @@ -0,0 +1,358 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ``DisaggCancellationStressHarness._metrics_thread_body``. + +The metrics thread is testable without a real disagg cluster: stand +up a tiny HTTP server in the test process that responds at +``/prometheus/metrics``, point a ``WorkerLaunchSpec`` at it, run the +thread for a few scrape intervals, and assert on the collected +samples. + +Parse-only behaviour (handling of ``# HELP`` / ``# TYPE`` lines, +labels, scientific notation, missing metric) is verified directly +against the standalone parser; transport behaviour (timeouts, +connection refused, restart-mid-scrape) is verified against the +in-process HTTP server. +""" + +from __future__ import annotations + +import socket +import textwrap +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +from ._testing import DUMMY_YAML, make_spec +from .harness import ( + DisaggCancellationStressHarness, + WorkerLaunchSpec, + _fetch_kv_cache_utilization, + _parse_kv_cache_utilization, +) + +# --------------------------------------------------------------------------- +# Fixtures and helpers +# --------------------------------------------------------------------------- + + +def _pick_port() -> int: + """Bind-and-release to get an OS-allocated free TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _MetricsHandler(BaseHTTPRequestHandler): + """Serves a single configurable body at ``/prometheus/metrics``. + + The body is read off the server instance (``server.metrics_body``) + on every request so tests can mutate it mid-run to simulate + utilization drift. + """ + + def do_GET(self) -> None: # noqa: N802 — fixed by BaseHTTPRequestHandler + if self.path != "/prometheus/metrics": + self.send_response(404) + self.end_headers() + return + body: str = getattr(self.server, "metrics_body", "") + if body is None: + # Simulate a worker that's mid-restart: serve a 503 so the + # scraper records a miss rather than a parse failure. + self.send_response(503) + self.end_headers() + return + encoded = body.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *_args, **_kwargs) -> None: # silence test noise + pass + + +@pytest.fixture +def metrics_server(): + """A single in-process HTTP server serving ``/prometheus/metrics``. + + Yields ``(server, port)``. Tests mutate ``server.metrics_body`` to + change what the next scrape sees. + """ + port = _pick_port() + server = HTTPServer(("127.0.0.1", port), _MetricsHandler) + # Default empty body; tests set this to drive scrape outcomes. + server.metrics_body = "" # type: ignore[attr-defined] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server, port + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2.0) + + +def _make_harness(tmp_path: Path, specs: list[WorkerLaunchSpec]) -> DisaggCancellationStressHarness: + """Construct a harness with a minimal YAML and the given worker specs.""" + yaml_path = tmp_path / "marathon.yaml" + yaml_path.write_text(DUMMY_YAML) + h = DisaggCancellationStressHarness( + yaml_path, + metrics_scrape_interval_s=0.05, + metrics_scrape_timeout_s=0.5, + ) + h._worker_specs = list(specs) + return h + + +def _run_metrics_thread_briefly( + harness: DisaggCancellationStressHarness, duration_s: float +) -> None: + """Drive ``_metrics_thread_body`` for ``duration_s`` seconds then stop.""" + thread = threading.Thread(target=harness._metrics_thread_body, daemon=True) + thread.start() + time.sleep(duration_s) + harness.stop_event.set() + thread.join(timeout=2.0) + assert not thread.is_alive(), "metrics thread failed to exit after stop_event" + + +def _wait_until(predicate, *, timeout_s: float, poll_s: float = 0.01) -> None: + """Poll ``predicate`` until true or ``timeout_s`` elapses; assert on timeout.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(poll_s) + raise AssertionError(f"predicate did not become true within {timeout_s}s") + + +# --------------------------------------------------------------------------- +# Parser-only tests +# --------------------------------------------------------------------------- + + +def test_parse_simple_gauge_returns_float() -> None: + body = textwrap.dedent( + """\ + # HELP trtllm_kv_cache_utilization KV cache utilization fraction. + # TYPE trtllm_kv_cache_utilization gauge + trtllm_kv_cache_utilization 0.42 + """ + ) + assert _parse_kv_cache_utilization(body) == pytest.approx(0.42) + + +def test_parse_gauge_with_labels_returns_float() -> None: + body = textwrap.dedent( + """\ + # TYPE trtllm_kv_cache_utilization gauge + trtllm_kv_cache_utilization{instance="ctx_0",pool="kv"} 0.875 + """ + ) + assert _parse_kv_cache_utilization(body) == pytest.approx(0.875) + + +def test_parse_gauge_with_scientific_notation() -> None: + body = "trtllm_kv_cache_utilization 7.5e-2\n" + assert _parse_kv_cache_utilization(body) == pytest.approx(0.075) + + +def test_parse_returns_none_when_metric_absent() -> None: + body = textwrap.dedent( + """\ + # HELP trtllm_request_success_total Counter. + trtllm_request_success_total 17 + """ + ) + assert _parse_kv_cache_utilization(body) is None + + +def test_parse_skips_help_and_type_lines_with_same_prefix() -> None: + # The HELP/TYPE lines mention ``trtllm_kv_cache_utilization`` but + # are not data samples — must not be parsed as values. + body = textwrap.dedent( + """\ + # HELP trtllm_kv_cache_utilization KV cache utilization 0.99 (this is a comment). + # TYPE trtllm_kv_cache_utilization gauge + """ + ) + assert _parse_kv_cache_utilization(body) is None + + +def test_parse_returns_first_sample_when_multiple_present() -> None: + body = textwrap.dedent( + """\ + trtllm_kv_cache_utilization{instance="ctx_0"} 0.10 + trtllm_kv_cache_utilization{instance="gen_0"} 0.90 + """ + ) + assert _parse_kv_cache_utilization(body) == pytest.approx(0.10) + + +# --------------------------------------------------------------------------- +# Fetch-only tests (in-process HTTP server) +# --------------------------------------------------------------------------- + + +def test_fetch_success_returns_value_and_no_error(metrics_server) -> None: + server, port = metrics_server + server.metrics_body = "trtllm_kv_cache_utilization 0.31\n" + util, err = _fetch_kv_cache_utilization("127.0.0.1", port, timeout_s=1.0) + assert util == pytest.approx(0.31) + assert err is None + + +def test_fetch_missing_metric_returns_none_with_metric_absent_error(metrics_server) -> None: + server, port = metrics_server + server.metrics_body = "trtllm_other_metric 99\n" + util, err = _fetch_kv_cache_utilization("127.0.0.1", port, timeout_s=1.0) + assert util is None + assert err == "metric_absent" + + +def test_fetch_connection_refused_returns_url_error() -> None: + # Pick a port and don't bind it. + port = _pick_port() + util, err = _fetch_kv_cache_utilization("127.0.0.1", port, timeout_s=0.5) + assert util is None + assert err is not None and err.startswith("url_error") + + +def test_fetch_http_503_returns_url_error(metrics_server) -> None: + server, port = metrics_server + server.metrics_body = None # handler responds 503 + util, err = _fetch_kv_cache_utilization("127.0.0.1", port, timeout_s=1.0) + assert util is None + assert err is not None and err.startswith("url_error") + + +# --------------------------------------------------------------------------- +# Thread-body integration tests +# --------------------------------------------------------------------------- + + +def test_thread_records_sample_per_scrape(tmp_path, metrics_server) -> None: + server, port = metrics_server + server.metrics_body = "trtllm_kv_cache_utilization 0.25\n" + spec = make_spec("ctx", 0, host="127.0.0.1", port=port) + harness = _make_harness(tmp_path, [spec]) + + _run_metrics_thread_briefly(harness, duration_s=0.2) + + assert len(harness._kv_utilization_samples) >= 2 + for sample in harness._kv_utilization_samples: + assert sample["role"] == "ctx" + assert sample["index"] == 0 + assert sample["host"] == "127.0.0.1" + assert sample["port"] == port + assert sample["utilization"] == pytest.approx(0.25) + assert sample["error"] is None + assert sample["timestamp"] > 0 + + +def test_thread_exits_immediately_when_no_specs(tmp_path) -> None: + harness = _make_harness(tmp_path, []) + thread = threading.Thread(target=harness._metrics_thread_body, daemon=True) + thread.start() + thread.join(timeout=2.0) + assert not thread.is_alive() + assert harness._kv_utilization_samples == [] + + +def test_thread_records_miss_when_scrape_fails(tmp_path) -> None: + # Worker port nobody is listening on → connection refused on every scrape. + port = _pick_port() + spec = make_spec("gen", 0, host="127.0.0.1", port=port) + harness = _make_harness(tmp_path, [spec]) + + _run_metrics_thread_briefly(harness, duration_s=0.15) + + assert len(harness._kv_utilization_samples) >= 1 + for sample in harness._kv_utilization_samples: + assert sample["utilization"] is None + assert sample["error"] is not None + assert sample["role"] == "gen" + + +def test_thread_recovers_when_worker_starts_serving_mid_run(tmp_path, metrics_server) -> None: + server, port = metrics_server + server.metrics_body = None # initial: handler 503s every request + spec = make_spec("ctx", 0, host="127.0.0.1", port=port) + harness = _make_harness(tmp_path, [spec]) + + thread = threading.Thread(target=harness._metrics_thread_body, daemon=True) + thread.start() + try: + # Drive the loop deterministically: wait until at least one miss has + # been recorded before flipping the handler to a success response. + _wait_until( + lambda: any(s["utilization"] is None for s in harness._kv_utilization_samples), + timeout_s=2.0, + ) + server.metrics_body = "trtllm_kv_cache_utilization 0.50\n" + _wait_until( + lambda: any(s["utilization"] is not None for s in harness._kv_utilization_samples), + timeout_s=2.0, + ) + finally: + harness.stop_event.set() + thread.join(timeout=2.0) + + samples = harness._kv_utilization_samples + assert any(s["utilization"] is None for s in samples), "expected a miss before serving" + hits = [s for s in samples if s["utilization"] is not None] + assert hits and all(s["utilization"] == pytest.approx(0.50) for s in hits) + + +def test_thread_scrapes_multiple_workers_per_cycle(tmp_path, metrics_server) -> None: + server, port = metrics_server + server.metrics_body = "trtllm_kv_cache_utilization 0.6\n" + specs = [ + make_spec("ctx", 0, host="127.0.0.1", port=port), + make_spec("ctx", 1, host="127.0.0.1", port=port), + make_spec("gen", 0, host="127.0.0.1", port=port), + ] + harness = _make_harness(tmp_path, specs) + + _run_metrics_thread_briefly(harness, duration_s=0.15) + + # Each cycle should produce one sample per worker. We tolerate + # extra samples (timing-dependent) but require coverage of all + # three roles+indices. + seen = {(s["role"], s["index"]) for s in harness._kv_utilization_samples} + assert ("ctx", 0) in seen + assert ("ctx", 1) in seen + assert ("gen", 0) in seen + + +def test_thread_exits_promptly_on_failed_event(tmp_path, metrics_server) -> None: + server, port = metrics_server + server.metrics_body = "trtllm_kv_cache_utilization 0.0\n" + spec = make_spec("ctx", 0, host="127.0.0.1", port=port) + harness = _make_harness(tmp_path, [spec]) + + thread = threading.Thread(target=harness._metrics_thread_body, daemon=True) + thread.start() + time.sleep(0.05) + harness.failed_event.set() + thread.join(timeout=2.0) + assert not thread.is_alive(), "metrics thread must exit on failed_event too" From 96534502f1c2d6d0009d6caa843355a4687d9166 Mon Sep 17 00:00:00 2001 From: tcherckez-nvidia <127761168+tcherckez-nvidia@users.noreply.github.com> Date: Tue, 2 Jun 2026 19:59:29 +0300 Subject: [PATCH 193/308] [None][chore] Update AD model list (#14686) Signed-off-by: Yueh-Ting (eop) Chen Signed-off-by: tcherckez-nvidia <127761168+tcherckez-nvidia@users.noreply.github.com> Co-authored-by: Yueh-Ting (eop) Chen --- .../configs/granite_4.0_h_small.yaml | 3 +- .../configs/granite_4.0_micro.yaml | 3 +- .../configs/granite_4.0_tiny_preview.yaml | 3 +- .../configs/hunyuan_a13b_ep.yaml | 3 +- .../model_registry/configs/qwen3Next.yaml | 2 + .../auto_deploy/model_registry/models.yaml | 712 ++++++++++-------- 6 files changed, 407 insertions(+), 319 deletions(-) diff --git a/examples/auto_deploy/model_registry/configs/granite_4.0_h_small.yaml b/examples/auto_deploy/model_registry/configs/granite_4.0_h_small.yaml index 205ad566f972..e875c6123c2d 100644 --- a/examples/auto_deploy/model_registry/configs/granite_4.0_h_small.yaml +++ b/examples/auto_deploy/model_registry/configs/granite_4.0_h_small.yaml @@ -7,7 +7,8 @@ # MoE experts distributed via EP across 4 GPUs. This avoids the TP incompatibility # where n_groups=1 makes Mamba B/C SSM dimensions non-shardable. attn_backend: flashinfer -max_batch_size: 16 +cuda_graph_config: + max_batch_size: 16 kv_cache_config: free_gpu_memory_fraction: 0.5 transforms: diff --git a/examples/auto_deploy/model_registry/configs/granite_4.0_micro.yaml b/examples/auto_deploy/model_registry/configs/granite_4.0_micro.yaml index 3df75853ebc1..31d97fb91314 100644 --- a/examples/auto_deploy/model_registry/configs/granite_4.0_micro.yaml +++ b/examples/auto_deploy/model_registry/configs/granite_4.0_micro.yaml @@ -4,6 +4,7 @@ # Configuration for granite-4.0-micro (and similar GraniteMoeHybrid models) # Uses flashinfer attention backend (trtllm backend produces garbled output). attn_backend: flashinfer -max_batch_size: 16 +cuda_graph_config: + max_batch_size: 16 kv_cache_config: free_gpu_memory_fraction: 0.5 diff --git a/examples/auto_deploy/model_registry/configs/granite_4.0_tiny_preview.yaml b/examples/auto_deploy/model_registry/configs/granite_4.0_tiny_preview.yaml index 4060b7e228d3..bc5b2824c451 100644 --- a/examples/auto_deploy/model_registry/configs/granite_4.0_tiny_preview.yaml +++ b/examples/auto_deploy/model_registry/configs/granite_4.0_tiny_preview.yaml @@ -5,7 +5,8 @@ # Uses flashinfer attention backend (trtllm backend produces garbled output with muP scaling) # dtype is missing from config for this model, so we provide it explicitly. attn_backend: flashinfer -max_batch_size: 16 +cuda_graph_config: + max_batch_size: 16 kv_cache_config: free_gpu_memory_fraction: 0.5 model_kwargs: diff --git a/examples/auto_deploy/model_registry/configs/hunyuan_a13b_ep.yaml b/examples/auto_deploy/model_registry/configs/hunyuan_a13b_ep.yaml index 0444f71e6ff2..e0c6957a5697 100644 --- a/examples/auto_deploy/model_registry/configs/hunyuan_a13b_ep.yaml +++ b/examples/auto_deploy/model_registry/configs/hunyuan_a13b_ep.yaml @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # Hunyuan A13B MoE: replicate attention/dense layers and shard routed experts with EP. -max_batch_size: 16 +cuda_graph_config: + max_batch_size: 16 kv_cache_config: free_gpu_memory_fraction: 0.5 transforms: diff --git a/examples/auto_deploy/model_registry/configs/qwen3Next.yaml b/examples/auto_deploy/model_registry/configs/qwen3Next.yaml index 6b44f9d0a987..5d9c904ac055 100644 --- a/examples/auto_deploy/model_registry/configs/qwen3Next.yaml +++ b/examples/auto_deploy/model_registry/configs/qwen3Next.yaml @@ -7,6 +7,8 @@ attn_backend: torch max_seq_len: 4096 max_num_tokens: 4096 max_batch_size: 64 +cuda_graph_config: + max_batch_size: 64 enable_chunked_prefill: true model_factory: AutoModelForCausalLM # disable_overlap_scheduler: false diff --git a/examples/auto_deploy/model_registry/models.yaml b/examples/auto_deploy/model_registry/models.yaml index 704a2d9845b2..e71fef61adaa 100644 --- a/examples/auto_deploy/model_registry/models.yaml +++ b/examples/auto_deploy/model_registry/models.yaml @@ -13,18 +13,22 @@ models: - name: Qwen/Qwen3-0.6B config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] -- name: apple/OpenELM-270M-Instruct - config_id: openelm - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] -- name: apple/OpenELM-1_1B-Instruct - config_id: openelm - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] -- name: apple/OpenELM-3B-Instruct - config_id: openelm - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] -- name: microsoft/Phi-4-mini-instruct - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] +# TypeError: OpenELMConfig.__post_init__() got an unexpected keyword argument 'use_cache'. See https://github.com/NVIDIA/TensorRT-LLM/issues/14672 +# - name: apple/OpenELM-270M-Instruct +# config_id: openelm +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] +# TypeError: OpenELMConfig.__post_init__() got an unexpected keyword argument 'use_cache'. See https://github.com/NVIDIA/TensorRT-LLM/issues/14672 +# - name: apple/OpenELM-1_1B-Instruct +# config_id: openelm +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] +# TypeError: OpenELMConfig.__post_init__() got an unexpected keyword argument 'use_cache'. See https://github.com/NVIDIA/TensorRT-LLM/issues/14672 +# - name: apple/OpenELM-3B-Instruct +# config_id: openelm +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'openelm.yaml'] +# ImportError: cannot import name 'LossKwargs' from 'transformers.utils' (/usr/local/lib/python3.12/dist-packages/transformers/utils/__init__.py) +# - name: microsoft/Phi-4-mini-instruct +# config_id: default_ws_1 +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] - name: microsoft/Phi-4-mini-reasoning config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] @@ -37,9 +41,10 @@ models: - name: meta-llama/Llama-3.1-8B-Instruct config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: casperhansen/llama-3-8b-instruct-awq - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# tensorrt_llm.executor.utils.RequestError: Triton Error [CUDA]: device-side assert triggered +# - name: casperhansen/llama-3-8b-instruct-awq +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: meta-llama/Llama-3.2-1B-Instruct config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] @@ -55,36 +60,42 @@ models: - name: Qwen/Qwen2.5-7B-Instruct config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: Qwen/Qwen2.5-7B-Instruct-AWQ - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# tensorrt_llm.executor.utils.RequestError: Triton Error [CUDA]: device-side assert triggered +# - name: Qwen/Qwen2.5-7B-Instruct-AWQ +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: Qwen/Qwen3-4B config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/Qwen3-8B config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] -- name: microsoft/phi-4 - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: microsoft/Phi-4-reasoning - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: microsoft/Phi-4-reasoning-plus - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: google/gemma-1.1-7b-it - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# RuntimeError: a and b must have same reduction dim, but got [s44*s70, 5120] X [2560, 5120]. See https://github.com/NVIDIA/TensorRT-LLM/issues/14679 +# - name: microsoft/phi-4 +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# RuntimeError: a and b must have same reduction dim, but got [s44*s70, 5120] X [2560, 5120]. See https://github.com/NVIDIA/TensorRT-LLM/issues/14679 +# - name: microsoft/Phi-4-reasoning +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# RuntimeError: a and b must have same reduction dim, but got [s44*s70, 5120] X [2560, 5120]. See https://github.com/NVIDIA/TensorRT-LLM/issues/14679 +# - name: microsoft/Phi-4-reasoning-plus +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: google/gemma-1.1-7b-it +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: google/gemma-2-2b-it config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] - name: google/gemma-2-9b-it config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: google/codegemma-7b-it - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: google/codegemma-7b-it +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: mistralai/Mistral-7B-Instruct-v0.2 config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] @@ -157,9 +168,10 @@ models: - name: nvidia/Mistral-NeMo-Minitron-8B-Base config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: openai/gpt-oss-20b - config_id: gpt_oss_20b - yaml_extra: ['gpt_oss_20b.yaml'] +# OOM during AutoDeploy run. +# - name: openai/gpt-oss-20b +# config_id: gpt_oss_20b +# yaml_extra: ['gpt_oss_20b.yaml'] - name: ibm-granite/granite-3.0-8b-instruct config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] @@ -172,27 +184,30 @@ models: - name: nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4 config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] -- name: nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: google/gemma-3-27b-it - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: deepseek-ai/DeepSeek-V2.5 - config_id: deepseek_v2_ep - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'deepseek_v2_ep.yaml'] -- name: ai21labs/AI21-Jamba-1.5-Mini - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: meta-llama/Llama-3.2-11B-Vision-Instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: meta-llama/Meta-Llama-3-70B-Instruct - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: meta-llama/Llama-3.1-70B-Instruct - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# ValueError: Unrecognized configuration class for this kind of AutoModel: AutoModelForImageTextToText. +# - name: nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# TypeError: UserError.__init__() missing 1 required positional argument: 'msg' +# - name: google/gemma-3-27b-it +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-V2.5 +# config_id: deepseek_v2_ep +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek_v2_ep.yaml'] +# AttributeError: 'NoneType' object has no attribute 'shape' +# - name: meta-llama/Llama-3.2-11B-Vision-Instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# OOM during AutoDeploy run. +# - name: meta-llama/Meta-Llama-3-70B-Instruct +# config_id: default_ws_4 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# OOM during AutoDeploy run. +# - name: meta-llama/Llama-3.1-70B-Instruct +# config_id: default_ws_4 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] - name: meta-llama/Llama-3.3-70B-Instruct config_id: llama3_3_70b yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'llama3_3_70b.yaml'] @@ -211,24 +226,27 @@ models: - name: mistralai/Codestral-22B-v0.1 config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# OOM during AutoDeploy run. +# - name: neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] - name: Qwen/QwQ-32B config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] - name: google/gemma-2-27b-it config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: perplexity-ai/r1-1776-distill-llama-70b - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# OOM during AutoDeploy run. +# - name: perplexity-ai/r1-1776-distill-llama-70b +# config_id: default_ws_4 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] - name: nvidia/NVIDIA-Nemotron-Nano-31B-A3-v3 config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 - config_id: nano_v3 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] +# torch.AcceleratorError: CUDA error: device-side assert triggered +# - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 +# config_id: nano_v3 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] - name: Qwen/QwQ-32B-Preview config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] @@ -241,81 +259,102 @@ models: - name: nvidia/OpenReasoning-Nemotron-32B config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: mistralai/Mistral-Large-Instruct-2407 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: deepseek-ai/DeepSeek-R1-Distill-Llama-70B - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# OOM during AutoDeploy run. +# - name: mistralai/Mistral-Large-Instruct-2407 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-R1-Distill-Llama-70B +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] - name: deepseek-ai/DeepSeek-R1-Distill-Qwen-32B config_id: default_ws_8 yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: mistralai/Mixtral-8x22B-Instruct-v0.1 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# OOM during AutoDeploy run. +# - name: mistralai/Mixtral-8x22B-Instruct-v0.1 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] - name: nvidia/Llama-3.1-70B-Instruct-FP8 config_id: default_ws_8 yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] - name: nvidia/Llama-3.1-405B-Instruct-FP8 config_id: default_ws_8 yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: nvidia/Llama-3_1-Nemotron-51B-Instruct - config_id: simple_shard_only - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] -- name: nvidia/Llama-3_1-Nemotron-Ultra-253B-v1 - config_id: simple_shard_only - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] -- name: nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8 - config_id: simple_shard_only - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] -- name: nvidia/Llama-3_3-Nemotron-Super-49B-v1 - config_id: nemotron_super_49b - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'nemotron_super_49b.yaml'] +# OOM during AutoDeploy run. +# - name: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# TypeError: unsupported operand type(s) for //: 'int' and 'NoneType' +# - name: nvidia/Llama-3_1-Nemotron-51B-Instruct +# config_id: simple_shard_only +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] +# ValueError: DeciLMForCausalLM does not support Flash Attention 2 yet. Please request to add support where the model is hosted, on its model hub page: https://huggingface.co//opt/huggingface/hub/models--nvidia--Llama-3... +# - name: nvidia/Llama-3_1-Nemotron-Ultra-253B-v1 +# config_id: simple_shard_only +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] +# ValueError: DeciLMForCausalLM does not support Flash Attention 2 yet. Please request to add support where the model is hosted, on its model hub page: https://huggingface.co//opt/huggingface/hub/models--nvidia--Llama-3... +# - name: nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8 +# config_id: simple_shard_only +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] +# ValueError: DeciLMForCausalLM does not support Flash Attention 2 yet. Please request to add support where the model is hosted, on its model hub page: https://huggingface.co//opt/huggingface/hub/models--nvidia--Llama-3... +# - name: nvidia/Llama-3_3-Nemotron-Super-49B-v1 +# config_id: nemotron_super_49b +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'nemotron_super_49b.yaml'] - name: Qwen/Qwen3-30B-A3B config_id: simple_shard_only yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'simple_shard_only.yaml'] - name: Qwen/Qwen3-235B-A22B config_id: simple_shard_only yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] -- name: deepseek-ai/DeepSeek-R1 - config_id: deepseek_r1 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] -- name: deepseek-ai/DeepSeek-R1-0528 - config_id: deepseek_r1 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] -- name: deepseek-ai/DeepSeek-Coder-V2-Instruct - config_id: deepseek_v2_ep - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek_v2_ep.yaml'] -- name: Qwen/Qwen3-VL-8B-Instruct - config_id: multimodal__qwen3_vl - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] -- name: Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] -- name: codellama/CodeLlama-70b-Instruct-hf - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: meta-llama/Llama-3.2-90B-Vision-Instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] -- name: openai/gpt-oss-120b - config_id: gpt_oss_120b - yaml_extra: ['gpt_oss_120b.yaml'] -- name: meta-llama/Llama-4-Scout-17B-16E-Instruct - config_id: multimodal__llama4_scout - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_scout.yaml'] -- name: meta-llama/Llama-4-Maverick-17B-128E-Instruct - config_id: multimodal__llama4_maverick_lite - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_maverick_lite.yaml'] -- name: nvidia/NVIDIA-Nemotron-3-Super-120B-BF16-BF16KV-010726 - config_id: super_v3 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml','super_v3.yaml'] -- name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 - config_id: super_v3_mtp - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'super_v3_mtp.yaml', 'enable_sharder_ir.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-R1 +# config_id: deepseek_r1 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-R1-0528 +# config_id: deepseek_r1 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek-r1.yaml', 'enable_sharder_ir.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-Coder-V2-Instruct +# config_id: deepseek_v2_ep +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'deepseek_v2_ep.yaml'] +# Timed out after AutoDeploy compile in multimodal run. +# - name: Qwen/Qwen3-VL-8B-Instruct +# config_id: multimodal__qwen3_vl +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] +# OOM during AutoDeploy run. +# - name: codellama/CodeLlama-70b-Instruct-hf +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# AttributeError: 'NoneType' object has no attribute 'shape' +# - name: meta-llama/Llama-3.2-90B-Vision-Instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] +# torch.distributed.DistStoreError: Timed out after 601 seconds waiting for clients. 1/4 clients joined. +# - name: openai/gpt-oss-120b +# config_id: gpt_oss_120b +# yaml_extra: ['gpt_oss_120b.yaml'] +# [RANK 3] Error querying confidential compute state: Function Not Found +# - name: meta-llama/Llama-4-Scout-17B-16E-Instruct +# config_id: multimodal__llama4_scout +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_scout.yaml'] +# Timed out after AutoDeploy compile in multimodal run. +# - name: meta-llama/Llama-4-Maverick-17B-128E-Instruct +# config_id: multimodal__llama4_maverick_lite +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_maverick_lite.yaml'] +# OOM during AutoDeploy run. +# - name: nvidia/NVIDIA-Nemotron-3-Super-120B-BF16-BF16KV-010726 +# config_id: super_v3 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'super_v3.yaml'] +# OOM during AutoDeploy run. +# - name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 +# config_id: super_v3_mtp +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'super_v3_mtp.yaml', +# 'enable_sharder_ir.yaml'] - name: zai-org/GLM-4.7-Flash config_id: glm_4_7_flash yaml_extra: ['glm-4.7-flash.yaml'] @@ -326,52 +365,63 @@ models: # Model list for sprint # ============================================================================= # --- Qwen3.5 dense (Feb 2026) --- -- name: Qwen/Qwen3.5-0.8B - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: Qwen/Qwen3.5-27B - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# TypeError: Qwen3_5ForCausalLM.__init__() got an unexpected keyword argument 'use_cache'. See https://github.com/NVIDIA/TensorRT-LLM/issues/14672 +# - name: Qwen/Qwen3.5-0.8B +# config_id: default_ws_1 +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] +# TypeError: Qwen3_5ForCausalLM.__init__() got an unexpected keyword argument 'use_cache'. See https://github.com/NVIDIA/TensorRT-LLM/issues/14672 +# - name: Qwen/Qwen3.5-27B +# config_id: default_ws_4 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] # --- Qwen3.5 MoE (Feb 2026) --- -- name: Qwen/Qwen3.5-35B-A3B - config_id: qwen3_5_moe_35b - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'qwen3.5_moe_35b.yaml', 'enable_sharder_ir.yaml'] -- name: Qwen/Qwen3.5-397B-A17B - config_id: qwen3_5_moe_400b - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'qwen3.5_moe_400b.yaml', 'enable_sharder_ir.yaml'] +# tensorrt_llm.executor.utils.RequestError: Ran into a kwarg keyword mismatch. +# - name: Qwen/Qwen3.5-35B-A3B +# config_id: qwen3_5_moe_35b +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'qwen3.5_moe_35b.yaml', 'enable_sharder_ir.yaml'] +# OOM during AutoDeploy run. +# - name: Qwen/Qwen3.5-397B-A17B +# config_id: qwen3_5_moe_400b +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'qwen3.5_moe_400b.yaml', 'enable_sharder_ir.yaml'] # --- GLM-5 (Feb 2026) --- - name: zai-org/GLM-5 config_id: glm_5 yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'glm_5.yaml', 'num_hidden_layers_5.yaml'] -- name: zai-org/GLM-5-FP8 - config_id: glm_5_fp8_bf16 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'glm_5_fp8_bf16.yaml', 'num_hidden_layers_5.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: zai-org/GLM-5-FP8 +# config_id: glm_5_fp8_bf16 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'glm_5_fp8_bf16.yaml', 'num_hidden_layers_5.yaml'] # --- MiniMax-M2.5 / M2.7 (Feb 2026) --- -- name: MiniMaxAI/MiniMax-M2.5 - config_id: minimax_m2 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.yaml'] -- name: MiniMaxAI/MiniMax-M2.7 - config_id: minimax_m2_7 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.7.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: MiniMaxAI/MiniMax-M2.5 +# config_id: minimax_m2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: MiniMaxAI/MiniMax-M2.7 +# config_id: minimax_m2_7 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.7.yaml'] # --- MiMo-V2-Flash (Feb 2026) --- -- name: XiaomiMiMo/MiMo-V2-Flash - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# KeyError: 'default' +# - name: XiaomiMiMo/MiMo-V2-Flash +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] # --- Kimi-K2.5 (Jan 2026) --- -- name: moonshotai/Kimi-K2.5 - config_id: kimi_k2 - yaml_extra: ['kimi_k2.yaml'] +# TypeError: 'NoneType' object is not subscriptable +# - name: moonshotai/Kimi-K2.5 +# config_id: kimi_k2 +# yaml_extra: ['kimi_k2.yaml'] # --- GLM-4.7 (Dec 2025) --- - name: zai-org/GLM-4.7 config_id: glm4_moe yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'glm4_moe.yaml'] # --- DeepSeek V3.2 (Dec 2025) --- -- name: deepseek-ai/DeepSeek-V3.2 - config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] -- name: deepseek-ai/DeepSeek-V3.2-Speciale - config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: deepseek-ai/DeepSeek-V3.2 +# config_id: num_hidden_layers_5 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: deepseek-ai/DeepSeek-V3.2-Speciale +# config_id: num_hidden_layers_5 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] - name: nvidia/DeepSeek-V3.2-NVFP4 config_id: num_hidden_layers_5 yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml'] @@ -380,9 +430,10 @@ models: config_id: glm4_moe yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'glm4_moe.yaml'] # --- Qwen3-Next (Sep 2025) --- -- name: Qwen/Qwen3-Next-80B-A3B-Instruct - config_id: qwen3next - yaml_extra: ['qwen3Next.yaml'] +# OOM during AutoDeploy run. +# - name: Qwen/Qwen3-Next-80B-A3B-Instruct +# config_id: qwen3next +# yaml_extra: ['qwen3Next.yaml'] # --- OLMo 3 (Nov 2025) --- - name: allenai/Olmo-3-7B-Instruct config_id: default_ws_1 @@ -391,19 +442,22 @@ models: config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # --- Command A (2025) --- -- name: CohereLabs/c4ai-command-a-03-2025 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: CohereLabs/command-a-vision-07-2025 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: CohereLabs/c4ai-command-a-03-2025 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# OOM during AutoDeploy run. +# - name: CohereLabs/command-a-vision-07-2025 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] # --- Aya Expanse (2025) - multilingual --- - name: CohereForAI/aya-expanse-8b config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: CohereForAI/aya-expanse-32b - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: CohereForAI/aya-expanse-32b +# config_id: default_ws_4 +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] # --- Tencent Hunyuan (2025) --- - name: tencent/Hunyuan-A13B-Instruct config_id: hunyuan_a13b_ep @@ -426,12 +480,14 @@ models: config_id: granite_4_0_h_small yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'granite_4.0_h_small.yaml'] # --- AI21 Jamba (2025) - hybrid SSM-Transformer --- -- name: ai21labs/AI21-Jamba-Large-1.7 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: ai21labs/AI21-Jamba-Reasoning-3B - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# TypeError: UserError.__init__() missing 1 required positional argument: 'msg' +# - name: ai21labs/AI21-Jamba-Large-1.7 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# TypeError: UserError.__init__() missing 1 required positional argument: 'msg' +# - name: ai21labs/AI21-Jamba-Reasoning-3B +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- Skywork (2025) --- - name: Skywork/Skywork-R1V2-38B config_id: default_ws_4 @@ -464,23 +520,28 @@ models: - name: google/gemma-3n-E2B-it config_id: gemma3n_e2b_it yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma3n_e2b_it.yaml'] -- name: google/gemma-3n-E4B-it - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# TimmWrapperModel requires the timm library but it was not found in your environment. You can install it with pip: +# - name: google/gemma-3n-E4B-it +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] # --- Gemma 4 (2026) - MoE with K=V attention --- -- name: google/gemma-4-E2B-it - config_id: gemma4_e2b - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_e2b.yaml'] -- name: google/gemma-4-26B-A4B - config_id: gemma4_moe_base - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_moe_base.yaml'] -- name: google/gemma-4-26B-A4B-it - config_id: gemma4_moe - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_moe.yaml'] +# Error querying confidential compute state: Function Not Found +# - name: google/gemma-4-E2B-it +# config_id: gemma4_e2b +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_e2b.yaml'] +# AttributeError: 'GraphModule' object has no attribute 'get_per_layer_inputs' +# - name: google/gemma-4-26B-A4B +# config_id: gemma4_moe_base +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_moe_base.yaml'] +# AttributeError: 'GraphModule' object has no attribute 'get_per_layer_inputs' +# - name: google/gemma-4-26B-A4B-it +# config_id: gemma4_moe +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'gemma4_moe.yaml'] # --- Gemma 4 (2026) - Dense 31B --- -- name: google/gemma-4-31B-it - config_id: gemma4_dense - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'gemma4_dense.yaml'] +# AttributeError: 'GraphModule' object has no attribute 'get_per_layer_inputs' +# - name: google/gemma-4-31B-it +# config_id: gemma4_dense +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'gemma4_dense.yaml'] # --- JetBrains Mellum (Apr 2025) - code specialist --- - name: JetBrains/Mellum-4b-sft-all config_id: default_ws_1 @@ -497,23 +558,28 @@ models: config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'enable_sharder_ir.yaml'] # --- Llama 4 base models (Apr 2025) --- -- name: meta-llama/Llama-4-Scout-17B-16E - config_id: multimodal__llama4_scout - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_scout.yaml'] -- name: meta-llama/Llama-4-Maverick-17B-128E - config_id: multimodal__llama4_maverick_lite - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_maverick_lite.yaml'] +# OOM during AutoDeploy run. +# - name: meta-llama/Llama-4-Scout-17B-16E +# config_id: multimodal__llama4_scout +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_scout.yaml'] +# [RANK 1] Error querying confidential compute state: Function Not Found +# - name: meta-llama/Llama-4-Maverick-17B-128E +# config_id: multimodal__llama4_maverick_lite +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'llama4_maverick_lite.yaml'] # --- Phi-4 variants (2025) --- -- name: microsoft/Phi-4-multimodal-instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: microsoft/Phi-4-reasoning-vision-15B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# ValueError: Unrecognized configuration class for this kind of AutoModel: AutoModelForImageTextToText. +# - name: microsoft/Phi-4-multimodal-instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# AttributeError: module 'transformers.models.siglip2.image_processing_siglip2' has no attribute 'filter_out_non_signature_kwargs' +# - name: microsoft/Phi-4-reasoning-vision-15B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] # --- MiniMax M2 (2025) --- -- name: MiniMaxAI/MiniMax-M2 - config_id: minimax_m2 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: MiniMaxAI/MiniMax-M2 +# config_id: minimax_m2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'minimax_m2.yaml'] # --- Tencent Hunyuan small (2025) --- - name: tencent/Hunyuan-1.8B-Instruct config_id: default_ws_1 @@ -522,37 +588,44 @@ models: config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # --- UI-TARS (2025) - GUI agent VLM --- -- name: ByteDance-Seed/UI-TARS-1.5-7B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# [RANK 1] Error querying confidential compute state: Function Not Found +# - name: ByteDance-Seed/UI-TARS-1.5-7B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] # --- Nvidia Nemotron Flash (2025) --- -- name: nvidia/Nemotron-Flash-3B-Instruct - config_id: nemotron_flash - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'nemotron_flash.yaml'] +# ValueError: The `layer_types` entries must be in ('full_attention', 'sliding_attention', 'chunked_attention', 'linear_attention', 'conv', 'mamba', 'attention', 'sparse', 'dense', 'hybrid', 'moe') but got ['deltanet', ... +# - name: nvidia/Nemotron-Flash-3B-Instruct +# config_id: nemotron_flash +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'nemotron_flash.yaml'] # --- InternLM3 (Jan 2025) --- - name: internlm/internlm3-8b-instruct config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- Gemma 3 missing sizes (Mar 2025) --- -- name: google/gemma-3-4b-it - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: google/gemma-3-12b-it - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# TypeError: UserError.__init__() missing 1 required positional argument: 'msg' +# - name: google/gemma-3-4b-it +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# TypeError: UserError.__init__() missing 1 required positional argument: 'msg' +# - name: google/gemma-3-12b-it +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] # --- Mistral Small (2025) --- - name: mistralai/Mistral-Small-24B-Instruct-2501 config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: mistralai/Mistral-Small-3.1-24B-Instruct-2503 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] -- name: mistralai/Mistral-Small-3.2-24B-Instruct-2506 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] -- name: mistralai/Mistral-Small-4-119B-2603 - config_id: multimodal__mistral_small_4_119b - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'mistral_small_4_119b.yaml'] +# [RANK 0] Error querying confidential compute state: Function Not Found +# - name: mistralai/Mistral-Small-3.1-24B-Instruct-2503 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# OSError: Can't load image processor for '/opt/huggingface/hub/models--mistralai--Mistral-Small-3.2-24B-Instruct-2506/snapshots/95a6d26c4bfb886c58daf9d3f7332c857cb27b43'. If you were trying to load it from 'https://hug... +# - name: mistralai/Mistral-Small-3.2-24B-Instruct-2506 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# huggingface_hub.errors.HFValidationError: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/opt/huggingface/hub/models--mistralai--Mistral-Small-4-119B-2603/snapshots/86caf1b82960d2b0fa735f6f169e13d5... +# - name: mistralai/Mistral-Small-4-119B-2603 +# config_id: multimodal__mistral_small_4_119b +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml', 'mistral_small_4_119b.yaml'] # --- DeepSeek R1 distills (Jan 2025) --- - name: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B config_id: default_ws_2 @@ -564,9 +637,10 @@ models: config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- DeepSeek Prover V2 671B (2025) --- -- name: deepseek-ai/DeepSeek-Prover-V2-671B - config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: deepseek-ai/DeepSeek-Prover-V2-671B +# config_id: num_hidden_layers_5 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] # --- OLMo 2 (Mar 2025) --- - name: allenai/OLMo-2-0325-32B-Instruct config_id: default_ws_1 @@ -575,12 +649,14 @@ models: config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # --- Command A variants (2025) --- -- name: CohereLabs/command-a-translate-08-2025 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] -- name: CohereLabs/command-a-reasoning-08-2025 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: CohereLabs/command-a-translate-08-2025 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: CohereLabs/command-a-reasoning-08-2025 +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] # --- Falcon3 (Dec 2024) --- - name: tiiuae/Falcon3-1B-Instruct config_id: default_ws_1 @@ -599,13 +675,15 @@ models: - name: HuggingFaceTB/SmolLM2-135M-Instruct config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: HuggingFaceTB/SmolLM2-1.7B-Instruct - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: HuggingFaceTB/SmolLM2-1.7B-Instruct +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- Qwen2.5-Coder (Nov 2024) --- -- name: Qwen/Qwen2.5-Coder-1.5B-Instruct - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] +# IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 +# - name: Qwen/Qwen2.5-Coder-1.5B-Instruct +# config_id: default_ws_2 +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] - name: Qwen/Qwen2.5-Coder-32B-Instruct config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] @@ -618,16 +696,18 @@ models: config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'enable_sharder_ir.yaml'] # --- DeepSeek V3 (Jan 2025) --- -- name: deepseek-ai/DeepSeek-V3 - config_id: num_hidden_layers_5 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: deepseek-ai/DeepSeek-V3 +# config_id: num_hidden_layers_5 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] # --- Qwen2.5 larger sizes --- - name: Qwen/Qwen2.5-14B-Instruct config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: Qwen/Qwen2.5-72B-Instruct - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# OOM during AutoDeploy run. +# - name: Qwen/Qwen2.5-72B-Instruct +# config_id: default_ws_8 +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] # --- Qwen2.5-Math --- - name: Qwen/Qwen2.5-Math-7B-Instruct config_id: default_ws_2 @@ -639,32 +719,19 @@ models: - name: nvidia/NVIDIA-Nemotron-Nano-12B-v2 config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -# --- Perplexity R1-1776 distill Qwen (2025) --- -- name: perplexity-ai/r1-1776-distill-qwen-32b - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -# --- Nanbeige 8B (2025) --- -- name: Nanbeige/Nanbeige4.1-8B - config_id: default_ws_2 - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- Qwen3 MoE updates (2025) --- - name: Qwen/Qwen3-30B-A3B-Instruct-2507 config_id: simple_shard_only yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'simple_shard_only.yaml'] -- name: Qwen/Qwen3-0.6B-FP8 - config_id: qwen3_fp8_bf16 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'qwen3_fp8_bf16.yaml', 'enable_sharder_ir.yaml'] -# --- Mistral updates (2025) --- -- name: mistralai/Codestral-25.01 - config_id: default_ws_4 - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: mistralai/Mistral-Large-Instruct-2501 - config_id: default_ws_8 - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml'] +# RuntimeError: NVRTC compilation failed. See https://github.com/NVIDIA/TensorRT-LLM/issues/14676 +# - name: Qwen/Qwen3-0.6B-FP8 +# config_id: qwen3_fp8_bf16 +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'qwen3_fp8_bf16.yaml', 'enable_sharder_ir.yaml'] # --- Qwen3-VL 2B (2025) --- -- name: Qwen/Qwen3-VL-2B-Instruct - config_id: multimodal__qwen3_vl - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] +# Timed out after AutoDeploy compile in multimodal run. +# - name: Qwen/Qwen3-VL-2B-Instruct +# config_id: multimodal__qwen3_vl +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] # --- Granite 4.0 tiny (2025) --- - name: ibm-granite/granite-4.0-tiny-preview config_id: granite_4_0_tiny_preview @@ -678,9 +745,10 @@ models: config_id: default_ws_2 yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] # --- Pixtral (2025) - VLM --- -- name: mistralai/Pixtral-12B-2409 - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# ValueError: Unrecognized processing class in /opt/huggingface/hub/models--mistralai--Pixtral-12B-2409/snapshots/2bf7df564a9b56007bbdaa4d299a9315bee5ca47. Can't instantiate a processor, a tokenizer, an image processor,... +# - name: mistralai/Pixtral-12B-2409 +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] # --- DeepSeek Coder V2 Lite (2025) --- - name: deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct config_id: deepseek_v2_ep @@ -694,57 +762,71 @@ models: config_id: default_ws_1 yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # --- Phi-4-mini flash reasoning (2025) --- -- name: microsoft/Phi-4-mini-flash-reasoning - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] +# ImportError: This modeling file requires the following packages that were not found in your environment: causal_conv1d, causal_conv1d_cuda, mamba_ssm, selective_scan_cuda. Run `pip install causal_conv1d causal_conv1d_... +# - name: microsoft/Phi-4-mini-flash-reasoning +# config_id: default_ws_1 +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # --- Qwen2.5-VL (2025) - top VLM family --- -- name: Qwen/Qwen2.5-VL-7B-Instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: Qwen/Qwen2.5-VL-72B-Instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] +# Timed out after AutoDeploy compile in multimodal run. +# - name: Qwen/Qwen2.5-VL-7B-Instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# OOM during AutoDeploy run. +# - name: Qwen/Qwen2.5-VL-72B-Instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] # --- Qwen3-VL MoE (2025) - flagship VLM --- -- name: Qwen/Qwen3-VL-30B-A3B-Instruct - config_id: multimodal__qwen3_vl - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] +# AssertionError: Start linear node (index 234) not found in opening linear nodes - start_linear node: layers_46_mlp_gate_torch_linear_simple_234, opening_linear_nodes: ['layers_47_self_attn_v_proj_torch_linear_simple_2... +# - name: Qwen/Qwen3-VL-30B-A3B-Instruct +# config_id: multimodal__qwen3_vl +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml', 'qwen3_vl.yaml'] # --- InternVL3 (Apr 2025) - #1 open-source VLM --- -- name: OpenGVLab/InternVL3-8B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] -- name: OpenGVLab/InternVL3-78B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] +# AttributeError: TokenizersBackend has no attribute tokenizer. Did you mean: '_tokenizer'? +# - name: OpenGVLab/InternVL3-8B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# AttributeError: TokenizersBackend has no attribute tokenizer. Did you mean: '_tokenizer'? +# - name: OpenGVLab/InternVL3-78B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'multimodal.yaml'] # --- InternVL3.5 (2025) - latest gen --- -- name: OpenGVLab/InternVL3_5-8B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# AttributeError: TokenizersBackend has no attribute start_image_token +# - name: OpenGVLab/InternVL3_5-8B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] # --- SmolVLM2 (2025) - tiny VLM --- -- name: HuggingFaceTB/SmolVLM2-2.2B-Instruct - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# ImportError: Package `num2words` is required to run SmolVLM processor. Install it with `pip install num2words`. +# - name: HuggingFaceTB/SmolVLM2-2.2B-Instruct +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] # --- Molmo2 (2025) - fully open VLM --- -- name: allenai/Molmo2-8B - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'multimodal.yaml'] +# TypeError: Unexpected keyword argument image_use_col_tokens. +# - name: allenai/Molmo2-8B +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml', 'multimodal.yaml'] # --- DeepSeek-VL2 (2025) - MoE VLM --- -- name: deepseek-ai/deepseek-vl2-small - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] +# AttributeError: TokenizersBackend has no attribute tokenizer. Did you mean: '_tokenizer'? +# - name: deepseek-ai/deepseek-vl2-small +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml'] # --- Aya Vision (2025) - multilingual VLM --- -- name: CohereLabs/aya-vision-8b - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml', 'aya_vision_8b.yaml'] -- name: CohereLabs/aya-vision-32b - config_id: multimodal - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] +# tensorrt_llm.executor.utils.RequestError: Triton Error [CUDA]: device-side assert triggered +# - name: CohereLabs/aya-vision-8b +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml', 'multimodal.yaml', 'aya_vision_8b.yaml'] +# tensorrt_llm.executor.utils.RequestError: Triton Error [CUDA]: device-side assert triggered +# - name: CohereLabs/aya-vision-32b +# config_id: multimodal +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'multimodal.yaml'] # ============================================================================= # IR sharding (hint-driven) models # These use enable_sharder_ir.yaml to opt in to the new sharding system. # ============================================================================= -- name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-IR - config_id: nano_v3__enable_sharder_ir - yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] -- name: deepseek-ai/DeepSeek-R1-IR - config_id: num_hidden_layers_5__enable_sharder_ir - yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] +# tensorrt_llm.executor.utils.RequestError: Triton Error [CUDA]: device-side assert triggered +# - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 +# config_id: nano_v3__enable_sharder_ir +# yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml', 'nano_v3.yaml', 'enable_sharder_ir.yaml'] +# OOM during AutoDeploy run. +# - name: deepseek-ai/DeepSeek-R1 +# config_id: num_hidden_layers_5__enable_sharder_ir +# yaml_extra: ['dashboard_default.yaml', 'world_size_8.yaml', 'num_hidden_layers_5.yaml', 'enable_sharder_ir.yaml'] From cd38dfb2e101f60b45b9fa7e375b6ab50c0ee262 Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:00:33 -0700 Subject: [PATCH 194/308] [https://nvbugs/6226933][fix] canonicalize multimodal cache-key serialization to prevent hash collisions (#14800) Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> --- tensorrt_llm/inputs/multimodal.py | 18 +- tensorrt_llm/inputs/multimodal_data.py | 118 +++++++--- .../llmapi/test_llm_kv_cache_events.py | 203 ++++++++++++++++-- 3 files changed, 285 insertions(+), 54 deletions(-) diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index 09e83581fab1..f19c0af72007 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -18,6 +18,10 @@ default_hasher = blake3 _INT32_MAX = 2**31 - 1 +# Versioned tag prefixed to every content hash so the canonical, self-describing +# serialization scheme can evolve without silently reusing stale cache keys. +_HASH_SCHEME_TAG = b"trtllm.mm.hash.v1" + def strip_mm_data_for_generation(mm_data: Dict[str, Any]) -> None: """Clear `mm_data` in place, retaining only `mrope_config.mrope_position_deltas`. @@ -666,21 +670,10 @@ class MultimodalServerConfig(): def _update_hash(hasher, item: object) -> None: """Hash the content of a multimodal item into the provided hasher.""" + hasher.update(_HASH_SCHEME_TAG) if isinstance(item, BaseModalityData): item.update_hash(hasher) return - if isinstance(item, torch.Tensor): - item = item.detach().cpu().contiguous() - hasher.update(serialize_item(item)) - return - if isinstance(item, list): - for element in item: - hasher.update(b"") - if isinstance(element, torch.Tensor): - element = element.detach().cpu().contiguous() - hasher.update(serialize_item(element)) - return - hasher.update(serialize_item(item)) @@ -711,7 +704,6 @@ def apply_mm_hashes( def _hash_item(item): """Hash only the content of a multimodal item (no UUID).""" - # TODO: possible hash collision w/ this simplified version (vllm/PR/17378) hasher = hash_lib() _update_hash(hasher, item) return hasher.hexdigest() diff --git a/tensorrt_llm/inputs/multimodal_data.py b/tensorrt_llm/inputs/multimodal_data.py index 6cee4ff199be..4c8e468e1f00 100644 --- a/tensorrt_llm/inputs/multimodal_data.py +++ b/tensorrt_llm/inputs/multimodal_data.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import struct from dataclasses import dataclass from typing import Any, Protocol @@ -8,6 +9,15 @@ import torch from PIL import Image +# Video metadata fields that participate in the cache-key hash. These describe +# how frames were sampled and therefore change the model-visible content. +_VIDEO_HASH_METADATA_FIELDS = ( + "frames_indices", + "fps", + "duration", + "total_num_frames", +) + class ContentHasher(Protocol): """Hash object that accepts bytes.""" @@ -16,30 +26,93 @@ def update(self, data: bytes) -> None: """Update the hash with raw bytes.""" +def _u8(value: int) -> bytes: + """Encode an unsigned 8-bit integer.""" + return value.to_bytes(1, "big", signed=False) + + +def _u32(value: int) -> bytes: + """Encode an unsigned 32-bit big-endian integer.""" + return value.to_bytes(4, "big", signed=False) + + +def _u64(value: int) -> bytes: + """Encode an unsigned 64-bit big-endian integer.""" + return value.to_bytes(8, "big", signed=False) + + +def _len_prefixed(payload: bytes) -> bytes: + """Encode a byte payload prefixed with its u64 length.""" + return _u64(len(payload)) + payload + + def serialize_item(obj: object) -> bytes: - """Serialize a supported multimodal hash leaf to bytes.""" + """Serialize a supported multimodal hash leaf to bytes. + + The encoding is canonical and self-describing: every value is + `[1-byte type tag][typed metadata][length-prefixed payload]` with all + multi-byte integers big-endian. This prevents cache-key hash collisions + between distinct values that happen to share a raw byte payload (for + example transposed image dimensions or reshaped arrays). + """ if isinstance(obj, str): - return obj.encode("utf-8") + return _u8(0x01) + _len_prefixed(obj.encode("utf-8")) if isinstance(obj, bytes): - return obj - if isinstance(obj, (int, float)): - return np.array(obj).tobytes() + return _u8(0x02) + _len_prefixed(obj) + # bool must be checked before int: bool is a subclass of int. + if isinstance(obj, bool): + return _u8(0x05) + _u8(1 if obj else 0) + if isinstance(obj, int): + nbytes = (obj.bit_length() + 8) // 8 # +1 sign bit, then ceil-divide. + return _u8(0x03) + _u8(nbytes) + obj.to_bytes(nbytes, "big", signed=True) + if isinstance(obj, float): + return _u8(0x04) + struct.pack(">d", obj) if isinstance(obj, Image.Image): - return np.array(obj.convert("RGBA")).tobytes() - if isinstance(obj, torch.Tensor): - return obj.numpy().tobytes() - if isinstance(obj, np.ndarray): - return obj.tobytes() + width, height = obj.size + payload = np.array(obj.convert("RGBA")).tobytes() + return ( + _u8(0x10) + + _len_prefixed(obj.mode.encode("utf-8")) + + _u32(width) + + _u32(height) + + _len_prefixed(payload) + ) + if isinstance(obj, (torch.Tensor, np.ndarray)): + # The container (torch.Tensor vs np.ndarray) is not part of the content + # identity -- only dtype, shape, and raw bytes are. Normalize both to a + # contiguous NumPy array so identical content hashes identically. + if isinstance(obj, torch.Tensor): + obj = obj.detach().cpu().contiguous().numpy() + array = np.ascontiguousarray(obj) + parts = [ + _u8(0x11), + _len_prefixed(array.dtype.str.encode("utf-8")), + _u8(array.ndim), + ] + parts.extend(_u64(dim) for dim in array.shape) + parts.append(_len_prefixed(array.tobytes())) + return b"".join(parts) if isinstance(obj, (tuple, list)): - container_tag = b"T" if isinstance(obj, tuple) else b"L" - parts = [container_tag, len(obj).to_bytes(8, "big", signed=False)] - for item in obj: - payload = serialize_item(item) - parts.append(len(payload).to_bytes(8, "big", signed=False)) - parts.append(payload) + # Ordered sequence; the container (tuple vs list) is not part of the + # content identity. + parts = [_u8(0x20), _u64(len(obj))] + parts.extend(serialize_item(item) for item in obj) + return b"".join(parts) + if isinstance(obj, dict): + parts = [_u8(0x22), _u64(len(obj))] + for key in sorted(obj): + parts.append(serialize_item(key)) + parts.append(serialize_item(obj[key])) return b"".join(parts) + if isinstance(obj, np.generic): + # numpy scalar (e.g. np.int64 / np.float32 / np.bool_): normalize to the + # equivalent Python scalar and recurse, so numpy-typed values hash + # identically to their Python counterparts. In numpy 2.x these are not + # subclasses of Python int/float/bool, so they bypass the checks above. + return serialize_item(obj.item()) + raise ValueError(f"Unsupported object type: {type(obj)}") @@ -65,11 +138,8 @@ def __post_init__(self) -> None: self.sample_rate = int(self.sample_rate) def update_hash(self, hasher: ContentHasher) -> None: - samples = self.samples - if isinstance(samples, torch.Tensor): - samples = samples.detach().cpu().contiguous() hasher.update(b"