Skip to content

[FIX_FOR_VLLM_LATEST] Upstream vllm fixes for #26355 and #26737 #407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 15, 2025
Merged

[FIX_FOR_VLLM_LATEST] Upstream vllm fixes for #26355 and #26737 #407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7b46dd3
Fix for name change compilation level to compilation mode, deprecati…
iboiko-habana Oct 15, 2025
2858598
Fix for #26737
iboiko-habana Oct 15, 2025
44ca644
Merge branch 'main' into fix26355
iboiko-habana Oct 15, 2025
df27603
Merge branch 'main' into fix26355
michalkuligowski Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions vllm_gaudi/platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,17 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
vllm_config.model_config.dtype = torch.bfloat16

if envs.VLLM_USE_V1:
from vllm.config import CompilationLevel, CUDAGraphMode
from vllm.config import CompilationMode, CUDAGraphMode
compilation_config = vllm_config.compilation_config
# Activate custom ops for v1.
compilation_config.custom_ops = ["all"]
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
compilation_config.cudagraph_capture_sizes = []

if compilation_config.level != CompilationLevel.NO_COMPILATION:
logger.info("[HPU] Forcing CompilationLevel.NO_COMPILATION "
"compilation level")
compilation_config.level = CompilationLevel.NO_COMPILATION
if compilation_config.mode != CompilationMode.NONE:
logger.info("[HPU] Forcing CompilationMode.NONE "
"compilation mode")
compilation_config.mode = CompilationMode.NONE

print(f"========={compilation_config.custom_ops=}===========")

Expand Down
5 changes: 2 additions & 3 deletions vllm_gaudi/v1/worker/hpu_model_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2568,9 +2568,8 @@ def apply_grammar_bitmask(
# Reorder the bitmask to match the order of the requests in the batch.
sorted_bitmask = np.zeros_like(grammar_bitmask, shape=(logits.shape[0], grammar_bitmask.shape[1]))
cumulative_index = 0
seq = sorted(scheduler_output.structured_output_request_ids.items(), key=lambda x: x[1])

for req_id, _ in seq:
for req_id in scheduler_output.structured_output_request_ids:
logit_index = struct_out_req_batch_indices[req_id]
num_spec_tokens = len(scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
for i in range(1 + num_spec_tokens):
Expand Down Expand Up @@ -3199,7 +3198,7 @@ def execute_model(
logits_combined = logits_decode + logits_prompt
logits = torch.cat(logits_combined, dim=0)
# Apply structured output bitmasks if present
if scheduler_output.grammar_bitmask is not None:
if scheduler_output.structured_output_request_ids:
self.apply_grammar_bitmask(scheduler_output, logits)
sampler_output, _sampling_metadata = self._run_sampling(batch_changed, logits,
pd_info.prompt_req_ids + pd_info.decode_req_ids,
Expand Down