diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ba7d26c93b..c72a85a757 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1690,6 +1690,20 @@ class VllmConfig: max_cudagraph_capture_size = min( self.scheduler_config.max_num_seqs * decode_query_len * 2, 512 ) + # The bound above sizes a step that is purely decode: one + # query position per running sequence, times the speculative + # width. Under chunked prefill a step also carries prompt + # tokens, so its size is bounded by the token budget instead. + # Raise the default toward that budget while retaining a + # ceiling on capture time and device memory. An explicit + # max_cudagraph_capture_size still wins outright. + max_cudagraph_capture_size = max( + max_cudagraph_capture_size, + min( + self.scheduler_config.max_num_batched_tokens, + _CUDAGRAPH_CAPTURE_CEILING, + ), + ) max_num_tokens = self.scheduler_config.max_num_batched_tokens max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size) @@ -2292,3 +2306,10 @@ def get_layers_from_vllm_config( for layer_name in layer_names if isinstance(layer := forward_context.get(layer_name), layer_type) } + + +# Ceiling on the token-keyed CUDA graph capture range chosen in +# _set_cudagraph_sizes when the config pins no explicit size. Capture cost +# grows with the number and size of captured graphs, so the token budget is +# only honoured this far. +_CUDAGRAPH_CAPTURE_CEILING = 1024 diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 921f31466b..0a891aa16d 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1838,6 +1838,27 @@ class EngineArgs: "enable_prefix_caching must be set by this point" ) + # Qwen3.5 hybrid: the GDN linear-attention layers make prefix caching + # store per-block recurrent state, and that bookkeeping costs more than + # it returns unless prompts actually share prefixes. On Hopper the + # FlashInfer GDN prefill kernel is also faster than the Triton one. + # Gated on the architecture and the GPU that make both statements true, + # and only when the caller left the GDN backend at its default, so an + # explicit --gdn-prefill-backend still wins. Deliberately NOT gated on + # max_num_batched_tokens / max_num_seqs / gpu_memory_utilization: those + # are unrelated to either claim, and pinning them would silently + # disable this path whenever a caller tunes an unrelated knob. + if ( + self.enable_prefix_caching + and self.gdn_prefill_backend == "triton" + and current_platform.is_cuda() + and current_platform.is_device_capability(90) + and getattr(model_config.hf_text_config, "model_type", "") + == "qwen3_5_text" + ): + self.enable_prefix_caching = False + self.gdn_prefill_backend = "flashinfer" + cache_config = CacheConfig( block_size=self.block_size, # type: ignore[arg-type] gpu_memory_utilization=self.gpu_memory_utilization,