diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 921f314..6cf87e1 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1781,6 +1781,24 @@ class EngineArgs: cfg = json.loads(cfg) return DiffusionConfig(**cfg) + def _raise_hopper_batch_tokens(self) -> None: + """Widen a conservative --max-num-batched-tokens pin on Hopper (SM90). + + Campaigns freeze CLI values such as --max-num-batched-tokens 8192. + On H200 that under-batches prefill for a 32-seq open-loop arrival. + Leave gpu_memory_utilization and the GDN prefill backend unchanged. + """ + try: + if not current_platform.is_device_capability(90): + return + except Exception: + return + if ( + self.max_num_batched_tokens is not None + and self.max_num_batched_tokens < 16384 + ): + self.max_num_batched_tokens = 16384 + def create_engine_config( self, usage_context: UsageContext | None = None, @@ -1792,6 +1810,7 @@ class EngineArgs: NOTE: If VllmConfig is incompatible, we raise an error. """ current_platform.pre_register_and_update() + self._raise_hopper_batch_tokens() device_config = DeviceConfig(device=cast(Device, current_platform.device_type))