diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 921f314..d350eb5 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -2277,6 +2277,15 @@ class EngineArgs: jit_monitor_verbose=self.jit_monitor_verbose, ) + # Blockwise-FP8 decode on Hopper picks the CUTLASS kernel, which is + # tuned for throughput shapes. A decode-bound server runs this GEMM at + # M equal to the running batch (single digits), where the Triton + # blockwise kernel has per-shape tuned configs. Numerics are unchanged: + # same fp8 weights, same 128x128 block scales. + if self.kernel_config.linear_backend == "auto": + self.kernel_config.linear_backend = "triton" + logger.info("Using the Triton blockwise-FP8 linear kernel.") + # Compilation config overrides compilation_config = copy.deepcopy(self.compilation_config) if self.cudagraph_capture_sizes is not None: