Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 71 additions & 1 deletion python/tvm/relax/frontend/nn/llm/_kernel_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def _alloc_tile_walk_state():
"""Return (tile_id, batch_idx, batch_tiles, batch_rows, iterator, kv_chunk_len) int32 scalars for the paged/ragged/MLA tile-walk state machine."""
return _var("int32"), _var("int32"), _var("int32"), _var("int32"), _var("int32"), _var("int32")

def _make_prefill_macros(tile_x, tile_y, tile_z, tile_o, bdx, num_warps, group_size):
def _make_prefill_macros(tile_x, tile_y, tile_z, tile_o, bdx, num_warps, group_size, softmax_simd_width=0):
"""Build @T.macro helpers shared across tiled online-softmax prefill kernels.

Parameters
Expand All @@ -205,6 +205,7 @@ def _make_prefill_macros(tile_x, tile_y, tile_z, tile_o, bdx, num_warps, group_s
tile_y : int # QK reduction dim (head_dim for MHA, d_qk for MLA/ragged)
tile_z : int # key/value column tile
tile_o : int # output/V column dim (d for MHA/sequence, d_v for ragged, d_latent for MLA)
softmax_simd_width : int # if > 0, split softmax_update_causal rows across lanes of SIMD groups this wide
"""
@T.macro
def init_states(
Expand Down Expand Up @@ -282,6 +283,66 @@ def softmax_update_causal(
m_prev_smem[row] = m_prev[i]
T.gpu_storage_sync("shared")

lanes = bdx * num_warps // tile_x
use_lanes = (
bdx * num_warps == tile_x * lanes
and 1 < lanes <= softmax_simd_width # a row's lanes must share one SIMD group
and bdx % lanes == 0
and lanes & (lanes - 1) == 0
and tile_z % lanes == 0
)
shuffle_steps = lanes.bit_length() - 1

@T.macro
def shuffle_lane_pair(shuffled: T.Tensor, value: T.float32, offset: T.int32):
# Relative shuffles: a subgroup may span several threadIdx.y rows, so tx is not its lane id.
shuffled[0] = T.gpu_warp_shuffle_up(T.gpu_warp_activemask(), value, offset, softmax_simd_width, softmax_simd_width)
shuffled[1] = T.gpu_warp_shuffle_down(T.gpu_warp_activemask(), value, offset, softmax_simd_width, softmax_simd_width)

@T.macro
def softmax_update_causal_lanes(
S_smem: T.Tensor, m_smem: T.Tensor, d_smem: T.Tensor, m_prev_smem: T.Tensor,
m_new: T.Tensor, m_prev: T.Tensor, d_new: T.Tensor,
ty: T.int32, tx: T.int32, LH_start: T.int32, L_kv_start: T.int32,
causal: T.int32, kv_len: T.int32, qo_len: T.int32, sliding_window_size: T.int32,
):
row: T.let[T.int32] = (ty * bdx + tx) // lanes
lane: T.let[T.int32] = tx % lanes
row_: T.let[T.int32] = (LH_start + row) // group_size
with Ts.sblock("update"):
S_lane = Ts.sblock_alloc_tensor((tile_z // lanes,), "float32", scope="local")
# Values from lane +- offset, read by every lane before picking lane ^ offset.
shuffled = Ts.sblock_alloc_tensor((2,), "float32", scope="local")
m_prev[0] = m_smem[row]
m_new[0] = m_smem[row]
for jj in T.serial(tile_z // lanes):
S_lane[jj] = T.if_then_else(
_causal_or_sliding_cross_mask(causal, row=row_, col=L_kv_start + jj * lanes + lane, kv_len=kv_len, qo_len=qo_len, sliding_window_size=sliding_window_size),
S_smem[row, jj * lanes + lane],
-5e4,
)
m_new[0] = T.max(m_new[0], S_lane[jj])
for step in T.unroll(shuffle_steps):
shuffle_lane_pair(shuffled, m_new[0], (lanes // 2) >> step)
m_new[0] = T.max(m_new[0], T.if_then_else(T.bitwise_and(tx, (lanes // 2) >> step) != 0, shuffled[0], shuffled[1]))
d_new[0] = 0.0
for jj in T.serial(tile_z // lanes):
S_lane[jj] = T.exp2(S_lane[jj] - m_new[0])
d_new[0] += S_lane[jj]
for jj in T.serial(tile_z // lanes):
S_smem[row, jj * lanes + lane] = S_lane[jj]
for step in T.unroll(shuffle_steps):
shuffle_lane_pair(shuffled, d_new[0], (lanes // 2) >> step)
d_new[0] += T.if_then_else(T.bitwise_and(tx, (lanes // 2) >> step) != 0, shuffled[0], shuffled[1])
if lane == 0:
m_smem[row] = m_new[0]
d_smem[row] = d_smem[row] * T.exp2(m_prev[0] - m_new[0]) + d_new[0]
m_prev_smem[row] = m_prev[0]
T.gpu_storage_sync("shared")

if use_lanes:
softmax_update_causal = softmax_update_causal_lanes

@T.macro
def compute_o_gemm(
S_smem: T.Tensor, V_smem: T.Tensor, O_local: T.Tensor,
Expand Down Expand Up @@ -602,6 +663,15 @@ def _fit_prefill_config_to_shared_memory(
f"{max_shared_memory_per_block} bytes"
)

def _prefill_softmax_simd_width(target: Target) -> int:
"""SIMD width for splitting prefill softmax rows across lanes, or 0 to keep one thread per row.

Only Metal enables it; other targets keep one thread per row.
"""
if target.kind.name == "metal":
return int(target.attrs["thread_warp_size"])
return 0

def _get_prefill_kernel_config(
h_kv, h_q, d, dtype, target: Target, *, d_v=None, merged_kv=False
):
Expand Down
5 changes: 3 additions & 2 deletions python/tvm/relax/frontend/nn/llm/_prefill_kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
_get_seq_offset,
_length_info_buffer,
_make_prefill_macros,
_prefill_softmax_simd_width,
_rope,
_schedule_prefill_kernel,
)
Expand Down Expand Up @@ -228,7 +229,7 @@ def _attention_prefill(
if sliding_window:
global_symbol += "_sliding_window"

init_states, compute_s_gemm, softmax_update_causal, compute_o_gemm, _, advance_tile_batch, paged_store_output_lse, *_ = _make_prefill_macros(tile_x, tile_y, tile_z, tile_y, bdx, num_warps, group_size)
init_states, compute_s_gemm, softmax_update_causal, compute_o_gemm, _, advance_tile_batch, paged_store_output_lse, *_ = _make_prefill_macros(tile_x, tile_y, tile_z, tile_y, bdx, num_warps, group_size, softmax_simd_width=_prefill_softmax_simd_width(target))

# pylint: disable=too-many-branches
batch_size = T.dynamic("batch_size", "int32")
Expand Down Expand Up @@ -759,7 +760,7 @@ def _attention_prefill_ragged(h_kv, h_q, d_qk, d_v, dtype, rope_scaling: dict[st
if target.kind.name == "webgpu" and dtype == "float32" and d_qk == d_v == 512:
return _attention_prefill_ragged_parallel(h_kv, h_q, rope_scaling)
NUM_BLKS, LOAD_VEC, group_size, bdx, num_warps, tile_x, tile_y, tile_z = _get_prefill_kernel_config(h_kv, h_q, d_qk, dtype, target, d_v=d_v)
init_states, compute_s_gemm, softmax_update_causal, compute_o_gemm, _, advance_tile_batch, paged_store_output_lse, *_ = _make_prefill_macros(tile_x, tile_y, tile_z, d_v, bdx, num_warps, group_size)
init_states, compute_s_gemm, softmax_update_causal, compute_o_gemm, _, advance_tile_batch, paged_store_output_lse, *_ = _make_prefill_macros(tile_x, tile_y, tile_z, d_v, bdx, num_warps, group_size, softmax_simd_width=_prefill_softmax_simd_width(target))

batch_size = T.dynamic("batch_size", "int32")
qo_len = T.dynamic("qo_len", "int32")
Expand Down
143 changes: 143 additions & 0 deletions tests/python/relax/test_nn_llm_prefill_simd_softmax.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Metal prefill kernels split each softmax row across SIMD lanes."""

import math

import numpy as np
import pytest
import tvm_ffi

import tvm
import tvm_tirx
import tvm_tirx.testing
from tvm.relax.frontend.nn.llm._prefill_kernels import _attention_prefill, _attention_prefill_ragged
from tvm.testing import device_enabled

METAL = {
"kind": "metal",
"max_shared_memory_per_block": 32768,
"max_threads_per_block": 1024,
"thread_warp_size": 32,
}


@pytest.mark.parametrize("dtype", ["float16", "float32"])
def test_simd_softmax_only_on_metal(dtype):
for kind, uses_shuffle in (
(METAL, True),
({"kind": "cuda", "arch": "sm_80"}, False),
("vulkan", False),
):
target = tvm_tirx.target.Target(kind)
for func in (
_attention_prefill(2, 8, 64, dtype, False, {}, target),
_attention_prefill_ragged(2, 8, 64, 64, dtype, {}, target),
):
assert ("gpu_warp_shuffle" in func.script()) == uses_shuffle, (kind, dtype)


def _reference(q, k, v, q_lens, kv_lens, causal, scale, group):
"""Base-2 attention over per-sequence (q, k, v) slices, as the kernels compute it."""
out = np.zeros((q.shape[0], q.shape[1], v.shape[2]))
lse = np.zeros(q.shape[:2])
q_off = kv_off = 0
for n, m in zip(q_lens, kv_lens):
for head in range(q.shape[1]):
scores = (
q[q_off : q_off + n, head].astype("float64")
@ k[kv_off : kv_off + m, head // group].astype("float64").T
)
scores *= scale / np.log(2)
if causal:
# Query row r sees keys up to r + m - n, the last n keys being its own tokens.
scores[np.arange(m)[None, :] > np.arange(n)[:, None] + m - n] = -np.inf
peak = scores.max(axis=1, keepdims=True)
w = np.exp2(scores - peak)
out[q_off : q_off + n, head] = (
w @ v[kv_off : kv_off + m, head // group] / w.sum(axis=1, keepdims=True)
)
lse[q_off : q_off + n, head] = peak[:, 0] + np.log2(w.sum(axis=1))
q_off += n
kv_off += m
return out, lse


@pytest.mark.gpu
@pytest.mark.parametrize("kernel", ["ragged", "paged"])
@pytest.mark.parametrize("causal", [0, 1])
@pytest.mark.parametrize("dtype", ["float16", "float32"])
def test_prefill_runtime_metal(kernel, causal, dtype):
if not device_enabled("metal"):
pytest.skip("metal not enabled")
device = tvm_ffi.device("metal")
target = tvm_tirx.target.Target.from_device(device)
h_kv, h_q, d, page_size = 2, 8, 64, 16
# Several query and KV tiles, a partial last tile, and a one-token sequence.
q_lens = [1, 37, 129]
kv_lens = q_lens if kernel == "ragged" else [40, 37, 200]
rng = np.random.default_rng(3)
q = rng.normal(0, 0.5, (sum(q_lens), h_q, d)).astype(dtype)
k = rng.normal(0, 0.5, (sum(kv_lens), h_kv, d)).astype(dtype)
v = rng.normal(size=(sum(kv_lens), h_kv, d)).astype(dtype)
scale = d**-0.5
ref, ref_lse = _reference(q, k, v, q_lens, kv_lens, causal, scale, h_q // h_kv)
q_indptr = np.array([0, *np.cumsum(q_lens)], dtype="int32")
zeros = np.zeros(len(q_lens), dtype="int32")
if kernel == "ragged":
func = _attention_prefill_ragged(h_kv, h_q, d, d, dtype, {}, target)
inputs = [q, q_indptr, k, v, q_indptr, np.arange(q_indptr[-1], dtype="int32"), zeros]
else:
func = _attention_prefill(h_kv, h_q, d, dtype, False, {}, target, page_size=page_size)
num_pages = [math.ceil(m / page_size) for m in kv_lens]
pages = np.zeros((sum(num_pages), 2, h_kv, page_size, d), dtype=dtype)
page, kv_off = 0, 0
for m, p in zip(kv_lens, num_pages):
for t in range(m):
pages[page + t // page_size, 0, :, t % page_size] = k[kv_off + t]
pages[page + t // page_size, 1, :, t % page_size] = v[kv_off + t]
page, kv_off = page + p, kv_off + m
last_page_len = np.array(
[m - (p - 1) * page_size for m, p in zip(kv_lens, num_pages)], dtype="int32"
)
page_indptr = np.array([0, *np.cumsum(num_pages)], dtype="int32")
inputs = [
q,
q_indptr,
pages,
page_indptr,
np.arange(page_indptr[-1], dtype="int32"),
last_page_len,
zeros,
np.zeros(q_indptr[-1], dtype="int32"),
]
module = tvm.compile(func.with_attr("global_symbol", "main"), target=target)

def run_and_check():
args = [tvm_tirx.runtime.tensor(a, device) for a in inputs]
output = tvm_tirx.runtime.tensor(np.full(ref.shape, np.nan, dtype=dtype), device)
lse = tvm_tirx.runtime.tensor(np.full(ref_lse.shape, np.nan, dtype="float32"), device)
module(*args, output, lse, causal, 0, 1.0, 10000.0, scale)
rtol, atol = (2e-3, 2e-3) if dtype == "float16" else (2e-4, 2e-5)
np.testing.assert_allclose(output.numpy(), ref, rtol=rtol, atol=atol)
np.testing.assert_allclose(lse.numpy(), ref_lse, rtol=0, atol=1e-4)

tvm_tirx.testing.run_with_gpu_lock(run_and_check)


if __name__ == "__main__":
tvm_tirx.testing.main()
Loading