Skip to content

Commit 7cd4fcb

Browse files
committed
[Bwd,Sm90] Enable local
1 parent 921f70c commit 7cd4fcb

4 files changed

Lines changed: 31 additions & 17 deletions

File tree

‎flash_attn/cute/flash_bwd_sm90.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ def __init__(
5252
head_dim_v: Optional[int] = None,
5353
qhead_per_kvhead: int = 1,
5454
is_causal: bool = False,
55+
is_local: bool = False,
5556
deterministic: bool = False,
5657
tile_m: int = 64,
5758
tile_n: int = 128,
@@ -84,7 +85,7 @@ def __init__(
8485
self.check_hdim_v_oob = head_dim_v != self.tile_hdimv
8586
self.qhead_per_kvhead = qhead_per_kvhead
8687
self.is_causal = is_causal
87-
self.is_local = False
88+
self.is_local = is_local
8889
self.deterministic = deterministic
8990
self.tile_m = tile_m
9091
self.tile_n = tile_n
@@ -496,6 +497,11 @@ def _qkv_transpose(t):
496497

497498
self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None)
498499

500+
if const_expr(window_size_left is not None):
501+
window_size_left = Int32(window_size_left)
502+
if const_expr(window_size_right is not None):
503+
window_size_right = Int32(window_size_right)
504+
499505
self.kernel(
500506
tma_tensor_Q,
501507
tma_tensor_K,
@@ -537,6 +543,8 @@ def _qkv_transpose(t):
537543
blocksparse_tensors,
538544
qhead_per_kvhead_divmod,
539545
mdQ_semaphore,
546+
window_size_left,
547+
window_size_right,
540548
).launch(
541549
grid=grid_dim,
542550
block=[self.num_threads, 1, 1],
@@ -587,6 +595,8 @@ def kernel(
587595
blocksparse_tensors: Optional[BlockSparseTensors] = None,
588596
qhead_per_kvhead_divmod: Optional[FastDivmodDivisor] = None,
589597
mdQ_semaphore: Optional[cute.Tensor] = None,
598+
window_size_left: Optional[Int32] = None,
599+
window_size_right: Optional[Int32] = None,
590600
):
591601
warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx())
592602

@@ -649,8 +659,8 @@ def kernel(
649659
self.is_causal,
650660
self.is_local,
651661
False, # is_split_kv
652-
None,
653-
None,
662+
window_size_left,
663+
window_size_right,
654664
qhead_per_kvhead_packgqa=1,
655665
)
656666
SeqlenInfoCls = partial(
@@ -668,8 +678,8 @@ def kernel(
668678
AttentionMask,
669679
self.tile_m,
670680
self.tile_n,
671-
window_size_left=None,
672-
window_size_right=None,
681+
window_size_left=window_size_left,
682+
window_size_right=window_size_right,
673683
swap_AB=self.SdP_swapAB,
674684
)
675685
TileSchedulerCls = partial(TileScheduler.create, tile_sched_params)
@@ -1276,8 +1286,8 @@ def mma(
12761286
qhead_per_kvhead_divmod,
12771287
)
12781288
else:
1279-
# Block sparsity: KV tile with zero Q blocks produces no dK/dV; write zeros.
1280-
if const_expr(self.use_block_sparsity):
1289+
# KV tile with zero Q blocks produces no dK/dV; write zeros.
1290+
if const_expr(self.use_block_sparsity or self.is_local or self.is_varlen_q):
12811291
acc_dK.fill(0.0)
12821292
acc_dV.fill(0.0)
12831293
self.epilogue_dKV(

‎flash_attn/cute/interface.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -915,7 +915,6 @@ def _flash_attn_bwd(
915915
AtomLayoutMdQ = 1
916916
cluster_size = 1
917917
use_2cta_instrs = False
918-
assert window_size_left is None and window_size_right is None, "local not supported yet on 9.x"
919918
is_varlen = (
920919
cu_seqlens_q is not None
921920
or cu_seqlens_k is not None
@@ -1185,6 +1184,8 @@ def _flash_attn_bwd(
11851184
head_dim_v,
11861185
qhead_per_kvhead,
11871186
causal,
1187+
window_size_left is not None,
1188+
window_size_right is not None,
11881189
softcap != 0.0,
11891190
m_block_size,
11901191
n_block_size,
@@ -1297,6 +1298,7 @@ def _flash_attn_bwd(
12971298
head_dim_v,
12981299
qhead_per_kvhead,
12991300
causal,
1301+
is_local=local,
13001302
deterministic=deterministic,
13011303
tile_m=m_block_size,
13021304
tile_n=n_block_size,

‎flash_attn/cute/mask.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -346,11 +346,18 @@ def mask_gen_fn(s: int) -> Uint32:
346346
# column, by setting row limit to be self.tile_m.
347347
row_limit_top = (
348348
self.tile_m
349-
if col0 >= seqlenk_col_limit
350-
else col0 - causal_row_offset - self.window_size_right
349+
if col0 >= seqlenk_col_limit and mask_seqlen
350+
else (
351+
col0 - causal_row_offset - self.window_size_right
352+
if const_expr(self.window_size_right is not None)
353+
else 0
354+
)
355+
)
356+
row_limit_bot = (
357+
col0 - causal_row_offset + self.window_size_left
358+
if const_expr(self.window_size_left is not None)
359+
else self.tile_m
351360
)
352-
# TODO: do we need col_limit_sink?
353-
row_limit_bot = col0 - causal_row_offset + self.window_size_left
354361
for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):
355362
row_idx = t0ScS_mn[r, 0][ROW]
356363
acc_S_mn[r, c] = (

‎tests/cute/test_flash_attn.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -299,9 +299,6 @@ def test_flash_attn_output(
299299
# Fix requires adjusting m_block_size or MMA config in flash_bwd_sm90.py
300300
if IS_SM90 and d == 64 and not causal:
301301
pytest.xfail("SM90 backward: d=64 + non-causal has invalid MMA tile config (m_block=80)")
302-
# TODO: SM90 backward pass does not support local attention yet
303-
if IS_SM90 and local:
304-
pytest.xfail("SM90 backward: local attention not supported yet")
305302
if d == 192 and local:
306303
pytest.xfail("hdim 192 backward: local attention not supported yet")
307304
g = torch.randn_like(out)
@@ -740,8 +737,6 @@ def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device):
740737
):
741738
if d == 192 and local:
742739
pytest.xfail("hdim 192 backward: local attention not supported yet")
743-
if IS_SM90 and local:
744-
pytest.xfail("SM90 backward: local attention not supported yet")
745740
g_unpad = torch.randn_like(out_unpad)
746741
# do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2)
747742
# import flash_attn_3_cuda

0 commit comments

Comments
 (0)