Skip to content

Commit ed5e2fe

Browse files
authored
[FIX][TIRx][CUDA] Support SM100 weight-stationary B collectors (#20329)
The SM100 `tcgen05.mma.ws` SS/TS entries omit the collector-B modifier slot, preventing callers from expressing weight-stationary B reuse through the PTX namespace. Add the existing optional B0–B3 fill/use/lastuse/discard domain to both entries, preserving calls that omit the qualifier. Also reject modifier-domain and operand-count mismatches during code generation, before stale table/codegen layouts can emit malformed CUDA helpers. Add dispatch and parser round-trip regressions, SM100 certification for all WS variants, and stale-layout diagnostics. Make the test source helper accept an explicit target so the existing SM107 collector test compiles at its required architecture. Validation with CUDA 13.4 NVCC/NVRTC and a matching worktree FFI: - PTX dialect, conversion, and address suites: 179 non-skipped cases passed across the full run and targeted reruns. - Full-table NVCC/ptxas certification: **32/32 shards passed**, including predicated helpers and address-offset samples at their declared architectures. - Regenerated IDE stubs: no diff needed; the tokens already exist in the ti16 families. - Changed-file pre-commit hooks and `git diff --check` passed. Closes #20328.
1 parent 7fca2e1 commit ed5e2fe

3 files changed

Lines changed: 95 additions & 5 deletions

File tree

python/tvm/backend/cuda/ptx/engine.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,12 @@ def codegen(*args):
186186
preserve_dst = "keep" in flags
187187
tokens = [parse_str(a) for a in args[len(args) - n_slots - 1 : -1]]
188188
rest = list(args[: len(args) - n_slots - 1]) # operands, plus pred when present
189+
for slot, token in zip(entry.slots, tokens, strict=True):
190+
if token not in slot.choices and not (slot.optional and token == ""):
191+
raise ValueError(
192+
f"{entry.name}: invalid codegen modifier {token!r} for slot {slot.name!r}; "
193+
"the PTX call and registered table entry may be inconsistent"
194+
)
189195
mod_map = mods(entry, tokens)
190196
layout = operand_layout(entry, mod_map)
191197
n_operands = sum(n for _, _, n in layout)
@@ -199,6 +205,12 @@ def codegen(*args):
199205
at[i] = pos
200206
pos += 1
201207
n_present = pos
208+
expected_args = n_present + int(predicated)
209+
if len(rest) != expected_args:
210+
raise ValueError(
211+
f"{entry.name}: expected {expected_args} codegen operand(s), got {len(rest)}; "
212+
"the PTX call and registered table entry may be inconsistent"
213+
)
202214
sinks = frozenset(
203215
(slot.name, lane)
204216
for slot, i, lanes in layout

python/tvm/backend/cuda/ptx/table.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11529,6 +11529,7 @@ def _check_set_packed(m):
1152911529
)
1153011530
for form in ("ss", "ts")
1153111531
],
11532+
# PTX ISA 9.7.18.10.10.3: optional collector::b{0,1,2,3}::{fill,use,lastuse,discard}.
1153211533
*[
1153311534
InstructionEntry( # weight-stationary: no mask vector, a zero-column desc
1153411535
name=f"tcgen05_mma_ws_{form}",
@@ -11538,6 +11539,7 @@ def _check_set_packed(m):
1153811539
ModifierSlot("ws", ("ws",)),
1153911540
ModifierSlot("cta_group", ("cta_group::1",)),
1154011541
ModifierSlot("kind", ("kind::f16", "kind::tf32", "kind::f8f6f4", "kind::i8")),
11542+
ModifierSlot("collector_b", _TCGEN05_WS_COLLECTOR_B, optional=True),
1154111543
),
1154211544
cert_arch="sm_100a",
1154311545
operands=(

tests/python/tirx/codegen/test_ptx_dialect.py

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@
3434
requires_nvcc = pytest.mark.skipif(shutil.which("nvcc") is None, reason="nvcc not available")
3535

3636

37-
def _cuda_source(func) -> str:
38-
with TARGET:
39-
mod = tvm.compile(tvm.IRModule({"main": func}), target=TARGET, tir_pipeline="tirx")
37+
def _cuda_source(func, target=TARGET) -> str:
38+
with target:
39+
mod = tvm.compile(tvm.IRModule({"main": func}), target=target, tir_pipeline="tirx")
4040
return mod.mod.imports[0].inspect_source("cuda")
4141

4242

@@ -2340,6 +2340,80 @@ def test_ptx_wgmma_integer_shape_domains_follow_concrete_syntax():
23402340
assert "m64n256k256" in b1_shapes
23412341

23422342

2343+
@pytest.mark.parametrize("form", ("ss", "ts"))
2344+
@pytest.mark.parametrize("collector", ("", "b0::fill", "b1::use", "b2::lastuse", "b3::discard"))
2345+
def test_ptx_tcgen05_mma_ws_collector_dispatch(form, collector):
2346+
opcode = "tcgen05.mma.ws.cta_group::1.kind::f16"
2347+
if collector:
2348+
opcode += f".collector::{collector}"
2349+
a_dtype = "uint64" if form == "ss" else "uint32"
2350+
2351+
@T.prim_func
2352+
def kernel():
2353+
T.device_entry()
2354+
T.cta_id([1])
2355+
T.thread_id([32])
2356+
T.ptx[opcode](
2357+
T.uint32(0),
2358+
T.cast(0, a_dtype),
2359+
T.uint64(0),
2360+
T.uint32(0),
2361+
T.ptx.pred(T.uint32(0)),
2362+
T.uint64(0),
2363+
)
2364+
2365+
src = _cuda_source(kernel, tvm.target.Target({"kind": "cuda", "arch": "sm_100a"}))
2366+
a_operand = "%1" if form == "ss" else "[%1]"
2367+
assert f"{opcode} [%0], {a_operand}, %2, %3, ps0, %5;" in src
2368+
tvm.ir.assert_structural_equal(kernel, tvm.script.from_source(kernel.script()))
2369+
2370+
2371+
@requires_nvcc
2372+
def test_ptx_tcgen05_mma_ws_collectors_certify_sm100a():
2373+
"""All WS kinds, B buffers/operations, and predication assemble on SM100a."""
2374+
from tvm.backend.cuda.ptx.render import render_variant
2375+
from tvm.backend.cuda.ptx.table import TABLE, renderings
2376+
2377+
by_arch = {}
2378+
for form in ("ss", "ts"):
2379+
entry = TABLE[f"tcgen05_mma_ws_{form}"]
2380+
for rendering in renderings(entry):
2381+
_, helper, source = render_variant(entry, *_as_render_args(rendering))
2382+
_append_certification(by_arch, "sm_100a", helper, source)
2383+
_assert_certifications_ok(by_arch)
2384+
2385+
2386+
@pytest.mark.parametrize("mismatch", ("modifier", "operand"))
2387+
def test_ptx_codegen_rejects_stale_table_layout(mismatch):
2388+
"""A traced call must not silently feed modifier strings to an old helper."""
2389+
from dataclasses import replace
2390+
2391+
from tvm.backend.cuda.ptx.engine import PTXNamespace, _make_codegen
2392+
from tvm.backend.cuda.ptx.table import TABLE, ModifierSlot, OperandSlot
2393+
2394+
entry = TABLE["tcgen05_mma_ws_ss"]
2395+
if mismatch == "modifier":
2396+
changed = replace(entry, slots=(*entry.slots, ModifierSlot("extra", ("extra",))))
2397+
opcode = "tcgen05.mma.ws.cta_group::1.kind::f16.extra"
2398+
extra = ()
2399+
else:
2400+
changed = replace(entry, operands=(*entry.operands, OperandSlot("extra", dtype="u64")))
2401+
opcode = "tcgen05.mma.ws.cta_group::1.kind::f16"
2402+
extra = (T.uint64(0),)
2403+
namespace = PTXNamespace({changed.name: changed})
2404+
call = namespace[opcode](
2405+
T.uint32(0),
2406+
T.uint64(0),
2407+
T.uint64(0),
2408+
T.uint32(0),
2409+
namespace.pred(T.uint32(0)),
2410+
T.uint64(0),
2411+
*extra,
2412+
)
2413+
with pytest.raises(ValueError, match="PTX call and registered table entry may be inconsistent"):
2414+
_make_codegen(entry)(*call.args)
2415+
2416+
23432417
@requires_nvcc
23442418
def test_ptx_tcgen05_mma_block_size_form():
23452419
@T.prim_func
@@ -2405,7 +2479,9 @@ def sm107_collector_kernel(a_ptr: T.handle):
24052479
](tmem, tmem, desc, idesc, tmem, tmem, T.ptx.pred(flag))
24062480
A[tx] = A[tx]
24072481

2408-
collector_src = _cuda_source(sm107_collector_kernel)
2482+
collector_src = _cuda_source(
2483+
sm107_collector_kernel, tvm.target.Target({"kind": "cuda", "arch": "sm_107f"})
2484+
)
24092485
ss_collector_opcode = (
24102486
"tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16"
24112487
".collector::a::discard.collector::b::fill"
@@ -4144,7 +4220,7 @@ def test_ptx_all_variants_render_unique():
41444220
_, helper, _ = render_variant(entry, *args, addr_offsets=addr_offsets)
41454221
assert helper not in names, f"address-offset helper name collision: {helper}"
41464222
names.add(helper)
4147-
assert total == 762050 # update when the table grows or a ptxas gap narrows it
4223+
assert total == 762178 # update when the table grows or a ptxas gap narrows it
41484224

41494225

41504226
def test_ptx_no_instruction_registered_twice():

0 commit comments

Comments
 (0)