# matmul_v2 on h100. the ranked shape runs ltile_kernel; other shapes use coop_kernel / pair_kernelfrom triton.backends.nvidia.driver import TMA_DTYPE_DEVICE_TO_HOSTfrom triton.experimental import gluonfrom triton.experimental.gluon import language as glfrom triton.experimental.gluon.language.nvidia.hopper import (fence_async_shared, mbarrier, tma, warpgroup_mma, warpgroup_mma_wait)from triton.experimental.gluon.nvidia.hopper import TensorDescriptorfrom triton.runtime import driverfrom task import input_t, output_tNUM_SMS = 132 # h100 sxm# smem byte offset of the a ring (the b ring sits at 0). the inline ptx needs raw addressesA_RING = gl.constexpr(131072)# wgmma accumulator layouts: 64-wide pieces and full 256-wide tilesL64 = gl.constexpr(gl.NVMMADistributedLayout(version=[3, 0], warps_per_cta=[4, 1], instr_shape=[16, 64, 16]))L256 = gl.constexpr(gl.NVMMADistributedLayout(version=[3, 0], warps_per_cta=[4, 1], instr_shape=[16, 256, 16]))# one m64n192k16 wgmma. gluon layouts only do power-of-two n, so this one is raw ptxN192_MMA = "wgmma.mma_async.sync.aligned.m64n192k16.f32.f16.f16 {%s}, da, db, p, 1, 1, 0, 1;" % ",".join( f"${i}" for i in range(96))# builds both smem descriptors by hand (128b swizzle + strides), then 4 wgmmas = one 64-wide k-stepN192_ASM = gl.constexpr("\n".join([ "{", ".reg .b32 sb, ta, tb;", ".reg .b64 da, db;", ".reg .pred p;", "mov.u32 sb, global_smem;", "setp.eq.u32 p, sb, sb;", "add.u32 ta, sb, $192;", "add.u32 tb, sb, $224;", "bfe.u32 ta, ta, 4, 14;", "bfe.u32 tb, tb, 4, 14;", "cvt.u64.u32 da, ta;", "cvt.u64.u32 db, tb;", "or.b64 da, da, 0x4000004000000000;", "or.b64 db, db, 0x4000004002000000;", "wgmma.fence.sync.aligned;", "\nadd.s64 da, da, 2;\nadd.s64 db, db, 128;\n".join([N192_MMA] * 4), "wgmma.commit_group.sync.aligned;", "}"]))# 96 fp32 accumulator regs in and out, plus the a / b smem offsetsN192_CON = gl.constexpr(",".join(["=f"] * 96 + [str(i) for i in range(96)] + ["r"] * 64))# tile order: groups of 16 tile-rows. go down one column of the group, then move to the next column@gluon.jitdef tile_coords(tile, tm, tn, GROUP_M: gl.constexpr): size = GROUP_M * tn group = tile // size rows = gl.minimum(tm - group * GROUP_M, GROUP_M) pid_n = tile % size // rows # odd groups run right to left, so the turn starts on b columns that are still in l2 return group * GROUP_M + tile % size % rows, gl.where(group % 2 == 1, tn - 1 - pid_n, pid_n)# last wave: 448-wide windows, each split between two ctas. kind = which warpgroup takes the wide 64x256 half@gluon.jitdef ltile_coords(t, ROWS: gl.constexpr, M0: gl.constexpr, N0: gl.constexpr): u = t // ROWS return M0 + t % ROWS * 128, N0 + u // 2 * 448 + u % 2 * 192, u % 2# split the n dim into equal pieces (64-wide pieces for the tma stores)@gluon.jitdef split_n(x, SPLIT: gl.constexpr): xs = (x, ) for _ in gl.static_range(SPLIT.bit_length() - 1): halves = () for j in gl.static_range(len(xs)): halves += xs[j].reshape(xs[j].shape[0], 2, xs[j].shape[1] // 2).permute(0, 2, 1).split() xs = halves return xs@gluon.jitdef acc_layout(BN: gl.constexpr, num_warps: gl.constexpr): return gl.NVMMADistributedLayout(version=[3, 0], warps_per_cta=[num_warps, 1], instr_shape=[16, BN, 16])# circular buffer of pipeline stages in smem@gluon.jitdef ring(desc, STAGES: gl.constexpr): return gl.allocate_shared_memory(gl.float16, [STAGES] + desc.block_type.shape, desc.layout)# ready: tma bytes landed. empty: every consumer warpgroup is done with the stage@gluon.jitdef barriers(STAGES: gl.constexpr, CONSUMERS: gl.constexpr): ready = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout()) empty = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout()) for i in gl.static_range(STAGES): mbarrier.init(ready.index(i), count=1) mbarrier.init(empty.index(i), count=CONSUMERS) return ready, empty# producer: k_tiles loads for one tile, each into the next free stage@gluon.jitdef load_tile(a_desc, b_desc, e_desc, a_bufs, b_bufs, e_bufs, ready, empty, off_m, off_n, k_tiles, it): BK: gl.constexpr = a_desc.block_type.shape[1] STAGES: gl.constexpr = a_bufs.type.shape[0] NBYTES: gl.constexpr = a_desc.block_type.nbytes + b_desc.block_type.nbytes + (0 if e_desc is None else e_desc.block_type.nbytes) for kk in range(k_tiles): s = it % STAGES mbarrier.wait(empty.index(s), it // STAGES & 1 ^ 1) # wait until the consumers freed this stage mbarrier.expect(ready.index(s), NBYTES) # ready flips once all 48 kb land tma.async_copy_global_to_shared(a_desc, [off_m, kk * BK], ready.index(s), a_bufs.index(s)) tma.async_copy_global_to_shared(b_desc, [kk * BK, off_n], ready.index(s), b_bufs.index(s)) if e_desc is not None: tma.async_copy_global_to_shared(e_desc, [kk * BK, off_n + b_desc.block_type.shape[1]], ready.index(s), e_bufs.index(s)) it += 1 return it# producer warp: same tile order as the consumers, then its l piece@gluon.jitdef load_partition(a_desc, b_desc, e_desc, a_bufs, b_bufs, e_bufs, ready, empty, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, FULL, TAIL: gl.constexpr = 0, ROWS: gl.constexpr = 1, M0: gl.constexpr = 0, N0: gl.constexpr = 0): BM: gl.constexpr = a_desc.block_type.shape[0] BN: gl.constexpr = b_desc.block_type.shape[1] + (0 if e_desc is None else e_desc.block_type.shape[1]) pid = gl.program_id(0) it = 0 for tile in range(pid, FULL, GRID): pid_m, pid_n = tile_coords(tile, tm, tn, GROUP_M) it = load_tile(a_desc, b_desc, e_desc, a_bufs, b_bufs, e_bufs, ready, empty, pid_m * BM, pid_n * BN, k_tiles, it) # the l piece loads the same full a / b boxes as a normal tile if pid < TAIL: off_m, off_n, _ = ltile_coords(pid, ROWS, M0, N0) load_tile(a_desc, b_desc, e_desc, a_bufs, b_bufs, e_bufs, ready, empty, off_m, off_n, k_tiles, it)# non-ranked shapes: plain persistent kernel, one consumer group@gluon.jitdef coop_partition(c_desc, a_bufs, b_bufs, ready, empty, c_bufs, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, num_warps: gl.constexpr): BM: gl.constexpr = c_desc.block_type.shape[0] BN: gl.constexpr = b_bufs.type.shape[2] SPLIT: gl.constexpr = c_bufs.type.shape[0] STAGES: gl.constexpr = a_bufs.type.shape[0] it = 0 for tile in range(gl.program_id(0), tm * tn, GRID): pid_m, pid_n = tile_coords(tile, tm, tn, GROUP_M) s = it % STAGES mbarrier.wait(ready.index(s), it // STAGES & 1) acc = gl.zeros((BM, BN), gl.float32, acc_layout(BN, num_warps)) acc = warpgroup_mma(a_bufs.index(s), b_bufs.index(s), acc, is_async=True) acc = warpgroup_mma_wait(1, deps=(acc, )) prev = s it += 1 for _ in range(1, k_tiles): s = it % STAGES mbarrier.wait(ready.index(s), it // STAGES & 1) acc = warpgroup_mma(a_bufs.index(s), b_bufs.index(s), acc, is_async=True) acc = warpgroup_mma_wait(1, deps=(acc, )) mbarrier.arrive(empty.index(prev), count=1) prev = s it += 1 acc = warpgroup_mma_wait(0, deps=(acc, )) mbarrier.arrive(empty.index(prev), count=1) parts = split_n(acc, SPLIT) for i in gl.static_range(SPLIT): tma.store_wait(pendings=SPLIT - 1) c_bufs.index(i).store(parts[i].to(gl.float16)) fence_async_shared() tma.async_copy_shared_to_global(c_desc, [pid_m * BM, pid_n * BN + i * (BN // SPLIT)], c_bufs.index(i)) tma.store_wait(pendings=0)@gluon.jitdef coop_kernel(a_desc, b_desc, c_desc, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, STAGES: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, SPLIT: gl.constexpr, num_warps: gl.constexpr): a_bufs = ring(a_desc, STAGES) b_bufs = ring(b_desc, STAGES) c_bufs = ring(c_desc, SPLIT) ready, empty = barriers(STAGES, 1) gl.warp_specialize([ (coop_partition, (c_desc, a_bufs, b_bufs, ready, empty, c_bufs, tm, tn, k_tiles, GROUP_M, GRID, num_warps)), (load_partition, (a_desc, b_desc, None, a_bufs, b_bufs, None, ready, empty, tm, tn, k_tiles, GROUP_M, GRID, tm * tn)), ], [1], [40])# non-ranked 1024x1536: two accumulators per tile (n128 + n64)@gluon.jitdef pair_partition(c_desc, f_desc, a_bufs, b_bufs, e_bufs, ready, empty, c_buf, f_buf, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, num_warps: gl.constexpr): BM: gl.constexpr = c_desc.block_type.shape[0] BN0: gl.constexpr = b_bufs.type.shape[2] BN1: gl.constexpr = e_bufs.type.shape[2] STAGES: gl.constexpr = a_bufs.type.shape[0] it = 0 for tile in range(gl.program_id(0), tm * tn, GRID): pid_m, pid_n = tile_coords(tile, tm, tn, GROUP_M) acc0 = gl.zeros((BM, BN0), gl.float32, acc_layout(BN0, num_warps)) acc1 = gl.zeros((BM, BN1), gl.float32, acc_layout(BN1, num_warps)) prev = 0 for kk in range(k_tiles): s = it % STAGES mbarrier.wait(ready.index(s), it // STAGES & 1) acc0 = warpgroup_mma(a_bufs.index(s), b_bufs.index(s), acc0, is_async=True) acc1 = warpgroup_mma(a_bufs.index(s), e_bufs.index(s), acc1, is_async=True) acc0, acc1 = warpgroup_mma_wait(2, deps=(acc0, acc1)) mbarrier.arrive(empty.index(prev), count=1, pred=kk > 0) prev = s it += 1 acc0, acc1 = warpgroup_mma_wait(0, deps=(acc0, acc1)) mbarrier.arrive(empty.index(prev), count=1) tma.store_wait(pendings=0) c_buf.store(acc0.to(gl.float16)) f_buf.store(acc1.to(gl.float16)) fence_async_shared() tma.async_copy_shared_to_global(c_desc, [pid_m * BM, pid_n * (BN0 + BN1)], c_buf) tma.async_copy_shared_to_global(f_desc, [pid_m * BM, pid_n * (BN0 + BN1) + BN0], f_buf) tma.store_wait(pendings=0)@gluon.jitdef pair_kernel(a_desc, b_desc, e_desc, c_desc, f_desc, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, STAGES: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, num_warps: gl.constexpr): a_bufs = ring(a_desc, STAGES) b_bufs = ring(b_desc, STAGES) e_bufs = ring(e_desc, STAGES) c_buf = gl.allocate_shared_memory(gl.float16, c_desc.block_type.shape, c_desc.layout) f_buf = gl.allocate_shared_memory(gl.float16, f_desc.block_type.shape, f_desc.layout) ready, empty = barriers(STAGES, 1) gl.warp_specialize([ (pair_partition, (c_desc, f_desc, a_bufs, b_bufs, e_bufs, ready, empty, c_buf, f_buf, tm, tn, k_tiles, GROUP_M, GRID, num_warps)), (load_partition, (a_desc, b_desc, e_desc, a_bufs, b_bufs, e_bufs, ready, empty, tm, tn, k_tiles, GROUP_M, GRID, tm * tn)), ], [1], [40])# narrow half of an l piece: one n192 wgmma instead of n128 + n64, so a is read from smem once (40 -> 32 kb per k-step)@gluon.jitdef n192(acc, a_off, b_off): ao = gl.zeros([64, 64], gl.int32, L64) + a_off bo = gl.zeros([64, 64], gl.int32, L64) + b_off return gl.inline_asm_elementwise(N192_ASM, N192_CON, [acc[0], acc[1], acc[2], ao, bo], dtype=(gl.float32, gl.float32, gl.float32), is_pure=False, pack=32)# wait for the stage, then issue this k-step's wgmma (async)@gluon.jitdef k_issue(a_bufs, b_bufs, ready, it, acc, WG: gl.constexpr, NARROW: gl.constexpr): STAGES: gl.constexpr = a_bufs.type.shape[0] s = it % STAGES mbarrier.wait(ready.index(s), it // STAGES & 1) if NARROW: acc = n192(acc, A_RING + s * 16384 + WG * 8192, s * 32768 + (1 - WG) * 8192) # a: this wg's 64 rows. b: wg0's narrow half starts 64 cols in else: acc = warpgroup_mma(a_bufs.index(s).slice(WG * 64, 64, dim=0), b_bufs.index(s), acc, is_async=True) return acc, s# wait until at most pending wgmma groups are still in flight@gluon.jitdef k_retire(acc, PENDING: gl.constexpr, NARROW: gl.constexpr): if NARROW: acc = warpgroup_mma_wait(PENDING, deps=acc) else: acc = warpgroup_mma_wait(PENDING, deps=(acc, )) return acc# n plain k-steps. a stage is released once the wgmma reading it has retired@gluon.jitdef k_steps(a_bufs, b_bufs, ready, empty, it, prev, n, acc, WG: gl.constexpr, NARROW: gl.constexpr): for _ in range(n): acc, s = k_issue(a_bufs, b_bufs, ready, it, acc, WG, NARROW) acc = k_retire(acc, 1, NARROW) mbarrier.arrive(empty.index(prev), count=1) prev = s it += 1 return acc, it, prev# one 64x64 fp16 piece (8 kb): registers -> smem -> tma store. two staging buffers per warpgroup@gluon.jitdef store_piece(c_desc, c_bufs, part, row, col, g: gl.constexpr, WG: gl.constexpr): view = c_bufs.index(2 * WG + g % 2) tma.store_wait(pendings=1) # reuse a buffer only after its older store has left view.store(part) fence_async_shared() tma.async_copy_shared_to_global(c_desc, [row, col], view)# burst store, only when there is no next tile to hide behind@gluon.jitdef store_all(c_desc, c_bufs, parts, row, col, WG: gl.constexpr): for g in gl.static_range(len(parts)): store_piece(c_desc, c_bufs, parts[g], row, col + g * 64, g, WG)# one tile for one consumer warpgroup, while the previous tile's output goes out in pieces@gluon.jitdef wg_tile(a_bufs, b_bufs, ready, empty, c_desc, c_bufs, k_tiles, it, out, row, col, pending, WG: gl.constexpr, NARROW: gl.constexpr, P1: gl.constexpr, OFF: gl.constexpr): if NARROW: z = gl.zeros((64, 64), gl.float32, L64) acc = (z, z, z) else: acc = gl.zeros((64, 256), gl.float32, L256) acc, prev = k_issue(a_bufs, b_bufs, ready, it, acc, WG, NARROW) acc = k_retire(acc, 1, NARROW) # off: wg1 starts its stores 4 k-steps after wg0 acc, it, prev = k_steps(a_bufs, b_bufs, ready, empty, it + 1, prev, OFF, acc, WG, NARROW) # 4 pieces, one every p1 = 8 k-steps -> k-steps 1, 9, 17, 25 (+ off) parts = split_n(out, 4) for g in gl.static_range(4): acc, s = k_issue(a_bufs, b_bufs, ready, it, acc, WG, NARROW) # issue the wgmma, then the store, then wait: the store hides under the math if pending: store_piece(c_desc, c_bufs, parts[g], row, col + g * 64, g, WG) acc = k_retire(acc, 1, NARROW) mbarrier.arrive(empty.index(prev), count=1) acc, it, prev = k_steps(a_bufs, b_bufs, ready, empty, it + 1, s, P1 - 1, acc, WG, NARROW) # rest of the 64 k-steps acc, it, prev = k_steps(a_bufs, b_bufs, ready, empty, it, prev, k_tiles - 1 - OFF - 4 * P1, acc, WG, NARROW) acc = k_retire(acc, 0, NARROW) mbarrier.arrive(empty.index(prev), count=1) return acc, it# consumer warpgroup: rows wg*64 .. wg*64+63 of every tile@gluon.jitdef wg_partition(c_desc, a_bufs, b_bufs, ready, empty, c_bufs, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, FULL: gl.constexpr, TAIL: gl.constexpr, ROWS: gl.constexpr, M0: gl.constexpr, N0: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, P1: gl.constexpr, OFF: gl.constexpr, WG: gl.constexpr): pid = gl.program_id(0) out = gl.zeros((64, 256), gl.float16, L256) # previous tile's result, kept in registers (64 regs/thread) row = 0 col = 0 it = 0 for tile in range(pid, FULL, GRID): pid_m, pid_n = tile_coords(tile, tm, tn, GROUP_M) # pending = tile != pid: nothing to store on the first tile acc, it = wg_tile(a_bufs, b_bufs, ready, empty, c_desc, c_bufs, k_tiles, it, out, row, col, tile != pid, WG, False, P1, OFF) out = acc.to(gl.float16) row = pid_m * 128 + WG * 64 col = pid_n * 256 # last wave: this cta computes one l piece if pid < TAIL: off_m, off_n, kind = ltile_coords(pid, ROWS, M0, N0) if kind == WG: # this wg takes the wide 64x256 half acc, it = wg_tile(a_bufs, b_bufs, ready, empty, c_desc, c_bufs, k_tiles, it, out, row, col, pid < FULL, WG, False, P1, OFF) # nothing left to overlap with, so the last pieces go out as a burst store_all(c_desc, c_bufs, split_n(acc.to(gl.float16), 4), off_m + WG * 64, off_n, WG) else: # narrow 64x192 half, as three 64x64 accumulators thirds, it = wg_tile(a_bufs, b_bufs, ready, empty, c_desc, c_bufs, k_tiles, it, out, row, col, pid < FULL, WG, True, P1, OFF) parts = (thirds[0].to(gl.float16), thirds[1].to(gl.float16), thirds[2].to(gl.float16)) store_all(c_desc, c_bufs, parts, off_m + WG * 64, off_n + 64 - 64 * WG, WG) elif pid < FULL: # ctas without an l piece flush their last full tile store_all(c_desc, c_bufs, split_n(out, 4), row, col, WG) tma.store_wait(pendings=0)# ranked shape (4096x5120x4096): 2 consumer warpgroups + 1 producer warp, persistent on 132 sms@gluon.jitdef ltile_kernel(a_desc, b_desc, c_desc, tm: gl.constexpr, tn: gl.constexpr, k_tiles: gl.constexpr, STAGES: gl.constexpr, FULL: gl.constexpr, TAIL: gl.constexpr, ROWS: gl.constexpr, M0: gl.constexpr, N0: gl.constexpr, GROUP_M: gl.constexpr, GRID: gl.constexpr, P1: gl.constexpr, OFF1: gl.constexpr): a_bufs = ring(a_desc, STAGES) b_bufs = ring(b_desc, STAGES) c_bufs = ring(c_desc, 4) # 4 x 8 kb store buffers. smem: 4 x 48 kb stages + 32 kb = 224 of 227 kb ready, empty = barriers(STAGES, 2) # both consumers arrive on empty gl.warp_specialize([ (wg_partition, (c_desc, a_bufs, b_bufs, ready, empty, c_bufs, tm, tn, k_tiles, FULL, TAIL, ROWS, M0, N0, GROUP_M, GRID, P1, 0, 0)), (wg_partition, (c_desc, a_bufs, b_bufs, ready, empty, c_bufs, tm, tn, k_tiles, FULL, TAIL, ROWS, M0, N0, GROUP_M, GRID, P1, OFF1, 1)), (load_partition, (a_desc, b_desc, None, a_bufs, b_bufs, None, ready, empty, tm, tn, k_tiles, GROUP_M, GRID, FULL, TAIL, ROWS, M0, N0)), ], [4, 1], [232, 40]) # setmaxnreg: producer drops to 40 regs so the consumers get 232# host copy of tile_coords, used to plan the last wavedef host_tile_coords(tile, tm, tn, group_m): size = group_m * tn group = tile // size rows = min(tm - group * group_m, group_m) pid_n = tile % size // rows return group * group_m + tile % size % rows, tn - 1 - pid_n if group % 2 else pid_n# tiles left after 4 full waves (528). with this tile order they form a 16 x 7 rectangle# -> 4 windows x 16 rows x 2 ctas = 128 l piecesdef ltile_plan(tm, tn, group_m): full = tm * tn // NUM_SMS * NUM_SMS left = {host_tile_coords(t, tm, tn, group_m) for t in range(full, tm * tn)} (r0, c0), (r1, c1) = min(left), max(left) pairs, rem = divmod((c1 - c0 + 1) * 256, 448) assert left == {(r, c) for r in range(r0, r1 + 1) for c in range(c0, c1 + 1)} and not rem assert 2 * (r1 - r0 + 1) * pairs <= NUM_SMS return full, 2 * (r1 - r0 + 1) * pairs, r1 - r0 + 1, r0 * 128, c0 * 256# tma descriptor with the default nvmma swizzledef desc(t, *block): return TensorDescriptor.from_tensor(t, list(block), gl.NVMMASharedLayout.get_default_for(list(block), gl.float16))def ltile_launch(a, b, c, STAGES, GROUP_M, P1, OFF1): tm, tn = a.shape[0] // 128, b.shape[1] // 256 args = (desc(a, 128, 64), desc(b, 64, 256), desc(c, 64, 64), tm, tn, a.shape[1] // 64, STAGES, *ltile_plan(tm, tn, GROUP_M), GROUP_M, NUM_SMS, P1, OFF1) return ltile_kernel, NUM_SMS, args, dict(num_warps=4, maxnreg=168) # 12 warps x 168 = 8 x 232 + 4 x 40def coop_launch(a, b, c, BM, BN, BK, STAGES, GROUP_M, SPLIT=1, GRID=NUM_SMS): (m, k), n = a.shape, b.shape[1] tm, tn = -(-m // BM), -(-n // BN) grid = min(GRID, tm * tn) args = (desc(a, BM, BK), desc(b, BK, BN), desc(c, BM, BN // SPLIT), tm, tn, -(-k // BK), STAGES, GROUP_M, grid, SPLIT) return coop_kernel, grid, args, dict(num_warps=BM // 16, maxnreg=168 if BM == 128 else 232)def pair_launch(a, b, c, BM, BN0, BN1, BK, STAGES, GROUP_M): (m, k), n = a.shape, b.shape[1] tm, tn = m // BM, n // (BN0 + BN1) grid = min(NUM_SMS, tm * tn) args = (desc(a, BM, BK), desc(b, BK, BN0), desc(b, BK, BN1), desc(c, BM, BN0), desc(c, BM, BN1), tm, tn, k // BK, STAGES, GROUP_M, grid) return pair_kernel, grid, args, dict(num_warps=BM // 16, maxnreg=168)# lean host launch: compile once, then per call refill the tma descriptors and call triton's c launcher directlyclass Launcher: def __init__(self, kernel, grid, args, kw, tensors): compiled = kernel.warmup(*args, grid=(grid, ), **kw) run = compiled.run bound = kernel.device_caches[driver.active.get_current_device()][4](*args, **kw)[0] metas = iter(compiled.metadata.tensordesc_meta) self.template, self.maps = [], [] for v in bound.values(): if isinstance(v, TensorDescriptor): md = next(metas) src = next(i for i, t in enumerate(tensors) if v.base is t) self.maps.append((len(self.template), src, md["swizzle"], md["elem_size"], TMA_DTYPE_DEVICE_TO_HOST[md["elem_type"]], list(md["block_size"]), list(v.shape), list(v.strides))) self.template += [None, *v.shape, *v.strides] else: self.template.append(v) self.head = (grid, 1, 1, 0, compiled.function, run.launch_cooperative_grid, run.launch_pdl, compiled.packed_metadata, None, None, None, None, None, run.arg_annotations, run.kernel_signature) def __call__(self, tensors): args = self.template.copy() for pos, src, *encoding in self.maps: args[pos] = driver.active.utils.fill_tma_descriptor_tiled(tensors[src].data_ptr(), *encoding, 0) driver.active.utils.launch(*self.head, args)CONFIGS = { (4096, 5120, 4096): (ltile_launch, dict(STAGES=4, GROUP_M=16, P1=8, OFF1=4)), # ranked: 4 stages, 16-row groups, a store every 8 k-steps, wg1 offset 4 (2048, 3072, 2048): (coop_launch, dict(BM=128, BN=128, BK=64, STAGES=6, GROUP_M=8, SPLIT=2)), # other shapes: per-shape tiles (2048, 2048, 2048): (coop_launch, dict(BM=128, BN=128, BK=64, STAGES=6, GROUP_M=8, SPLIT=2, GRID=128)), (1024, 1536, 1024): (pair_launch, dict(BM=64, BN0=128, BN1=64, BK=64, STAGES=6, GROUP_M=4)), (1024, 1024, 1024): (coop_launch, dict(BM=64, BN=128, BK=64, STAGES=8, GROUP_M=8)), (512, 512, 512): (coop_launch, dict(BM=64, BN=32, BK=128, STAGES=4, GROUP_M=8)), (256, 256, 256): (coop_launch, dict(BM=64, BN=16, BK=64, STAGES=4, GROUP_M=8)), (128, 128, 128): (coop_launch, dict(BM=64, BN=32, BK=128, STAGES=2, GROUP_M=8)),}DEFAULT = (coop_launch, dict(BM=64, BN=64, BK=64, STAGES=3, GROUP_M=8))LAUNCHERS = {}# one launcher per shape, built on the first calldef custom_kernel(data: input_t) -> output_t: a, b, c = data key = (a.shape[0], b.shape[1], a.shape[1]) if key not in LAUNCHERS: launch, cfg = CONFIGS.get(key, DEFAULT) LAUNCHERS[key] = Launcher(*launch(a, b, c, **cfg), data) LAUNCHERS[key](data) return c