# conv2d_v2 on a100. the ranked shape runs spectral_conv (4 launches); every other shape is a helion fallbackimport ctypesimport globimport osimport sysfrom functools import lru_cacheimport torchimport helionimport helion.language as hlfrom triton.experimental import gluon as gfrom triton.experimental.gluon import language as gl# ---------------------------------------------------------------------------------------------------------------# small test shapes: exact direct conv as one ieee gemm. these only have to pass@helion.kernel(static_shapes=True, dot_precision='ieee', config=helion.Config(block_sizes=[64, 64, 32], num_warps=4, num_stages=3))def exact_conv(x, w, out): batch, channels, size, _ = x.shape width = w.shape[-1] side = size - width + 1 for co, pos in hl.tile([channels, batch * side * side]): p = pos.index b = p // (side * side) oy = p // side % side ox = p % side acc = hl.zeros([co, pos], dtype=torch.float32) for red in hl.tile(channels * width * width): r = red.index ci = r // (width * width) ky = r // width % width kx = r % width a = w[co.index[:, None], ci[None, :], ky[None, :], kx[None, :]] v = x[b[None, :], ci[:, None], ky[:, None] + oy[None, :], kx[:, None] + ox[None, :]] acc = hl.dot(a, v, acc) out[b[None, :], co.index[:, None], oy[None, :], ox[None, :]] = acc# ---------------------------------------------------------------------------------------------------------------# non-ranked big shapes: helion overlap-save fft (48- or 80-point tiles, cufft, tf32x3 mixer)# cut x into overlapping t x t tiles, zeros past the border@helion.kernel(config=helion.Config(block_sizes=[256], num_warps=4))def gather_tiles(x, tiles, step: hl.constexpr, nt: hl.constexpr): b, c, s, _ = x.shape t = tiles.shape[-1] flat = tiles.view(-1) source = x.view(-1) for pos in hl.tile(flat.numel()): p = pos.index tile = p // (c * t * t) ci = p // (t * t) % c y = tile // nt % nt * step + p // t % t z = tile % nt * step + p % t flat[pos] = hl.load(source, [((tile // (nt * nt) * c + ci) * s + y) * s + z], extra_mask=(y < s) & (z < s))# [rows, freq] <-> [freq, rows], real and imag planes@helion.kernel(config=helion.Config(block_sizes=[32, 32], num_warps=4))def transpose_spectra(xr, xi, yr, yi): rows, freq = xr.shape for r, f in hl.tile([rows, freq]): yr[f, r] = xr[r, f].T yi[f, r] = xi[r, f].Tdef transpose_complex(z): z = z.reshape(-1, z.shape[-1]) out = torch.empty((z.shape[1], z.shape[0]), dtype=z.dtype, device=z.device) transpose_spectra(z.real, z.imag, out.real, out.imag) return out# s256 k16: weight spectrum as two tf32x3 dft gemms (rows, then columns) instead of a padded cufft@helion.kernel(static_shapes=True, dot_precision='tf32x3', config=helion.Config(block_sizes=[64, 64, 32], num_warps=4, num_stages=2))def dft_rows(w, cr, si, yr, yi): rows, width = w.shape freq = cr.shape[1] for mt, nt in hl.tile([rows, freq]): real = hl.zeros([mt, nt], dtype=torch.float32) imag = hl.zeros([mt, nt], dtype=torch.float32) for kt in hl.tile(width): a = w[mt, kt] real = hl.dot(a, cr[kt, nt], real) imag = hl.dot(a, si[kt, nt], imag) yr[nt, mt] = real.T yi[nt, mt] = imag.T@helion.kernel(static_shapes=True, dot_precision='tf32x3', config=helion.Config(block_sizes=[1, 32, 64, 32], num_warps=4, num_stages=2))def dft_columns(xr, xi, cr, si, yr, yi): freq, filters, width = xr.shape height = cr.shape[1] for ft, mt, nt in hl.tile([freq, height, filters]): real = hl.zeros([ft, mt, nt], dtype=torch.float32) imag = hl.zeros([ft, mt, nt], dtype=torch.float32) for kt in hl.tile(width): ar = cr[ft, mt, kt] ai = si[ft, mt, kt] br = xr[ft, nt, kt].transpose(1, 2) bi = xi[ft, nt, kt].transpose(1, 2) real = hl.dot(ar, br, real) real = hl.dot(-ai, bi, real) imag = hl.dot(ar, bi, imag) imag = hl.dot(ai, br, imag) yr[mt.index[None, :, None] * freq + ft.index[:, None, None], nt.index[None, None, :]] = real yi[mt.index[None, :, None] * freq + ft.index[:, None, None], nt.index[None, None, :]] = imag@lru_cache(None)def twiddles(width, t, device): phase = -2 * torch.pi * torch.arange(t, dtype=torch.float64)[:, None] * torch.arange(width, dtype=torch.float64)[None, :] / t cr, si = (phase.cos().float().to(device), phase.sin().float().to(device)) freq = t // 2 + 1 return (cr[:freq].T.contiguous(), si[:freq].T.contiguous(), cr[None].expand(freq, -1, -1), si[None].expand(freq, -1, -1))def weight_dft(w, t): c, _, width, _ = w.shape freq = t // 2 + 1 cr, si, hc, hs = twiddles(width, t, w.device) tmp = torch.empty((freq, c * c * width), device=w.device, dtype=torch.complex64) dft_rows(w.view(-1, width), cr, si, tmp.real, tmp.imag) out = torch.empty((t * freq, c * c), device=w.device, dtype=torch.complex64) dft_columns(tmp.real.view(freq, c * c, width), tmp.imag.view(freq, c * c, width), hc, hs, out.real, out.imag) return out.view(t * freq, c, c)# channel mixer per frequency: y[f, m, co] = sum over ci of x[f, m, ci] * conj(w[f, co, ci])@helion.kernel(dot_precision='tf32x3', static_shapes=True, config=helion.Config(block_sizes=[1, 32, 32, 32], num_warps=4, num_stages=2))def mix_tiles(xr, xi, wr, wi, yr, yi): freq, m, c = xr.shape n = wr.shape[1] for ft, mt, nt in hl.tile([freq, m, n]): real = hl.zeros([ft, mt, nt], dtype=torch.float32) imag = hl.zeros([ft, mt, nt], dtype=torch.float32) for kt in hl.tile(c): ar, ai = (xr[ft, mt, kt], xi[ft, mt, kt]) br, bi = (wr[ft, nt, kt].transpose(1, 2), wi[ft, nt, kt].transpose(1, 2)) real = hl.dot(ar, br, real) real = hl.dot(ai, bi, real) imag = hl.dot(ai, br, imag) imag = hl.dot(-ar, bi, imag) yr[ft, mt, nt] = real yi[ft, mt, nt] = imag# keep the valid step x step corner of each inverse tile@helion.kernel(config=helion.Config(block_sizes=[256], num_warps=4))def scatter_tiles(tiles, out, step: hl.constexpr, nt: hl.constexpr): b, c, o, _ = out.shape flat = out.view(-1) source = tiles.view(-1) t = tiles.shape[-1] for pos in hl.tile(flat.numel()): p = pos.index y, x = (p // o % o, p % o) tile = p // (c * o * o) * nt * nt + y // step * nt + x // step flat[pos] = source[((tile * c + p // (o * o) % c) * t + y % step) * t + x % step]def overlap_conv(data, t): x, w, out = data b, c, s, _ = x.shape k = w.shape[-1] step = t - k + 1 nt = (s - k + step) // step m, freq = (b * nt * nt, t * (t // 2 + 1)) tiles = torch.empty((m, c, t, t), device=x.device, dtype=x.dtype) gather_tiles(x, tiles, step, nt) xf = transpose_complex(torch.fft.rfft2(tiles).flatten(2)).view(freq, m, c) wf = transpose_complex(torch.fft.rfft2(w, s=(t, t)).flatten(2)).view(freq, c, c) if s == 128 else weight_dft(w, t) yf = torch.empty_like(xf) mix_tiles(xf.real, xf.imag, wf.real, wf.imag, yf.real, yf.imag) yf = transpose_complex(yf.view(freq, m * c)).view(m, c, t, t // 2 + 1) result = torch.fft.irfft2(yf, s=(t, t)) scatter_tiles(result, out, step, nt) return out# ---------------------------------------------------------------------------------------------------------------# cuda runtime: nvrtc compile + driver launch through ctypes. keeps the submission one python file, no build stepdef nvrtc_candidates(): roots = sorted({p for p in sys.path if p.endswith("site-packages")}) hits = [] for r in roots: hits += sorted(glob.glob(os.path.join(r, "nvidia", "**", "libnvrtc.so*"), recursive=True)) return hits + sorted(glob.glob("/usr/local/cuda*/lib64/libnvrtc.so*"))class Cuda: def __init__(self): self.nvrtc = None for h in nvrtc_candidates(): try: self.nvrtc = ctypes.CDLL(h) break except OSError: pass V, P = ctypes.c_void_p, ctypes.POINTER n = self.nvrtc n.nvrtcCreateProgram.argtypes = [P(V), ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int, V, V] n.nvrtcCompileProgram.argtypes = [V, ctypes.c_int, P(ctypes.c_char_p)] n.nvrtcGetProgramLogSize.argtypes = [V, P(ctypes.c_size_t)] n.nvrtcGetProgramLog.argtypes = [V, ctypes.c_char_p] n.nvrtcGetPTXSize.argtypes = [V, P(ctypes.c_size_t)] n.nvrtcGetPTX.argtypes = [V, ctypes.c_char_p] d = self.drv = ctypes.CDLL("libcuda.so.1") d.cuCtxGetCurrent.argtypes = [P(V)] d.cuModuleLoadData.argtypes = [P(V), V] d.cuModuleGetFunction.argtypes = [P(V), V, ctypes.c_char_p] d.cuFuncSetAttribute.argtypes = [V, ctypes.c_int, ctypes.c_int] d.cuLaunchKernel.argtypes = [V] + [ctypes.c_uint] * 7 + [V, V, V] torch.zeros(1, device="cuda") ctx = V() rc = d.cuCtxGetCurrent(ctypes.byref(ctx)) assert rc == 0 and ctx.value cap = torch.cuda.get_device_capability() self.arch = f"sm_{cap[0]}{cap[1]}" def build(self, source, name, defs=()): opts = [f"--gpu-architecture={self.arch}", "-std=c++17"] + [f"-D{k}={v}" for k, v in defs] arr = (ctypes.c_char_p * len(opts))(*[o.encode() for o in opts]) prog = ctypes.c_void_p() rc = self.nvrtc.nvrtcCreateProgram(ctypes.byref(prog), source.encode(), b"k.cu", 0, None, None) assert rc == 0, f"create {rc}" rc = self.nvrtc.nvrtcCompileProgram(prog, len(opts), arr) size = ctypes.c_size_t() self.nvrtc.nvrtcGetProgramLogSize(prog, ctypes.byref(size)) log = ctypes.create_string_buffer(size.value + 1) self.nvrtc.nvrtcGetProgramLog(prog, log) assert rc == 0, f"compile {rc}: {log.value.decode(errors='replace')[-3000:]}" self.nvrtc.nvrtcGetPTXSize(prog, ctypes.byref(size)) ptx = ctypes.create_string_buffer(size.value + 1) self.nvrtc.nvrtcGetPTX(prog, ptx) mod = ctypes.c_void_p() rc = self.drv.cuModuleLoadData(ctypes.byref(mod), ctypes.cast(ptx, ctypes.c_void_p)) assert rc == 0, f"module load {rc}" fn = ctypes.c_void_p() rc = self.drv.cuModuleGetFunction(ctypes.byref(fn), mod, name.encode()) assert rc == 0, f"get function {rc}" return fn def launch(self, fn, grid, block, smem, args): rc = self.drv.cuFuncSetAttribute(fn, 8, smem) # 8 = max dynamic smem assert rc == 0, f"set smem {rc}" arr = (ctypes.c_void_p * len(args))(*[ctypes.addressof(a) for a in args]) rc = self.drv.cuLaunchKernel(fn, grid, 1, 1, block, 1, 1, smem, None, ctypes.cast(arr, ctypes.c_void_p), None) assert rc == 0, f"launch {rc}"@lru_cache(None)def cuda_runtime(): return Cuda()def pointers(*tensors): return [ctypes.c_void_p(t.data_ptr()) for t in tensors]# ---------------------------------------------------------------------------------------------------------------# ranked shape (s256 k32 c128 b1): 16 tiles of 88x88, step 57. 88 = 8 x 11. bins f = fh*45 + fw, 88 x 45 = 3960# shared cuda pieces: tensor-core / fp16 helpers and the 88-point fft building blocksCU_MMA = r"""// mma.m16n8k16: fp16 inputs, fp32 accumulate#define MMA(d, a0, a1, a2, a3, b0, b1) asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};" : "=f"((d)[0]), "=f"((d)[1]), "=f"((d)[2]), "=f"((d)[3]) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "f"((d)[0]), "f"((d)[1]), "f"((d)[2]), "f"((d)[3]))#define CVT2(o, hi, lo) asm("cvt.rn.f16x2.f32 %0, %1, %2;" : "=r"(o) : "f"(hi), "f"(lo)) // two floats -> one fp16x2static __device__ __forceinline__ unsigned f2u(float v) { unsigned u; asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); return u; }static __device__ __forceinline__ float u2f(unsigned u) { float v; asm("mov.b32 %0, %1;" : "=f"(v) : "r"(u)); return v; }"""CU_FFT = r"""// twiddles for the 88- and 11-point transforms__constant__ float C88[88] = {1.000000000e+00f,9.974521146e-01f,9.898214419e-01f,9.771468660e-01f,9.594929736e-01f,9.369497250e-01f,9.096319954e-01f,8.776789896e-01f,8.412535328e-01f,8.005412409e-01f,7.557495744e-01f,7.071067812e-01f,6.548607339e-01f,5.992776665e-01f,5.406408175e-01f,4.792489867e-01f,4.154150130e-01f,3.494641796e-01f,2.817325568e-01f,2.125652896e-01f,1.423148383e-01f,7.133918320e-02f,2.832769449e-16f,-7.133918320e-02f,-1.423148383e-01f,-2.125652896e-01f,-2.817325568e-01f,-3.494641796e-01f,-4.154150130e-01f,-4.792489867e-01f,-5.406408175e-01f,-5.992776665e-01f,-6.548607339e-01f,-7.071067812e-01f,-7.557495744e-01f,-8.005412409e-01f,-8.412535328e-01f,-8.776789896e-01f,-9.096319954e-01f,-9.369497250e-01f,-9.594929736e-01f,-9.771468660e-01f,-9.898214419e-01f,-9.974521146e-01f,-1.000000000e+00f,-9.974521146e-01f,-9.898214419e-01f,-9.771468660e-01f,-9.594929736e-01f,-9.369497250e-01f,-9.096319954e-01f,-8.776789896e-01f,-8.412535328e-01f,-8.005412409e-01f,-7.557495744e-01f,-7.071067812e-01f,-6.548607339e-01f,-5.992776665e-01f,-5.406408175e-01f,-4.792489867e-01f,-4.154150130e-01f,-3.494641796e-01f,-2.817325568e-01f,-2.125652896e-01f,-1.423148383e-01f,-7.133918320e-02f,-1.836970199e-16f,7.133918320e-02f,1.423148383e-01f,2.125652896e-01f,2.817325568e-01f,3.494641796e-01f,4.154150130e-01f,4.792489867e-01f,5.406408175e-01f,5.992776665e-01f,6.548607339e-01f,7.071067812e-01f,7.557495744e-01f,8.005412409e-01f,8.412535328e-01f,8.776789896e-01f,9.096319954e-01f,9.369497250e-01f,9.594929736e-01f,9.771468660e-01f,9.898214419e-01f,9.974521146e-01f};__constant__ float S88[88] = {0.000000000e+00f,7.133918320e-02f,1.423148383e-01f,2.125652896e-01f,2.817325568e-01f,3.494641796e-01f,4.154150130e-01f,4.792489867e-01f,5.406408175e-01f,5.992776665e-01f,6.548607339e-01f,7.071067812e-01f,7.557495744e-01f,8.005412409e-01f,8.412535328e-01f,8.776789896e-01f,9.096319954e-01f,9.369497250e-01f,9.594929736e-01f,9.771468660e-01f,9.898214419e-01f,9.974521146e-01f,1.000000000e+00f,9.974521146e-01f,9.898214419e-01f,9.771468660e-01f,9.594929736e-01f,9.369497250e-01f,9.096319954e-01f,8.776789896e-01f,8.412535328e-01f,8.005412409e-01f,7.557495744e-01f,7.071067812e-01f,6.548607339e-01f,5.992776665e-01f,5.406408175e-01f,4.792489867e-01f,4.154150130e-01f,3.494641796e-01f,2.817325568e-01f,2.125652896e-01f,1.423148383e-01f,7.133918320e-02f,5.665538898e-16f,-7.133918320e-02f,-1.423148383e-01f,-2.125652896e-01f,-2.817325568e-01f,-3.494641796e-01f,-4.154150130e-01f,-4.792489867e-01f,-5.406408175e-01f,-5.992776665e-01f,-6.548607339e-01f,-7.071067812e-01f,-7.557495744e-01f,-8.005412409e-01f,-8.412535328e-01f,-8.776789896e-01f,-9.096319954e-01f,-9.369497250e-01f,-9.594929736e-01f,-9.771468660e-01f,-9.898214419e-01f,-9.974521146e-01f,-1.000000000e+00f,-9.974521146e-01f,-9.898214419e-01f,-9.771468660e-01f,-9.594929736e-01f,-9.369497250e-01f,-9.096319954e-01f,-8.776789896e-01f,-8.412535328e-01f,-8.005412409e-01f,-7.557495744e-01f,-7.071067812e-01f,-6.548607339e-01f,-5.992776665e-01f,-5.406408175e-01f,-4.792489867e-01f,-4.154150130e-01f,-3.494641796e-01f,-2.817325568e-01f,-2.125652896e-01f,-1.423148383e-01f,-7.133918320e-02f};__constant__ float C11[11] = {1.000000000e+00f,8.412535328e-01f,4.154150130e-01f,-1.423148383e-01f,-6.548607339e-01f,-9.594929736e-01f,-9.594929736e-01f,-6.548607339e-01f,-1.423148383e-01f,4.154150130e-01f,8.412535328e-01f};__constant__ float S11[11] = {0.000000000e+00f,5.406408175e-01f,9.096319954e-01f,9.898214419e-01f,7.557495744e-01f,2.817325568e-01f,-2.817325568e-01f,-7.557495744e-01f,-9.898214419e-01f,-9.096319954e-01f,-5.406408175e-01f};#define RS 91 // padded smem row stride, avoids bank conflicts on the column passesstruct cf { float x, y; };__device__ __forceinline__ cf cadd(cf a, cf b) { return {a.x + b.x, a.y + b.y}; }__device__ __forceinline__ cf csub(cf a, cf b) { return {a.x - b.x, a.y - b.y}; }template <int SG> __device__ __forceinline__ void dft4(cf &a0, cf &a1, cf &a2, cf &a3) { cf t0 = cadd(a0, a2), t1 = csub(a0, a2), t2 = cadd(a1, a3), d = csub(a1, a3); cf t3 = {-SG * d.y, SG * d.x}; a0 = cadd(t0, t2); a2 = csub(t0, t2); a1 = cadd(t1, t3); a3 = csub(t1, t3);}// radix-8 butterfly. sg = -1 forward, +1 inversetemplate <int SG> __device__ __forceinline__ void dft8(cf *a) { const float r = 0.70710678118654752f; cf u0 = cadd(a[0], a[4]), u1 = cadd(a[1], a[5]), u2 = cadd(a[2], a[6]), u3 = cadd(a[3], a[7]); cf v0 = csub(a[0], a[4]), v1 = csub(a[1], a[5]), v2 = csub(a[2], a[6]), v3 = csub(a[3], a[7]); v1 = {r * (v1.x - SG * v1.y), r * (SG * v1.x + v1.y)}; v2 = {-SG * v2.y, SG * v2.x}; v3 = {r * (-v3.x - SG * v3.y), r * (SG * v3.x - v3.y)}; dft4<SG>(u0, u1, u2, u3); dft4<SG>(v0, v1, v2, v3); a[0] = u0; a[2] = u1; a[4] = u2; a[6] = u3; a[1] = v0; a[3] = v1; a[5] = v2; a[7] = v3;}// direct 11-point dft from the symmetric pairs (n, 11-n)template <int SG> __device__ __forceinline__ void dft11(cf *a) { cf s[5], d[5];#pragma unroll for (int n = 1; n <= 5; ++n) { s[n - 1] = cadd(a[n], a[11 - n]); d[n - 1] = csub(a[n], a[11 - n]); } cf x0 = a[0], y0 = a[0];#pragma unroll for (int n = 0; n < 5; ++n) y0 = cadd(y0, s[n]);#pragma unroll for (int k = 1; k <= 5; ++k) { float cr = x0.x, ci = x0.y, sr = 0.f, si = 0.f;#pragma unroll for (int n = 1; n <= 5; ++n) { const int m = (n * k) % 11; cr += s[n - 1].x * C11[m]; ci += s[n - 1].y * C11[m]; sr += d[n - 1].y * S11[m]; si += d[n - 1].x * S11[m]; } a[k] = {cr - SG * sr, ci + SG * si}; a[11 - k] = {cr + SG * sr, ci - SG * si}; } a[0] = y0;}"""# launch 1 (combo): two unrelated jobs in one launch, blocks interleaved 1:1 so both share every sm# even blocks: fft of one (tile, ci) in smem -> z. smem / latency bound# odd blocks: weight row dft (kx -> fw) on tensor cores, registers only -> t (fp16 hi / lo)PREP_SRC = CU_MMA + CU_FFT + r"""#define NSIG 1 // signals per fft block#define NTS 256 // threads per block#define MINB 2 // min blocks per sm#define RNW 8 // warps per row-dft block#define RPP 1 // pairs per warp#define FPR 1 // fft blocks per row-dft block__device__ __forceinline__ cf rotr(cf a, float c, float s) { return {a.x * c - a.y * s, a.x * s + a.y * c}; }__device__ __forceinline__ unsigned smaddr(const void *p) { unsigned a; asm("{.reg .u64 t; cvta.to.shared.u64 t, %1; cvt.u32.u64 %0, t;}" : "=r"(a) : "l"(p)); return a;}// split-fp16: hi = v rounded to 11 significant bits, lo = v - hi. three fp16 products give ~22 bitsstatic __device__ __forceinline__ float rn11r(float v) { unsigned u = f2u(v); u = (u + 0x1000u) & 0xFFFFE000u; return u2f(u);}// weight row dft: one warp per (co, ci) pair, no smem, no barriersstatic __device__ __forceinline__ void rows_body(const int bxr, const float4 *__restrict__ wt, const uint4 *__restrict__ tab, uint4 *__restrict__ tp){ const int tid = threadIdx.x, lane = tid & 31, w = tid >> 5, g = lane >> 2, t = lane & 3; const int TLO = 45 * 16384 * 8; for (int it = 0; it < RPP; ++it) { const int pair = (bxr * RNW + w) * RPP + it; unsigned Bh[4][4], Bl[4][4]; // weights as fp16 hi / lo #pragma unroll for (int q = 0; q < 4; ++q) { const int ky = 8 * (g >> 1) + 2 * q + (g & 1); const float4 a = wt[(pair * 32 + ky) * 8 + 2 * t], b = wt[(pair * 32 + ky) * 8 + 2 * t + 1]; const float v[8] = {a.x, a.y, a.z, a.w, b.x, b.y, b.z, b.w}; #pragma unroll for (int j = 0; j < 4; ++j) { const float h0 = rn11r(v[2 * j]), h1 = rn11r(v[2 * j + 1]); CVT2(Bh[q][j], h1, h0); CVT2(Bl[q][j], v[2 * j + 1] - h1, v[2 * j] - h0); } } #pragma unroll for (int mtl = 0; mtl < 3; ++mtl) { // fw = 16*mtl + g (+8) #pragma unroll for (int cs = 0; cs < 2; ++cs) { // cos -> re, -sin -> im const uint4 *ta = tab + ((mtl * 2 + cs) * 4) * 32 + lane; const uint4 ah0 = ta[0], ah1 = ta[32], al0 = ta[64], al1 = ta[96]; float O[2][8]; // [fw row g / g+8][ky 8t .. 8t+7] #pragma unroll for (int q = 0; q < 4; ++q) { float acc[4] = {0.f, 0.f, 0.f, 0.f}; // small cross terms first (hi*lo, lo*hi), then hi*hi. lo*lo is dropped MMA(acc, ah0.x, ah0.y, ah0.z, ah0.w, Bl[q][0], Bl[q][1]); MMA(acc, ah1.x, ah1.y, ah1.z, ah1.w, Bl[q][2], Bl[q][3]); MMA(acc, al0.x, al0.y, al0.z, al0.w, Bh[q][0], Bh[q][1]); MMA(acc, al1.x, al1.y, al1.z, al1.w, Bh[q][2], Bh[q][3]); MMA(acc, ah0.x, ah0.y, ah0.z, ah0.w, Bh[q][0], Bh[q][1]); MMA(acc, ah1.x, ah1.y, ah1.z, ah1.w, Bh[q][2], Bh[q][3]); O[0][2 * q] = acc[0]; O[0][2 * q + 1] = acc[1]; O[1][2 * q] = acc[2]; O[1][2 * q + 1] = acc[3]; } #pragma unroll for (int row = 0; row < 2; ++row) { const int fw = 16 * mtl + g + 8 * row; // re-split the result into hi / lo, 16-byte stores in fragment order uint4 hi, lo; unsigned ph[4], pl[4]; #pragma unroll for (int j = 0; j < 4; ++j) { const float h0 = rn11r(O[row][2 * j]), h1 = rn11r(O[row][2 * j + 1]); CVT2(ph[j], h1, h0); CVT2(pl[j], O[row][2 * j + 1] - h1, O[row][2 * j] - h0); } hi.x = ph[0]; hi.y = ph[1]; hi.z = ph[2]; hi.w = ph[3]; lo.x = pl[0]; lo.y = pl[1]; lo.z = pl[2]; lo.w = pl[3]; // real input: only 45 of 88 fw bins are needed if (fw < 45) { const int o = (fw * 16384 + pair) * 8 + cs * 4 + t; tp[o] = hi; tp[TLO + o] = lo; } } } } }}// image fft: one 88x88 tile of one input channelstatic __device__ __forceinline__ void fwd_body(const int bxf, const float *__restrict__ x, float2 *__restrict__ z) { extern __shared__ float sm[]; float *twc = sm + NSIG * 88 * RS, *tws = twc + 88; const int t = bxf / (128 / NSIG), ci0 = (bxf % (128 / NSIG)) * NSIG, Y0 = 57 * (t >> 2), X0 = 57 * (t & 3); const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; if (tid < 88) { twc[tid] = C88[tid]; tws[tid] = -S88[tid]; } for (int r = warp; r < NSIG * 88; r += NTS / 32) { const int s = r / 88, Y = Y0 + r % 88; const float *src = x + ((size_t)(ci0 + s) * 256 + min(Y, 255)) * 256; for (int j = lane; j < 88; j += 32) { const int X = X0 + j; // cp.async with src-size 0 writes zeros: free padding past the image edge const int ok = (Y < 256 && X < 256) ? 4 : 0; asm volatile("cp.async.ca.shared.global [%0], [%1], 4, %2;" ::"r"(smaddr(sm + r * RS + j)), "l"(src + min(X, 255)), "r"(ok)); } } asm volatile("cp.async.wait_all;" ::: "memory"); __syncthreads(); // rows: two real rows (y, y+44) packed into one complex fft for (int idx = tid; idx < NSIG * 44 * 11; idx += NTS) { const int p = idx % (NSIG * 44), n2 = idx / (NSIG * 44), s = p / 44, y = p % 44; float *ra = sm + (s * 88 + y) * RS, *rb = ra + 44 * RS; cf a[8];#pragma unroll for (int n1 = 0; n1 < 8; ++n1) a[n1] = cf{ra[11 * n1 + n2], rb[11 * n1 + n2]}; // 88 = 8 x 11: radix-8 + twiddle here, radix-11 in the next pass dft8<-1>(a);#pragma unroll for (int k1 = 0; k1 < 8; ++k1) { cf v = a[k1]; if (k1) { const int m = (n2 * k1) % 88; v = rotr(v, twc[m], tws[m]); } ra[11 * k1 + n2] = v.x; rb[11 * k1 + n2] = v.y; } } __syncthreads(); for (int idx = tid; idx < NSIG * 44 * 8; idx += NTS) { const int p = idx >> 3, k1 = idx & 7, s = p / 44, y = p % 44; float *ra = sm + (s * 88 + y) * RS, *rb = ra + 44 * RS; cf b[11];#pragma unroll for (int n2 = 0; n2 < 11; ++n2) b[n2] = cf{ra[11 * k1 + n2], rb[11 * k1 + n2]}; dft11<-1>(b); __syncwarp();#pragma unroll for (int k2 = 0; k2 < 11; ++k2) { ra[k1 + 8 * k2] = b[k2].x; rb[k1 + 8 * k2] = b[k2].y; } } __syncthreads(); // unpack the two real-row spectra (conjugate symmetry), keep 45 bins for (int p = warp; p < NSIG * 44; p += NTS / 32) { const int s = p / 44, y = p % 44; float *ra = sm + (s * 88 + y) * RS, *rb = ra + 44 * RS; cf A[2], B[2];#pragma unroll for (int h = 0; h < 2; ++h) { const int k = lane + 32 * h; if (k < 45) { const int m = (88 - k) % 88; const cf zk{ra[k], rb[k]}, zm{ra[m], rb[m]}; A[h] = cf{0.5f * (zk.x + zm.x), 0.5f * (zk.y - zm.y)}; const cf d{zk.x - zm.x, zk.y + zm.y}; B[h] = cf{0.5f * d.y, -0.5f * d.x}; } } __syncwarp();#pragma unroll for (int h = 0; h < 2; ++h) { const int k = lane + 32 * h; if (k < 45) { ra[2 * k] = A[h].x; ra[2 * k + 1] = A[h].y; rb[2 * k] = B[h].x; rb[2 * k + 1] = B[h].y; } } __syncwarp(); } __syncthreads(); // columns: complex 88-point fft along y for each of the 45 fw for (int idx = tid; idx < NSIG * 45 * 11; idx += NTS) { const int fw = idx % 45, r = idx / 45, s = r % NSIG, n2 = r / NSIG; float *col = sm + s * 88 * RS + 2 * fw; cf a[8];#pragma unroll for (int n1 = 0; n1 < 8; ++n1) { const int n = 11 * n1 + n2; a[n1] = cf{col[n * RS], col[n * RS + 1]}; } dft8<-1>(a);#pragma unroll for (int k1 = 0; k1 < 8; ++k1) { cf v = a[k1]; if (k1) { const int m = (n2 * k1) % 88; v = rotr(v, twc[m], tws[m]); } const int sl = 11 * k1 + n2; col[sl * RS] = v.x; col[sl * RS + 1] = v.y; } } __syncthreads(); for (int idx = tid; idx < NSIG * 45 * 8; idx += NTS) { const int fw = idx % 45, r = idx / 45, s = r % NSIG, k1 = r / NSIG; const float *col = sm + s * 88 * RS + 2 * fw; cf b[11];#pragma unroll for (int n2 = 0; n2 < 11; ++n2) { const int sl = 11 * k1 + n2; b[n2] = cf{col[sl * RS], col[sl * RS + 1]}; } dft11<-1>(b); // z[tile*128 + ci][fh*45 + fw] float2 *dst = z + (size_t)(t * 128 + ci0 + s) * 3960 + fw;#pragma unroll for (int k2 = 0; k2 < 11; ++k2) dst[(k1 + 8 * k2) * 45] = make_float2(b[k2].x, b[k2].y); } __syncthreads();}// 2 blocks per sm, in practice one of each kindextern "C" __global__ void __launch_bounds__(NTS, MINB) combo(const float *__restrict__ x, float2 *__restrict__ z, const float4 *__restrict__ wt, const uint4 *__restrict__ tab, uint4 *__restrict__ tp) { const int g = blockIdx.x / (FPR + 1), p = blockIdx.x % (FPR + 1); if (p < FPR) fwd_body(g * FPR + p, x, z); // even: image fft else rows_body(g, wt, tab, tp); // odd: weight rows}"""# launch 3 (fused_cu): weight column dft + channel mixer in one kernel.# the 519 mb weight spectrum is never written; it lives in smem one chunk at a timeFUSED_SRC = CU_MMA + r"""#define RNADD 0x1000u// one cta = (fw, 8 base rows -> 32 fh, 32 co), 16 warpsstatic __device__ __forceinline__ float rn11(float v) { // round to 11 significant bits: exactly representable in fp16 unsigned u = f2u(v); u = (u + RNADD) & 0xFFFFE000u; return u2f(u);}#define MMX(...) MMA(__VA_ARGS__)#define AR(v) (v).x, (v).y, (v).z, (v).w#define XB(v, mt) ((mt) ? (v).z : (v).x), ((mt) ? (v).w : (v).y)// builds the w tile (weight spectrum) for 16 ci on tensor cores, straight into smem// one pass gives 4 spectrum rows: b, 88-b, b+44, 44-b (even / odd ky symmetry)// so 12 mma per 8 pairs build 32 rowsstatic __device__ __forceinline__ void wgen(const uint4 *__restrict__ tp, const uint4 *tabp, unsigned *sbw, int pbase, int cw, int lane, int w){ const int g = lane >> 2, t = lane & 3; const int TLO = 45 * 16384 * 8; const int wslot = (w & 7) * 4 + t, wreg = w >> 3; #pragma unroll for (int kh = 0; kh < 2; ++kh) { const int p8 = (pbase + cw * 16 + kh * 8 + g) * 8 + t; uint4 tv[2][2]; // [hi / lo][re / im] : ky 8t .. 8t+7 as 4 fp16 pairs tv[0][0] = tp[p8]; tv[1][0] = tp[TLO + p8]; tv[0][1] = tp[p8 + 4]; tv[1][1] = tp[TLO + p8 + 4]; unsigned B[2][2][2][2]; // [hi / lo][re / im][even / odd ky][frag] #pragma unroll for (int hl = 0; hl < 2; ++hl) #pragma unroll for (int pt = 0; pt < 2; ++pt) { const uint4 v = tv[hl][pt]; B[hl][pt][0][0] = (v.x & 0xffffu) | (v.y << 16); B[hl][pt][0][1] = (v.z & 0xffffu) | (v.w << 16); B[hl][pt][1][0] = (v.x >> 16) | (v.y & 0xffff0000u); B[hl][pt][1][1] = (v.z >> 16) | (v.w & 0xffff0000u); } uint4 TA[2][2]; // [even / odd ky][hi / lo] : rows = cos(8 base rows) ; sin(8 base rows) #pragma unroll for (int e = 0; e < 2; ++e) #pragma unroll for (int hl = 0; hl < 2; ++hl) TA[e][hl] = tabp[(e * 2 + hl) * 32]; float P[2][2][4]; // [even / odd][re / im][cos c0 c1, sin c0 c1] #pragma unroll for (int e = 0; e < 2; ++e) #pragma unroll for (int pt = 0; pt < 2; ++pt) #pragma unroll for (int i = 0; i < 4; ++i) P[e][pt][i] = 0.f;// hi*lo, lo*hi, then hi*hi#define WS(e, pt, ahl, bhl) MMX(P[e][pt], AR(TA[e][ahl]), B[bhl][pt][e][0], B[bhl][pt][e][1]) WS(0, 0, 0, 1); WS(1, 0, 0, 1); WS(0, 1, 0, 1); WS(1, 1, 0, 1); WS(0, 0, 1, 0); WS(1, 0, 1, 0); WS(0, 1, 1, 0); WS(1, 1, 1, 0); WS(0, 0, 0, 0); WS(1, 0, 0, 0); WS(0, 1, 0, 0); WS(1, 1, 0, 0);#undef WS #pragma unroll for (int hs = 0; hs < 2; ++hs) { // rows (b, 88-b) : even + odd ; rows (b+44, 44-b) : even - odd float Wv[4][2]; // [w re, w im, mirror re, mirror im][ci 2t, 2t+1] #pragma unroll for (int x = 0; x < 2; ++x) { const float cr = hs ? P[0][0][x] - P[1][0][x] : P[0][0][x] + P[1][0][x]; const float sr = hs ? P[0][0][2 + x] - P[1][0][2 + x] : P[0][0][2 + x] + P[1][0][2 + x]; const float ci = hs ? P[0][1][x] - P[1][1][x] : P[0][1][x] + P[1][1][x]; const float si = hs ? P[0][1][2 + x] - P[1][1][2 + x] : P[0][1][2 + x] + P[1][1][2 + x]; Wv[0][x] = cr + si; Wv[1][x] = ci - sr; Wv[2][x] = cr - si; Wv[3][x] = ci + sr; } #pragma unroll for (int ts = 0; ts < 4; ++ts) { const float v0 = Wv[ts][0], v1 = Wv[ts][1]; // split to fp16 hi / lo, park in smem in mma fragment order const float h0 = rn11(v0), h1 = rn11(v1); unsigned ph, pl; CVT2(ph, h1, h0); CVT2(pl, v1 - h1, v0 - h0); const int r = g + 8 * hs; // = the warp that mixes these two rows const int o = ((r * 8 + ts * 2) * 4 + wreg + 2 * kh) * 32 + ((wslot + 4 * r) & 31); sbw[o] = ph; sbw[o + 128] = pl; } } }}// mixer: one spectrum row, 16 co, all 16 tiles, one chunk of 16 ci#define ALOAD(q) { A[0] = sr[(q) * 128]; A[1] = sr[(q) * 128 + 32]; A[2] = sr[(q) * 128 + 64]; A[3] = sr[(q) * 128 + 96]; }#define MS(dst, xv) MMX(dst[0], A[0], A[1], A[2], A[3], XB(xv, 0)); MMX(dst[1], A[0], A[1], A[2], A[3], XB(xv, 1))static __device__ __forceinline__ void mix(const uint4 xrh, const uint4 xrl, const uint4 xih, const uint4 xil, const uint4 nxh, const uint4 nxl, const unsigned *sr, float (&yr)[2][4], float (&yi)[2][4]){ unsigned A[4]; float lr[2][4], li[2][4]; #pragma unroll for (int b = 0; b < 2; ++b) #pragma unroll for (int i = 0; i < 4; ++i) { lr[b][i] = 0.f; li[b][i] = 0.f; } // hi*hi -> long accumulators (yr, yi). hi*lo + lo*hi -> per-chunk lr, li ALOAD(0); MS(yr, xrh); MS(yi, xih); MS(lr, xrl); MS(li, xil); // w re hi ALOAD(2); MS(yr, xih); MS(yi, nxh); MS(lr, xil); MS(li, nxl); // w im hi ALOAD(1); MS(lr, xrh); MS(li, xih); // w re lo ALOAD(3); MS(lr, xih); MS(li, nxh); // w im lo #pragma unroll for (int b = 0; b < 2; ++b) #pragma unroll for (int i = 0; i < 4; ++i) { yr[b][i] += lr[b][i]; yi[b][i] += li[b][i]; } // small terms join once per chunk, not rounded against the big sum every step}static __device__ __forceinline__ void mixrow(const uint4 *__restrict__ xp, const unsigned *sb, float (&yr)[2][2][4], float (&yi)[2][2][4]){ const uint4 xrh = xp[0], xrl = xp[32], xih = xp[64], xil = xp[96]; uint4 nxh, nxl; // -x by flipping the fp16 sign bits nxh.x = xrh.x ^ 0x80008000u; nxh.y = xrh.y ^ 0x80008000u; nxh.z = xrh.z ^ 0x80008000u; nxh.w = xrh.w ^ 0x80008000u; nxl.x = xrl.x ^ 0x80008000u; nxl.y = xrl.y ^ 0x80008000u; nxl.z = xrl.z ^ 0x80008000u; nxl.w = xrl.w ^ 0x80008000u; // both co blocks share one x load mix(xrh, xrl, xih, xil, nxh, nxl, sb, yr[0], yi[0]); mix(xrh, xrl, xih, xil, nxh, nxl, sb + 16384, yr[1], yi[1]);}#define DOMIX(c) { const unsigned *sb = sm + ((lane + 4 * w) & 31) + w * 1024; \ if (fv) mixrow(xq + ((f0 * 45 + fw) * 1024 + (c) * 128 + lane), sb, yr[0], yi[0]); \ if (pv) mixrow(xq + ((f1 * 45 + fw) * 1024 + (c) * 128 + lane), sb + 512, yr[1], yi[1]); }#define DOW(cw) { wgen(tp, tabp, sm, pbase, cw, lane, w); wgen(tp, tabp, sm + 16384, pbase + 16 * 128, cw, lane, w); }extern "C" __global__ void __launch_bounds__(512, 1) fused_cu(const uint4 *__restrict__ xq, const uint4 *__restrict__ tp, const uint4 *__restrict__ tab, float2 *__restrict__ yf){ extern __shared__ unsigned sm[]; const int tid = threadIdx.x, lane = tid & 31, w = tid >> 5, g = lane >> 2, t = lane & 3; const int id = blockIdx.x, fb = id % 3, cb = (id / 3) & 3, fw = id / 12; // cb: block of 32 co const int bq = fb * 8 + (w & 7), hs = w >> 3; // mixer role: base row and which pair of its four rows const bool fv = bq <= 22, pv = bq >= 1 && bq <= 21; const int f0 = hs ? bq + 44 : bq, f1 = hs ? 44 - bq : 88 - bq; const int pbase = fw * 16384 + (cb * 32 + w) * 128; // cos / sin fragments live in smem to keep registers free for the mixer uint4 *tsm = (uint4 *)(sm + 32768); if (w == 0) { #pragma unroll for (int a = 0; a < 4; ++a) tsm[a * 32 + lane] = tab[(fb * 4 + a) * 32 + lane]; } __syncthreads(); const uint4 *tabp = tsm + lane; float yr[2][2][2][4], yi[2][2][2][4]; // [row / mirror][co block][m tile][frag] #pragma unroll for (int a = 0; a < 2; ++a) #pragma unroll for (int b = 0; b < 2; ++b) #pragma unroll for (int k = 0; k < 2; ++k) #pragma unroll for (int i = 0; i < 4; ++i) { yr[a][b][k][i] = 0.f; yi[a][b][k][i] = 0.f; } // 8 chunks of 16 ci: build w (2 x 64 kb, single buffer), barrier, mix, barrier for (int c = 0; c < 8; ++c) { DOW(c); __syncthreads(); DOMIX(c); __syncthreads(); } #pragma unroll // yf[f][tile][co] for (int fp = 0; fp < 2; ++fp) { if (fp ? pv : fv) { const int f = (fp ? f1 : f0) * 45 + fw; #pragma unroll for (int k = 0; k < 2; ++k) #pragma unroll for (int mt = 0; mt < 2; ++mt) { #pragma unroll for (int i = 0; i < 4; ++i) { const int m = mt * 8 + 2 * t + (i & 1), co = cb * 32 + k * 16 + g + 8 * (i >> 1); float2 v; v.x = yr[fp][k][mt][i]; v.y = yi[fp][k][mt][i]; yf[(f * 16 + m) * 128 + co] = v; } } } }}"""# launch 4 (inv88): inverse fft per (tile, co), crop to the valid 57x57 blockINV_SRC = CU_FFT + r"""template <int SG> __device__ __forceinline__ cf rot(cf a, int m) { float c = C88[m % 88], s = SG * S88[m % 88]; return {a.x * c - a.y * s, a.x * s + a.y * c};}// 88-point fft in registers. 88 = 8 x 11: n = 11 n1 + n2, k = k1 + 8 k2template <int SG, int K1LO = 0, int K1HI = 8, class LD, class ST> __device__ __forceinline__ void fft88(LD ld, ST st) { cf A[88];#pragma unroll for (int n2 = 0; n2 < 11; ++n2) { cf a[8];#pragma unroll for (int n1 = 0; n1 < 8; ++n1) a[n1] = ld(11 * n1 + n2); dft8<SG>(a);#pragma unroll for (int k1 = 0; k1 < 8; ++k1) A[n2 * 8 + k1] = (n2 * k1 == 0) ? a[k1] : rot<SG>(a[k1], n2 * k1); }#pragma unroll for (int k1 = K1LO; k1 < K1HI; ++k1) { cf b[11];#pragma unroll for (int n2 = 0; n2 < 11; ++n2) b[n2] = A[n2 * 8 + k1]; dft11<SG>(b);#pragma unroll for (int k2 = 0; k2 < 11; ++k2) st(k1 + 8 * k2, b[k2]); }}// block = (tile, 4 co). yf [3960][16][128] -> out [128, 225, 225]extern "C" __global__ void __launch_bounds__(256, 1) inv88(const float2 *__restrict__ yf, float *__restrict__ out) { extern __shared__ float sm[]; const int t = blockIdx.x >> 5, co0 = (blockIdx.x & 31) * 4, Y0 = 57 * (t >> 2), X0 = 57 * (t & 3); // columns first, keep only the 57 rows that survive the crop if (threadIdx.x < 180) { const int s = threadIdx.x & 3, fw = threadIdx.x >> 2; const float2 *src = yf + (size_t)fw * 2048 + t * 128 + co0 + s; float *col = sm + s * 57 * RS + 2 * fw; fft88<1>([&](int n) { float2 v = src[(size_t)n * 45 * 2048]; return cf{v.x, v.y}; }, [&](int k, cf v) { if (k < 57) { col[k * RS] = v.x; col[k * RS + 1] = v.y; } }); } __syncthreads(); // rows: rebuild each row from its half spectrum, only 57 rows left to do if (threadIdx.x < 228) { float *row = sm + threadIdx.x * RS; fft88<1>([&](int n) { const int m = n <= 44 ? n : 88 - n; cf v{row[2 * m], row[2 * m + 1]}; if (n == 0 || n == 44) v.y = 0.f; else if (n > 44) v.y = -v.y; return v; }, [&](int k, cf v) { if (k < 57) row[k] = v.x * (1.f / 7744.f); }); // 1 / (88 * 88) } __syncthreads(); // crop: write the valid block, skip anything past 225 for (int i = threadIdx.x; i < 4 * 3249; i += 256) { int s = i / 3249, r = i % 3249, oy = r / 57, ox = r % 57, Y = Y0 + oy, X = X0 + ox; if (Y < 225 && X < 225) out[((co0 + s) * 225 + Y) * 225 + X] = sm[(s * 57 + oy) * RS + ox]; }}"""# launch 1 constants: cos / -sin(2 pi fw kx / 88) as fp16 hi / lo, already in mma a-fragment order@lru_cache(None)def cu_rows_tables(device): m = torch.arange(3).view(3, 1, 1, 1, 1, 1) cs = torch.arange(2).view(1, 2, 1, 1, 1, 1) kb = torch.arange(2).view(1, 1, 2, 1, 1, 1) lane = torch.arange(32).view(1, 1, 1, 32, 1, 1) reg = torch.arange(4).view(1, 1, 1, 1, 4, 1) lo = torch.arange(2).view(1, 1, 1, 1, 1, 2) fw = 16 * m + lane // 4 + 8 * (reg % 2) k = 16 * kb + 2 * (lane % 4) + 8 * (reg // 2) + lo kx = 8 * (k % 8 // 2) + 4 * (k // 16) + 2 * (k // 8 % 2) + k % 2 th = 2 * torch.pi * fw.double() * kx.double() / 88 + 0 * cs v = torch.where((cs == 1).expand_as(th), -th.sin(), th.cos()) v = torch.where((fw < 45).expand_as(th), v, torch.zeros_like(v)).float() h = v.half() return torch.stack((h, (v - h.float()).half()), 2).contiguous().to(device)# launch 3 constants: 16 * cos / sin(2 pi fh ky / 88) as fp16 hi / lo.# x16 here and x/16 in launch 2 keep fp16 in range; powers of two, so exact@lru_cache(None)def cu_tables(device): b = torch.arange(3).view(3, 1, 1, 1, 1) pe = torch.arange(2).view(1, 2, 1, 1, 1) lane = torch.arange(32).view(1, 1, 32, 1, 1) reg = torch.arange(4).view(1, 1, 1, 4, 1) lo = torch.arange(2).view(1, 1, 1, 1, 2) j = 2 * (lane % 4) + 8 * (reg // 2) + lo ky = 8 * (j % 8 // 2) + 4 * (j // 8) + 2 * (j % 2) + pe th = 2 * torch.pi * (b * 8 + lane // 4).double() * ky.double() / 88 v = (16 * torch.where((reg % 2 == 1).expand_as(th), th.sin(), th.cos())).float() h = v.half() return torch.stack((h, (v - h.float()).half()), 2).contiguous().to(device)CUDA_SOURCES = {"combo": PREP_SRC, "fused_cu": FUSED_SRC, "inv88": INV_SRC}# compiled with nvrtc on first use@lru_cache(None)def cuda_kernel(name): return cuda_runtime().build(CUDA_SOURCES[name], name)# launch 2 (gluon): z -> x/16 as 4 fp16 planes (re hi, re lo, im hi, im lo) in the mixer's mma fragment order,# so each lane loads its operands with one aligned 16-byte load# offset(f, m, ci, p) = f*8192 + (ci//16)*1024 + p*256 + (m%8)*32 + ((ci%8)//2)*8 + (m//8)*4 + ((ci%16)//8)*2 + ci%2@g.constexpr_functiondef _xt_layout(): return gl.DistributedLinearLayout([[0, 0, 0, 1], [0, 0, 0, 2], [0, 0, 0, 4], [0, 0, 4, 0], [0, 0, 8, 0]], [[0, 1, 0, 0], [0, 2, 0, 0], [1, 0, 0, 0], [2, 0, 0, 0], [4, 0, 0, 0]], [[0, 0, 1, 0], [0, 0, 2, 0]], [], [8, 4, 16, 8])# hi = fp16(v), lo = fp16(v - hi)@g.jitdef _split16(v): h = v.to(gl.float16) return (h, (v - h.to(gl.float32)).to(gl.float16))@g.jitdef transpose_xf8(X, Y, F: gl.constexpr): src: gl.constexpr = gl.BlockedLayout([1, 1, 2, 1], [1, 1, 8, 4], [2, 2, 1, 1], [2, 3, 1, 0]) dst: gl.constexpr = _xt_layout() c = gl.program_id(0) f0 = gl.program_id(1) * 16 sg = gl.arange(0, 8, layout=gl.SliceLayout(1, gl.SliceLayout(2, gl.SliceLayout(3, src)))) st = gl.arange(0, 4, layout=gl.SliceLayout(0, gl.SliceLayout(2, gl.SliceLayout(3, src)))) sf = f0 + gl.arange(0, 16, layout=gl.SliceLayout(0, gl.SliceLayout(1, gl.SliceLayout(3, src)))) se = gl.arange(0, 8, layout=gl.SliceLayout(0, gl.SliceLayout(1, gl.SliceLayout(2, src)))) row = (sg * 128)[:, None, None, None] + (st * 2)[None, :, None, None] + (se // 4 * 1024 + se // 2 % 2 * 8 + se % 2)[None, None, None, :] + c * 16 v = gl.load(X + row * F + sf[None, None, :, None], sf[None, None, :, None] < F, other=0) v = gl.convert_layout(v, dst) rh, rl = _split16(v.to(gl.uint32).to(gl.float32, bitcast=True) * 0.0625) # x / 16 ih, il = _split16((v >> 32).to(gl.uint32).to(gl.float32, bitcast=True) * 0.0625) dg = gl.arange(0, 8, layout=gl.SliceLayout(1, gl.SliceLayout(2, gl.SliceLayout(3, dst)))) dt = gl.arange(0, 4, layout=gl.SliceLayout(0, gl.SliceLayout(2, gl.SliceLayout(3, dst)))) df = f0 + gl.arange(0, 16, layout=gl.SliceLayout(0, gl.SliceLayout(1, gl.SliceLayout(3, dst)))) de = gl.arange(0, 8, layout=gl.SliceLayout(0, gl.SliceLayout(1, gl.SliceLayout(2, dst)))) o = (dg * 32)[:, None, None, None] + (dt * 8)[None, :, None, None] + (df * 8192)[None, None, :, None] + de[None, None, None, :] + c * 1024 msk = df[None, None, :, None] < F gl.store(Y + o, rh, msk) gl.store(Y + o + 256, rl, msk) gl.store(Y + o + 512, ih, msk) gl.store(Y + o + 768, il, msk)def spectral_conv(x, w, out): rt = cuda_runtime() z = torch.empty((2048, 3960), device=x.device, dtype=torch.complex64) tp = torch.empty((2, 45, 16384, 64), device=w.device, dtype=torch.float16) rt.launch(cuda_kernel("combo"), 4096, 256, (88 * 91 + 176) * 4, pointers(x, z, w, cu_rows_tables(w.device), tp)) # 1: image fft + weight row dft (2048 + 2048 blocks) xq = torch.empty((3960 * 8192,), device=x.device, dtype=torch.float16) transpose_xf8[(8, 248)](z.view(torch.int64), xq, 3960, num_warps=4) # 2: x into fragment order yf = torch.empty((3960, 16, 128), device=x.device, dtype=torch.complex64) rt.launch(cuda_kernel("fused_cu"), 540, 512, 133120, pointers(xq, tp, cu_tables(xq.device), yf)) # 3: weight column dft + mixer, 540 ctas = 45 fw x 4 co blocks x 3 row blocks rt.launch(cuda_kernel("inv88"), 512, 256, 4 * 57 * 91 * 4, pointers(yf, out)) # 4: inverse fft + crop return out# ---------------------------------------------------------------------------------------------------------------def custom_kernel(data): x, w, out = data channels, size, width = x.shape[1], x.shape[-1], w.shape[-1] # ranked shape -> spectral path. everything else only has to pass if tuple(x.shape) == (1, 128, 256, 256) and width == 32: return spectral_conv(x, w, out) if (channels, size) == (64, 128) and width in (8, 16): return overlap_conv(data, 48) if (channels, size, width) == (128, 256, 16): return overlap_conv(data, 80) exact_conv(x, w, out) return out