Skip to content

Commit d9df40d

Browse files
hanwHan Wang
authored andcommitted
tinytpu: column-summed drain for op-46 + matching tests
Controller's do_drain branch for the generic-VPU epilogue now reads array.getResults (column-summed GEMM output, one row per dispatch) instead of array.getMatrix (per-PE rank-1 state). This unlocks the fused op-46 path for standard NN kernels — linear, attention, FFN, RoPE all need 'GEMM-then-elementwise', which is exactly what getResults produces. Controller.bsv: - applyCuratedOpRow / applyIpairRotateRow: 1-row helpers replacing the matrix-shaped ones. The case statement still elaborates once (no G0024 warnings). - Drain branch: reduced = array.getResults; src2_row = src2Reg[0] (caller materializes one VMEM slot per output row, with the coefficients placed at row 0); result_row written to outm[0], outm[1..rows-1] zero-filled. The SXU writeback still reads the full tile so existing read paths keep working. tests/test_tinytpu_backend.py (TestMxuVpuEpilogue): updated expected outputs to row 0 = op(act, src2_row0), lanes COLS..15 = 0. Helper _expected_row keeps the assertion form short. tests/test_e2e_rope_fusion.py: test_rope_runtime_matches_numpy_reference now uses a non-trivial 4x4 activation tile and asserts all 4 dispatch outputs match IPAIR_ROTATE(act_row_r, interleaved_cs_row_r) per row (meaningful lanes 0..3, zeros after). Identity weights so the GEMM result equals the input activation row; the fusion path itself is under test, not the systolic-array correctness. doc/tinyspec_coverage.md: added IPAIR_ROTATE and DISPATCH_MXU_VPU_EPILOGUE to the supported op lists. tinygrad submodule bump: matching lowerer + INTERLEAVE_CS materializer changes (per-(row, tile) src2 slots). Verification: 1231/1231 tests pass (full suite minus pre-existing test_benchmark.py::test_benchmark_runs_add_kernel which is a sim-side cycle-counter issue, not introduced by this work).
1 parent fcfc367 commit d9df40d

5 files changed

Lines changed: 130 additions & 125 deletions

File tree

‎doc/tinyspec_coverage.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ added here or to the sync plan.
5454
`PACKED_I8_SIGN`, `FSIGN`, `ARGMIN`, `ARGMAX`, `CLZ`, `POPCOUNT`, `CTZ`,
5555
`BYTE_REVERSE`, `SAT_ADD_I32`, `SAT_SUB_I32`, `ABS_DIFF_I32`,
5656
`PACKED_I8_ABS_DIFF`, `FABS`, `ROTL`, `ROTR`, `MIN_U32`, `MAX_U32`,
57-
`PAIR_ROTATE`.
57+
`PAIR_ROTATE`, `IPAIR_ROTATE`.
5858

5959
### SXU
6060

@@ -63,4 +63,5 @@ added here or to the sync plan.
6363
`BROADCAST_SCALAR`, `BROADCAST_ROW`, `BROADCAST_COL`,
6464
`DISPATCH_XLU_TRANSPOSE`, `LOAD_VPU_RESULT`, `LOAD_XLU_RESULT`,
6565
`PSUM_WRITE`, `PSUM_ACCUMULATE`, `PSUM_READ`, `DISPATCH_MXU_EPILOGUE`,
66-
`LOAD_EPILOGUE_STAT`, `SET_REQUANT_CONFIG`, `DISPATCH_MXU_REQUANT`.
66+
`LOAD_EPILOGUE_STAT`, `SET_REQUANT_CONFIG`, `DISPATCH_MXU_REQUANT`,
67+
`DISPATCH_MXU_VPU_EPILOGUE`.

‎src/Controller.bsv‎

Lines changed: 62 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -246,51 +246,55 @@ module mkController#(
246246
`endif
247247

248248
// --- Helpers for the generic-VPU MXU epilogue drain branch -------------
249-
// Pulled out of do_drain so the case statement elaborates once, not
250-
// inside the per-cell nested for-loop. (The inlined version pushed
251-
// do_drain past the BSV unfolder's warning threshold.)
252-
function Vector#(rows, Vector#(cols, Int#(32)))
253-
applyCuratedOp(VpuOp op,
254-
Vector#(rows, Vector#(cols, Int#(32))) m,
255-
Vector#(rows, Vector#(cols, Int#(32))) src2);
256-
Vector#(rows, Vector#(cols, Int#(32))) outm = m;
257-
for (Integer ri = 0; ri < valueOf(rows); ri = ri + 1)
258-
for (Integer ci = 0; ci < valueOf(cols); ci = ci + 1) begin
259-
Int#(32) a = m[ri][ci];
260-
Int#(32) b = src2[ri][ci];
261-
Int#(32) v;
262-
case (op)
263-
VPU_ADD: v = a + b;
264-
VPU_SUB: v = a - b;
265-
VPU_MUL: v = a * b;
266-
VPU_MAX: v = (a > b) ? a : b;
267-
VPU_MIN: v = (a < b) ? a : b;
268-
default: v = a;
269-
endcase
270-
outm[ri][ci] = v;
271-
end
272-
return outm;
249+
// Operate on the column-SUMMED GEMM result (array.getResults), not
250+
// the raw per-PE matrix. That lets fused epilogues (bias, residual,
251+
// RoPE, lane-wise activations) work on the actual GEMM output —
252+
// which is what real NN kernels expect — instead of the rank-1
253+
// outer-product the per-PE state holds.
254+
//
255+
// src2 is taken from row 0 of src2Reg (the SXU pre-loads a 1xCOLS
256+
// coefficient row per dispatch into row 0 of the vreg, padding
257+
// other rows with don't-cares).
258+
function Vector#(cols, Int#(32))
259+
applyCuratedOpRow(VpuOp op,
260+
Vector#(cols, Int#(32)) row,
261+
Vector#(cols, Int#(32)) src2);
262+
Vector#(cols, Int#(32)) outr = row;
263+
for (Integer ci = 0; ci < valueOf(cols); ci = ci + 1) begin
264+
Int#(32) a = row[ci];
265+
Int#(32) b = src2[ci];
266+
Int#(32) v;
267+
case (op)
268+
VPU_ADD: v = a + b;
269+
VPU_SUB: v = a - b;
270+
VPU_MUL: v = a * b;
271+
VPU_MAX: v = (a > b) ? a : b;
272+
VPU_MIN: v = (a < b) ? a : b;
273+
default: v = a;
274+
endcase
275+
outr[ci] = v;
276+
end
277+
return outr;
273278
endfunction
274279

275-
// Lane-pair rotation. Adjacent lane pairs (2p, 2p+1) on each row:
276-
// out[r][2p] = drain[r][2p] * src2[r][2p] - drain[r][2p+1] * src2[r][2p+1]
277-
// out[r][2p + 1] = drain[r][2p] * src2[r][2p+1] + drain[r][2p+1] * src2[r][2p]
280+
// Lane-pair rotation on a single row (RoPE primitive):
281+
// out[2p] = row[2p] * src2[2p] - row[2p+1] * src2[2p+1]
282+
// out[2p + 1] = row[2p] * src2[2p+1] + row[2p+1] * src2[2p]
278283
// Matches VPU.bsv's VPU_IPAIR_ROTATE so the lowerer can route via
279284
// either the VPU dispatch or the fused MXU epilogue interchangeably.
280-
function Vector#(rows, Vector#(cols, Int#(32)))
281-
applyIpairRotate(Vector#(rows, Vector#(cols, Int#(32))) m,
282-
Vector#(rows, Vector#(cols, Int#(32))) src2);
283-
Vector#(rows, Vector#(cols, Int#(32))) outm = m;
284-
for (Integer ri = 0; ri < valueOf(rows); ri = ri + 1)
285-
for (Integer p = 0; p < valueOf(cols) / 2; p = p + 1) begin
286-
Int#(32) de = m[ri][2 * p];
287-
Int#(32) doo = m[ri][2 * p + 1];
288-
Int#(32) c = src2[ri][2 * p];
289-
Int#(32) sn = src2[ri][2 * p + 1];
290-
outm[ri][2 * p] = de * c - doo * sn;
291-
outm[ri][2 * p + 1] = de * sn + doo * c;
292-
end
293-
return outm;
285+
function Vector#(cols, Int#(32))
286+
applyIpairRotateRow(Vector#(cols, Int#(32)) row,
287+
Vector#(cols, Int#(32)) src2);
288+
Vector#(cols, Int#(32)) outr = row;
289+
for (Integer p = 0; p < valueOf(cols) / 2; p = p + 1) begin
290+
Int#(32) de = row[2 * p];
291+
Int#(32) doo = row[2 * p + 1];
292+
Int#(32) c = src2[2 * p];
293+
Int#(32) sn = src2[2 * p + 1];
294+
outr[2 * p] = de * c - doo * sn;
295+
outr[2 * p + 1] = de * sn + doo * c;
296+
end
297+
return outr;
294298
endfunction
295299

296300
// Phase 1: Issue weight read request. The next-state depends on
@@ -497,19 +501,23 @@ module mkController#(
497501
epilogueStatBuf <= stat;
498502
end
499503
end else if (genericEpilogueActive) begin
500-
// Lane-wise apply vpuOpReg(drainMatrix, src2Reg) into epilogueBuf.
501-
// Two paths factored into helper functions (defined inside the
502-
// module so they close over `rows`/`cols`):
503-
// - applyCuratedOp: cell-wise subset (ADD/SUB/MUL/MAX/MIN),
504-
// default pass-through. The case statement elaborates once
505-
// inside the function instead of expanding inside the
506-
// per-cell nested for-loop.
507-
// - applyIpairRotate: pair-wise rotation, runs in its own
508-
// branch so it doesn't double-write the curated result.
509-
// A `?:` at the matrix level chooses one path's full output.
510-
epilogueBuf <= (vpuOpReg == VPU_IPAIR_ROTATE)
511-
? applyIpairRotate(drainMatrix, src2Reg)
512-
: applyCuratedOp(vpuOpReg, drainMatrix, src2Reg);
504+
// Column-summed GEMM result + lane-wise epilogue. This produces
505+
// one output row per dispatch (mirroring the legacy chain's
506+
// _load_mxu_result + _vpu_add semantics) so fused epilogues
507+
// apply to the actual GEMM output, not the per-PE rank-1 state.
508+
// src2Reg's row 0 carries the coefficients for this dispatch's
509+
// output row (caller materializes one VMEM slot per output row).
510+
// outm[0] holds the result; outm[1..] = 0 — the consumer reads
511+
// count=COLS lanes from row 0.
512+
let reduced = array.getResults;
513+
Vector#(cols, Int#(32)) src2_row = src2Reg[0];
514+
Vector#(cols, Int#(32)) result_row =
515+
(vpuOpReg == VPU_IPAIR_ROTATE)
516+
? applyIpairRotateRow(reduced, src2_row)
517+
: applyCuratedOpRow(vpuOpReg, reduced, src2_row);
518+
Vector#(rows, Vector#(cols, Int#(32))) outm = replicate(replicate(0));
519+
outm[0] = result_row;
520+
epilogueBuf <= outm;
513521
end else begin
514522
case (psumModeReg)
515523
PSUM_WRITE: begin

‎tests/test_e2e_rope_fusion.py‎

Lines changed: 28 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -180,33 +180,25 @@ def test_rope_lowering_emits_single_bundle(self):
180180
self.assertEqual(cs_entries[0]["s_param"], 4)
181181

182182
def test_rope_runtime_matches_numpy_reference(self):
183-
"""Run the fused bundle through the BSV sim and verify the output
184-
matches a numpy reference IPAIR_ROTATE applied to the per-PE drain
185-
matrix.
186-
187-
Important semantic note: op-46 reads array.getMatrix (per-PE state,
188-
drain[r][c] = act[r] * weight[r][c] for WS tile_len=1), NOT
189-
array.getResults (column-summed GEMM output). The "CODA primitive"
190-
fusion operates elementwise on the per-PE matrix. We use identity
191-
weights here so the per-PE drain is diag(act_vec) — same trick
192-
Slice 5 uses — and assert the IPAIR_ROTATE'd full 16-element tile.
183+
"""Run the fused bundle through the BSV sim. The drain now uses
184+
array.getResults (column-summed GEMM output), so for output row r:
185+
gemm_row_r = act_row_r @ weights
186+
out_row_r = IPAIR_ROTATE(gemm_row_r, interleaved_cs_row_r)
187+
result[r][0..COLS-1] = out_row_r ; rest of the tile is zero.
188+
With identity weights, gemm_row_r == act_row_r, so we compare
189+
against IPAIR_ROTATE(act_row_r, interleaved_cs_row_r) per row.
193190
"""
194191
sim = os.environ.get("TINYTPU_SIM",
195192
"/home/hanwang/p/tinytpu/build/mkTbTinyTPURuntime.bexe")
196193
if not os.path.exists(sim):
197194
self.skipTest(f"sim binary {sim} not built")
198195

199-
# UOp shape implies nv=4 (16 LOADs per operand → 4 act rows worth of
200-
# data). The fused path emits 4 op-46 dispatches, but only dispatch 0
201-
# carries our meaningful activation row at AMEM[0]; rows 1..3 are
202-
# zero-padded so those dispatches produce zero drain. We assert
203-
# against tiles[0].
204-
act_row_0 = np.array([-3, 2, -1, 4], dtype=np.int8)
205-
a_full = np.zeros((4, 4), dtype=np.int8)
206-
a_full[0] = act_row_0
196+
# UOp shape implies nv=4 → 4 op-46 dispatches, one per output row.
197+
rng = np.random.default_rng(123)
198+
a_full = rng.integers(-3, 3, size=(4, 4), dtype=np.int8)
207199
ident_w = np.eye(4, dtype=np.int8)
208-
c_tile = np.arange(1, 17, dtype=np.int32).reshape(4, 4)
209-
s_tile = (np.arange(1, 17, dtype=np.int32) * 2).reshape(4, 4)
200+
c_tile = rng.integers(-4, 4, size=(4, 4), dtype=np.int32)
201+
s_tile = rng.integers(-4, 4, size=(4, 4), dtype=np.int32)
210202

211203
out_buf = bytearray(16 * 4)
212204
a_buf = bytearray(a_full.astype(np.int32).tobytes())
@@ -223,15 +215,22 @@ def test_rope_runtime_matches_numpy_reference(self):
223215
bundle = kernel.build_bundle(bufs)
224216
stdout = run_bundle(sim, bundle)
225217
tiles = parse_multi_vmem_output(stdout)
226-
self.assertTrue(len(tiles) >= 1, f"expected at least 1 vmem_result tile, got {len(tiles)}")
227-
228-
# The first vmem_result tile is the dispatch 0 output. With identity
229-
# weights and a single-vec input, drain = diag(act_row_0).
230-
drain = np.diag(act_row_0).astype(np.int32)
231-
expected_full = _numpy_ipair_rotate(drain, c_tile.reshape(-1), s_tile.reshape(-1))
232-
got_full = np.array(tiles[0], dtype=np.int32).reshape(4, 4)
233-
np.testing.assert_array_equal(got_full, expected_full,
234-
"rope-fused per-PE elementwise output mismatch")
218+
self.assertEqual(len(tiles), 4, f"expected 4 vmem_result tiles, got {len(tiles)}")
219+
220+
# Per output row r: meaningful data in lanes 0..3, zeros in 4..15.
221+
for r in range(4):
222+
gemm_row = a_full[r].astype(np.int32) @ ident_w.astype(np.int32)
223+
c_row = c_tile[r]
224+
s_row = s_tile[r]
225+
expected_row = []
226+
for p in range(2):
227+
ae, ao = int(gemm_row[2 * p]), int(gemm_row[2 * p + 1])
228+
cc, ss = int(c_row[2 * p]), int(s_row[2 * p + 1])
229+
expected_row.append(ae * cc - ao * ss)
230+
expected_row.append(ae * ss + ao * cc)
231+
expected_tile = expected_row + [0] * 12
232+
self.assertEqual(tiles[r], expected_tile,
233+
f"rope-fused output mismatch at row {r}: got {tiles[r]} expected {expected_tile}")
235234

236235
def test_rope_uses_one_sim_invocation(self):
237236
"""Slice 8 guard: the fused RoPE path must use exactly 1 sim call.

‎tests/test_tinytpu_backend.py‎

Lines changed: 36 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -7787,10 +7787,13 @@ def test_fused_residual_guards(self):
77877787
class TestMxuVpuEpilogue(unittest.TestCase):
77887788
"""Slice 5: end-to-end sim coverage of DISPATCH_MXU_VPU_EPILOGUE (op 46).
77897789
7790-
Each test builds a tiny GEMM with identity weights so that the drained
7791-
per-PE matrix is diag(act_vec), then asserts the lane-wise application
7792-
of the curated VPU op against a known src2 tile. Covers ADD/SUB/MUL/
7793-
MAX/MIN + both writeback modes (DST_VMEM and DST_VREG).
7790+
The Controller's drain branch now reads array.getResults (column-summed
7791+
GEMM output) rather than getMatrix (per-PE state) and applies the
7792+
lane-wise op against src2 row 0. With identity weights, getResults
7793+
equals the input activation vector, so the test asserts:
7794+
lane c in [0, COLS): act[c] OP src2_row0[c]
7795+
lanes COLS..15: 0 (Controller zero-fills the other vreg rows)
7796+
Covers ADD/SUB/MUL/MAX/MIN + both writeback modes.
77947797
"""
77957798

77967799
_IDENT_W = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]
@@ -7799,10 +7802,8 @@ class TestMxuVpuEpilogue(unittest.TestCase):
77997802
5, 6, 7, 8,
78007803
9,10,11,12,
78017804
13,14,15,16]
7802-
_DRAIN = [-3, 0, 0, 0,
7803-
0, 2, 0, 0,
7804-
0, 0,-1, 0,
7805-
0, 0, 0, 4]
7805+
_SRC2_ROW0 = _SRC2[:4]
7806+
_GEMM_RESULT = _ACT # identity weights → getResults == act_vec
78067807

78077808
def _run(self, vpu_op_name: str, vmem_dst: bool):
78087809
sim = os.environ["TINYTPU_SIM"]
@@ -7823,54 +7824,50 @@ def _run(self, vpu_op_name: str, vmem_dst: bool):
78237824
out = _run_bundle(sim, _bundle(*lines))
78247825
return _parse_vmem_output(out)
78257826

7827+
def _expected_row(self, op):
7828+
row0 = [op(a, b) for a, b in zip(self._GEMM_RESULT, self._SRC2_ROW0)]
7829+
return row0 + [0] * 12
7830+
78267831
def test_vpu_add_dst_vmem(self):
7827-
got = self._run("ADD", vmem_dst=True)
7828-
expected = [d + s for d, s in zip(self._DRAIN, self._SRC2)]
7829-
self.assertEqual(got, expected)
7832+
self.assertEqual(self._run("ADD", vmem_dst=True),
7833+
self._expected_row(lambda a, b: a + b))
78307834

78317835
def test_vpu_sub_dst_vmem(self):
7832-
got = self._run("SUB", vmem_dst=True)
7833-
expected = [d - s for d, s in zip(self._DRAIN, self._SRC2)]
7834-
self.assertEqual(got, expected)
7836+
self.assertEqual(self._run("SUB", vmem_dst=True),
7837+
self._expected_row(lambda a, b: a - b))
78357838

78367839
def test_vpu_mul_dst_vmem(self):
7837-
got = self._run("MUL", vmem_dst=True)
7838-
expected = [d * s for d, s in zip(self._DRAIN, self._SRC2)]
7839-
self.assertEqual(got, expected)
7840+
self.assertEqual(self._run("MUL", vmem_dst=True),
7841+
self._expected_row(lambda a, b: a * b))
78407842

78417843
def test_vpu_max_dst_vmem(self):
7842-
got = self._run("MAX", vmem_dst=True)
7843-
expected = [max(d, s) for d, s in zip(self._DRAIN, self._SRC2)]
7844-
self.assertEqual(got, expected)
7844+
self.assertEqual(self._run("MAX", vmem_dst=True),
7845+
self._expected_row(max))
78457846

78467847
def test_vpu_min_dst_vmem(self):
7847-
got = self._run("MIN", vmem_dst=True)
7848-
expected = [min(d, s) for d, s in zip(self._DRAIN, self._SRC2)]
7849-
self.assertEqual(got, expected)
7848+
self.assertEqual(self._run("MIN", vmem_dst=True),
7849+
self._expected_row(min))
78507850

78517851
def test_vpu_add_dst_vreg(self):
78527852
# DST_VREG path exercises the alternate writeback branch in
78537853
# do_mxu_vpu_epilogue_wb. Bundle stores the vreg back to VMEM for
78547854
# readback.
7855-
got = self._run("ADD", vmem_dst=False)
7856-
expected = [d + s for d, s in zip(self._DRAIN, self._SRC2)]
7857-
self.assertEqual(got, expected)
7855+
self.assertEqual(self._run("ADD", vmem_dst=False),
7856+
self._expected_row(lambda a, b: a + b))
78587857

78597858
def test_vpu_ipair_rotate_dst_vmem(self):
7860-
# IPAIR_ROTATE: per-row pair rotation.
7861-
# out[r][2p] = d[r][2p] * s[r][2p] - d[r][2p+1] * s[r][2p+1]
7862-
# out[r][2p+1] = d[r][2p] * s[r][2p+1] + d[r][2p+1] * s[r][2p]
7859+
# IPAIR_ROTATE on the column-summed row + src2 row 0:
7860+
# out[2p] = a[2p] * s[2p] - a[2p+1] * s[2p+1]
7861+
# out[2p+1] = a[2p] * s[2p+1] + a[2p+1] * s[2p]
78637862
got = self._run("IPAIR_ROTATE", vmem_dst=True)
7864-
expected = []
7865-
for r in range(4):
7866-
d_row = self._DRAIN[r * 4:(r + 1) * 4]
7867-
s_row = self._SRC2[r * 4:(r + 1) * 4]
7868-
for p in range(2):
7869-
de, do = d_row[2 * p], d_row[2 * p + 1]
7870-
c, sn = s_row[2 * p], s_row[2 * p + 1]
7871-
expected.append(de * c - do * sn)
7872-
expected.append(de * sn + do * c)
7873-
self.assertEqual(got, expected)
7863+
row0 = []
7864+
a, s = self._GEMM_RESULT, self._SRC2_ROW0
7865+
for p in range(2):
7866+
ae, ao = a[2 * p], a[2 * p + 1]
7867+
se, so = s[2 * p], s[2 * p + 1]
7868+
row0.append(ae * se - ao * so)
7869+
row0.append(ae * so + ao * se)
7870+
self.assertEqual(got, row0 + [0] * 12)
78747871

78757872

78767873
if __name__ == "__main__":

0 commit comments

Comments
 (0)