From bb2b526e04de3648981a8b8f8f7479cb8a35cd6b Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 15 Jun 2026 17:21:55 +0000 Subject: [PATCH 01/20] [hipblaslt][tensilelite] Single-hop next-neighbor StreamK work stealing Add StreamKWorkStealing, a codegen-time solution parameter (not a kernel argument) that enables single-hop next-neighbor work stealing in the dynamic-queue StreamK fetch, for both SK4 (StreamKDynamic) and the dynamic sub-path of SK5 (StreamKHybrid). When off (default) the emitted assembly is identical to baseline. When a workgroup's home queue (one of 8 hardcoded queues) is drained, it makes exactly one atomic attempt to claim the structural-extra tile of the next queue ((queueIdx+1) & 0x7), gated so it only fires when there is a tile remainder and the neighbor owns an extra the home does not. To keep tile ownership unique while stealing, the home queue's atomic auto-reset is disabled whenever a remainder exists; the last workgroup restores the 8 queue counters plus a completion counter (placed at synchronizer offset 0x80, clear of both the 256B-strided queue counters and the partials/fixup flag region at 0x800) in kernelEnd. The host already zeroes the whole Synchronizer buffer once at handle creation, so no host-side workspace changes are needed. A lost steal race leaves the index out of range and is absorbed by the existing valid-work-item check. Shared helpers on the StreamK base are reused by SK4 and SK5 (parameterized on the skGrid/SKGrid SGPR). SK5's kernelEnd reset is gated on the hybrid mode bit so it is a no-op in the static (SK3) sub-path. StreamKWorkStealing is rejected unless StreamK in {4,5} and is incompatible with StreamKAtomic. --- .../Tensile/Common/GlobalParameters.py | 1 + .../Tensile/Common/RequiredParameters.py | 1 + .../Tensile/Common/ValidParameters.py | 8 + .../tensilelite/Tensile/Components/StreamK.py | 175 +++++++++++++++++- .../Tensile/SolutionStructs/Solution.py | 15 ++ 5 files changed, 199 insertions(+), 1 deletion(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index 06f652f33d7e..d42e9cfbbfc1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -547,6 +547,7 @@ {"StreamK": [0]}, {"StreamKForceDPOnly": [0]}, {"StreamKAtomic": [0]}, + {"StreamKWorkStealing": [0]}, {"StreamKXCCMapping": [0]}, {"StreamKFixupTreeReduction": [0]}, {"DebugStreamK": [0]}, diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/RequiredParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/RequiredParameters.py index 030625e71684..f81412877a2c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/RequiredParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/RequiredParameters.py @@ -118,6 +118,7 @@ def getRequiredParametersMin() -> set: 'StoreVectorWidth', 'StreamK', 'StreamKForceDPOnly', + 'StreamKWorkStealing', 'StreamKXCCMapping', 'StreamKFixupTreeReduction', 'SuppressNoLoadLoop', diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index b73f1654ee9b..fc8098225e52 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -811,6 +811,14 @@ def makeValidMatrixInstructions(): # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step # 1: uses atomics to accumulate partial tiles "StreamKAtomic": [0, 1], + # Codegen-time toggle for single-hop next-neighbor work stealing in the + # dynamic-queue StreamK fetch (auto-mode SK4 / SK5-dynamic, 8 hardcoded + # queues). When a workgroup's home queue is empty it makes ONE atomic + # attempt to claim the structural extra tile of the next queue. Valid only + # for StreamK in (4, 5). + # 0: off (emitted assembly is byte-for-byte the baseline) + # 1: on + "StreamKWorkStealing": [0, 1], # Enables XCC-based remapping of workgroups, set the value to the number of XCCs # for the device/configuration being used # 0: uses default workgroup assignment diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index c91aaaa93a1b..40649a5469b4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -26,7 +26,7 @@ VOP3PModifiers, ContinuousRegister, DSModifiers from rocisa.instruction import GlobalInv, GlobalWb, SAddCU32, SAddU32, SAndB32, SBarrier, \ SBranch, SCBranchSCC0, SCBranchSCC1, SCMovB32, SCSelectB32, SCmpEQU32, SCmpEQU64, \ - SCmpGtU32, SCmpLeU32, SCmpLtU32, SLShiftLeftB32, SLShiftLeftB64, SLShiftRightB32, VLShiftLeftB32, SLoadB32, \ + SCmpGeU32, SCmpGtU32, SCmpLeU32, SCmpLtU32, SLShiftLeftB32, SLShiftLeftB64, SLShiftRightB32, VLShiftLeftB32, SLoadB32, \ SMaxI32, SMinU32, SMovB32, SMovB64, SMulI32, SNop, SOrB32, SSleep, SStoreB32, SSubU32, \ SWaitCnt, SWaitXCnt, VAddF32, VAddF64, VAddPKF16, VAddU32, VSubU32, VLShiftRightB32, VMovB32, \ VReadfirstlaneB32, VCvtBF16toFP32, BufferLoadB32, BufferStoreB32, SAtomicInc, DSLoadB32, DSStoreB32, \ @@ -315,6 +315,138 @@ def _skv(self, writer, name): """Return the VGPR index holding a StreamK constant.""" return writer.states.skConstVgprs[name] + # ------------------------------------------------------------------ + # Single-hop next-neighbor work stealing (codegen-time, off by default) + # + # Topology: the dynamic-queue fetch (auto-mode SK4 / SK5-dynamic) uses 8 + # hardcoded queues. queueIdx = StreamKIdx & 0x7; per-queue counters live one + # per cache line at AddressFlags + (queueIdx << 8); the global tile index is + # (perQueueRaw << 3) + queueIdx; remainder tiles = TotalItems & 0x7. + # + # AddressFlags buffer layout (per problem), for reference: + # [0x000 .. 0x800) 8 queue counters, one per 256B cache line + # (only the first word of each line is used). + # [0x800 .. ) partials/fixup ready flags (one word per partial tile, + # see storeBranches: offset = (partialIdx << 2) + 256*8). + # + # The work-stealing completion counter is placed at offset 0x80. This is + # collision-free: it is NOT a queue-counter offset (those are multiples of + # 256) and it sits below the flags region that starts at 256*8 = 0x800, so + # it reuses already-reserved, host-zeroed padding inside queue 0's cache-line + # slot. Picking 0x80 (rather than the prior art's numXCDs*256 = 0x800, which + # would collide with the first partials flag) needs no host workspace growth: + # ContractionSolution.cpp already reserves the full 256*8 queue region and + # the Synchronizer buffer is zeroed once at handle creation. + _WS_NUM_QUEUES = 8 + _WS_QUEUE_MASK = 0x7 + _WS_QUEUE_LOG2 = 3 # log2(8) + _WS_COMPLETION_COUNTER_OFFSET = 0x80 + + def streamKWorkStealingHomeNoReset(self, writer, mod, kernel, sBound, mkLabel): + """Disable the home queue's atomic auto-reset when a neighbor could steal. + + When remainder (TotalItems & 0x7) != 0 a neighbor may steal this queue's + structural extra tile, which would throw off the auto-reset count, so the + home counter must run unbounded (kernelEnd resets it instead). Overrides + the precomputed auto-reset bound in sBound with 0xFFFFFFFF in that case; + remainder == 0 keeps the normal bound. Caller must gate on + kernel["StreamKWorkStealing"]. + """ + skKeepBound = mkLabel("SK_FetchHomeQueue") + sRemainder = writer.sgprPool.checkOut(1, "wsRemainder") + mod.add(SAndB32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=self._WS_QUEUE_MASK, comment="Remainder tiles")) + mod.add(SCmpEQU32(src0=sgpr(sRemainder), src1=0, comment="Stealing only helps when remainder != 0")) + mod.add(SCBranchSCC1(labelName=skKeepBound.getLabelName(), comment="No remainder; keep auto-reset bound")) + mod.add(SMovB32(dst=sgpr(sBound), src=hex(0xFFFFFFFF), comment="Disable home auto-reset (neighbor may steal)")) + mod.add(skKeepBound) + writer.sgprPool.checkIn(sRemainder) + + def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, mkLabel): + """Single-hop NEXT-neighbor steal on the 8-queue topology. + + On entry sQueueIdx holds the home queue index and sWorkItemIdx holds the + home global tile index; both must be live. If the home fetch came up + empty (index >= TotalItems) and the next queue (queueIdx+1)&0x7 owns a + structural extra tile that the home queue does not, make exactly ONE + atomic (auto-reset disabled) against that neighbor and recompute the + global index. A lost race leaves sWorkItemIdx >= TotalItems so the + existing downstream valid-index check turns this WG into a no-op. + sQueueIdx is clobbered. Caller must gate on kernel["StreamKWorkStealing"]. + """ + skFetchDone = mkLabel("SK_FetchDone") + mod.add(SCmpLtU32(src0=sgpr(sWorkItemIdx), src1=sgpr("TotalItems"), comment="Home fetch valid?")) + mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Valid work fetched; no steal")) + + sRemainder = writer.sgprPool.checkOut(1, "wsRemainder") + mod.add(SAndB32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=self._WS_QUEUE_MASK, comment="Remainder tiles")) + mod.add(SCmpEQU32(src0=sgpr(sRemainder), src1=0, comment="Stealing only helps when remainder != 0")) + mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="No remainder; skip steal")) + mod.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Home already owns a structural extra?")) + mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Home has extra; no fair steal")) + + # Walk to the immediate next queue (wrap within the 8-queue ring). + mod.add(SAddU32(dst=sgpr(sQueueIdx), src0=sgpr(sQueueIdx), src1=1, comment="Next queue")) + mod.add(SAndB32(dst=sgpr(sQueueIdx), src0=sgpr(sQueueIdx), src1=self._WS_QUEUE_MASK, comment="Wrap queue index")) + mod.add(SCmpGeU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Neighbor has no structural extra?")) + mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Neighbor has no extra; skip steal")) + + # One atomic on the neighbor's counter, auto-reset disabled. + sAddress = writer.sgprPool.checkOutAligned(2, 2, "wsStealAddress") + mod.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), shiftHex=log2(256), comment="Stride queues to cache lines (stolen queue)")) + mod.add(SAddU32(dst=sgpr(sAddress+0), src0=sgpr(sAddress+0), src1=sgpr("AddressFlags+0"))) + mod.add(SAddCU32(dst=sgpr(sAddress+1), src0=0, src1=sgpr("AddressFlags+1"))) + mod.add(SMovB32(dst=sgpr(sWorkItemIdx), src=hex(0xFFFFFFFF), comment="Disable atomic auto-reset")) + mod.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), comment="Fetch stolen work item index")) + mod.add(SWaitCnt(kmcnt=0, comment="Wait for scalar memory op")) + writer.sgprPool.checkIn(sAddress) + # Recompute global tile index from the neighbor's queue. + mod.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=self._WS_QUEUE_LOG2)) + mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) + mod.add(skFetchDone) + writer.sgprPool.checkIn(sRemainder) + + def streamKWorkStealingKernelEndReset(self, writer, kernel, sGridName, mkLabel): + """Last WG to finish zeroes the 8 queue counters + the completion counter. + + Required only when work stealing disabled the home auto-reset: the + Synchronizer buffer is zeroed by the host just once (at handle creation), + so the kernel must restore the counters to 0 itself for the next launch. + Wave 0 atomically counts completed WGs against the completion counter at + offset 0x80 (auto-reset disabled); the WG that observes the full grid + count (sGridName) performs the resets. Caller must gate on + kernel["StreamKWorkStealing"]. sGridName is "skGrid" (SK4) or "SKGrid" + (SK5-dynamic). + """ + module = Module("StreamK work-stealing kernelEnd reset") + completionOffset = self._WS_COMPLETION_COUNTER_OFFSET + skExit = mkLabel("SK_WSResetExit") + + module.add(SBarrier(comment="Wait for all waves before completion count")) + sWave = writer.sgprPool.checkOut(1, "wsWave") + module.add(VReadfirstlaneB32(dst=sgpr(sWave), src=vgpr("Serial"), comment="Wave 0 handles completion")) + module.add(SCmpEQU32(src0=sgpr(sWave), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skExit.getLabelName(), comment="Only wave 0 counts completed WGs")) + writer.sgprPool.checkIn(sWave) + + sCompleted = writer.sgprPool.checkOut(1, "wsCompletedWGs") + module.add(SMovB32(dst=sgpr(sCompleted), src=hex(0xFFFFFFFF), comment="Disable completion counter auto-reset")) + module.add(SAtomicInc(dst=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=completionOffset, smem=SMEMModifiers(glc=True), comment="Count completed workgroups")) + module.add(SWaitCnt(kmcnt=0, comment="Wait for scalar memory op")) + module.add(SAddU32(dst=sgpr(sCompleted), src0=sgpr(sCompleted), src1=1, comment="Completed WGs including this WG")) + module.add(SCmpEQU32(src0=sgpr(sCompleted), src1=sgpr(sGridName), comment="All workgroups completed?")) + module.add(SCBranchSCC0(labelName=skExit.getLabelName(), comment="Not the last WG; skip reset")) + + module.add(SMovB32(dst=sgpr(sCompleted), src=0, comment="Zero value for synchronizer resets")) + # Per-queue counters are one-per-cache-line (256B apart) so they cannot + # be coalesced into a wide store; keep separate b32 stores. + for queueIdx in range(self._WS_NUM_QUEUES): + module.add(SStoreB32(src=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=queueIdx * 256, smem=SMEMModifiers(glc=True), comment="Reset queue counter")) + module.add(SStoreB32(src=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=completionOffset, smem=SMEMModifiers(glc=True), comment="Reset completion counter")) + module.add(SWaitCnt(kmcnt=0, comment="Wait for synchronizer reset")) + module.add(skExit) + writer.sgprPool.checkIn(sCompleted) + return module + @abc.abstractmethod def preLoop(self, writer, kernel): pass @@ -3196,6 +3328,10 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): writer.sgprPool.checkIn(sTilesInQueue) writer.sgprPool.checkIn(sWorkgroupsInQueue) + # Work stealing: a stolen home queue must not auto-reset its counter. + if kernel["StreamKWorkStealing"]: + self.streamKWorkStealingHomeNoReset(writer, module, kernel, sWorkItemIdx, lambda base: Label(base, "")) + # Fetch next work item module.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), comment="Fetch next work item index")) # module.add(SMovB32(dst=sgpr(sWorkItemIdx), src=sgpr("WorkGroup0"))) @@ -3205,6 +3341,11 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): # Convert to global work item index module.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=log2(8))) module.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) + + # Work stealing: if the home queue was empty, attempt one steal from the + # next neighbor before sharing the index with the rest of the waves. + if kernel["StreamKWorkStealing"]: + self.streamKWorkStealingSteal(writer, module, kernel, sQueueIdx, sWorkItemIdx, lambda base: Label(base, "")) writer.sgprPool.checkIn(sQueueIdx) # Share work item index with all waves @@ -3488,6 +3629,12 @@ def kernelEnd(self, writer, kernel): # TODO will need to reset the rest of the synchronizer if tiles were split # Remaining reset can be done if workitem = grid + total - 1 + # Work stealing disables the home auto-reset, so the last WG must restore + # the queue + completion counters for the next launch (the Synchronizer + # is zeroed by the host only once). + if kernel["StreamKWorkStealing"]: + module.add(self.streamKWorkStealingKernelEndReset(writer, kernel, "skGrid", lambda base: Label(base, ""))) + return module def _extract_hybrid_mode(): @@ -3837,6 +3984,11 @@ def emitDynamicGRA(mod): writer.sgprPool.checkIn(sTilesInQueue) writer.sgprPool.checkIn(sWorkgroupsInQueue) + # Work stealing: a stolen home queue must not auto-reset its counter. + if kernel["StreamKWorkStealing"]: + self.streamKWorkStealingHomeNoReset(writer, mod, kernel, sWorkItemIdx, + lambda base: Label(writer.labels.getNameInc(base), "")) + mod.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), comment="Fetch next work item index")) @@ -3848,6 +4000,12 @@ def emitDynamicGRA(mod): shiftHex=log2(8))) mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) + + # Work stealing: if the home queue was empty, attempt one steal from + # the next neighbor before sharing the index with the rest of the waves. + if kernel["StreamKWorkStealing"]: + self.streamKWorkStealingSteal(writer, mod, kernel, sQueueIdx, sWorkItemIdx, + lambda base: Label(writer.labels.getNameInc(base), "")) writer.sgprPool.checkIn(sQueueIdx) # Share work item index with all waves @@ -4288,6 +4446,21 @@ def routeToGeneralBatchedOrStridedBatched(self, stridedBatchedGemmLoad, generalB def kernelEnd(self, writer, kernel): module = Module("StreamK Hybrid kernelEnd") + + # Work stealing only exists on the dynamic (SK4) sub-path; the static + # (SK3) sub-path has no per-XCD queue counters to reset. Gate the reset + # on the runtime mode bit so it is harmless when SK5 runs in static mode. + if kernel["StreamKWorkStealing"]: + def emitDynamicReset(mod): + mod.add(self.streamKWorkStealingKernelEndReset( + writer, kernel, "SKGrid", + lambda base: Label(writer.labels.getNameInc(base), ""))) + + def emitStaticReset(mod): + pass + + self._emitSk3Sk4Branch(writer, module, "WSReset", emitDynamicReset, emitStaticReset) + return module diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 89d51a3dd92a..5251bafc9aa3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1639,6 +1639,19 @@ def assignDerivedParameters( reject(state, printRejectionReason, "DebugPersistentKernelLoopForever requires StreamK in {1,2,3} (got %d)" % state["StreamK"]) + if state["StreamKWorkStealing"]: + # Work stealing only exists in the dynamic-queue fetch (auto-mode + # SK4 and the SK4 sub-path of SK5). + if state["StreamK"] not in (4, 5): + reject(state, printRejectionReason, + "StreamKWorkStealing requires StreamK in {4,5} (got %d)" + % state["StreamK"]) + # Stealing was designed/validated against the non-atomic + # partials+fixup path. Atomic SK4/SK5 is already rejected above; keep + # this explicit guard so the combination can never slip through. + if state["StreamKAtomic"]: + reject(state, printRejectionReason, + "StreamKWorkStealing is not supported with StreamKAtomic") if not state["Valid"]: print2("in assignDerivedParameters, state['Valid'] = False") return @@ -1646,6 +1659,7 @@ def assignDerivedParameters( # If not using StreamK, clear other stream-k settings to avoid duplicate kernels state["StreamKForceDPOnly"] = 0 state["StreamKAtomic"] = 0 + state["StreamKWorkStealing"] = 0 state["StreamKXCCMapping"] = 0 state["StreamKFixupTreeReduction"] = 0 state["DebugStreamK"] = 0 @@ -2614,6 +2628,7 @@ def evaluateExpertSchedulingMode() -> Literal[0, 1, 2]: "GroupLoadStore": not state["GroupLoadStore"], "StreamK": not state["StreamK"], "StreamKAtomic": not state["StreamKAtomic"], + "StreamKWorkStealing": not state["StreamKWorkStealing"], "StreamKXCCMapping": not state["StreamKXCCMapping"], "StreamKFixupTreeReduction": not state["StreamKFixupTreeReduction"], "DebugStreamK": not state["DebugStreamK"], From 948a5a1a3880ef1221055d4d21140734931f4e41 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 15 Jun 2026 17:37:00 +0000 Subject: [PATCH 02/20] [hipblaslt][tensilelite] Tests for single-hop StreamK work stealing Unit tests (test_streamk_work_stealing.py): assert the StreamKWorkStealing solution parameter exists, that the steal/no-reset/kernelEnd-reset assembly is emitted by the new StreamK helpers, and that Solution validation rejects the parameter outside StreamK in {4,5} and with StreamKAtomic. GPU YAML tests (sk_dynamic_work_stealing.yaml for SK4, sk_hybrid_work_stealing.yaml for SK5): fork StreamKWorkStealing over [0,1] (and StreamKHybridMode over [0,1] for SK5) with NumElementsToValidate=-1 over aligned and unaligned/batched shapes (640x640, 384x384x3) so the steal path and the kernelEnd counter-reset across repeated launches are exercised. Validated on gfx950: all off/on and hybrid-mode variants PASSED. --- .../streamk/sk_dynamic_work_stealing.yaml | 104 +++++ .../streamk/sk_hybrid_work_stealing.yaml | 110 +++++ .../Tests/unit/test_streamk_work_stealing.py | 409 ++++++++++++++++++ 3 files changed, 623 insertions(+) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml new file mode 100644 index 000000000000..687c2933e854 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml @@ -0,0 +1,104 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201, skip-gfx1250] # not supported by arch + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + MaxWorkspaceSize: 134217728 + NumWarmups: 0 + EnqueuesPerSync: 1 + SleepPercent: 50 + +# Shared SK4-dynamic fork entries (single-value) plus common benchmark params. +# StreamKWorkStealing is forked over [0, 1] so reference-vs-kernel validation +# runs for BOTH the stealing-off (must match baseline asm) and stealing-on +# paths in a single invocation. +.SkDynamicCommonFork: &sk_dynamic_common_fork + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + - 1LDSBuffer: [1] + - ExpandPointerSwap: [False] + - GlobalSplitU: [0] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + - StreamK: [4] + - StreamKWorkStealing: [0, 1] + - TransposeLDS: [0] + - WorkGroupMapping: [1] + +.SkDynamicBpsg: &sk_dynamic_bpsg + InitialSolutionParameters: + BenchmarkCommonParameters: *sk_dynamic_common_fork + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + # Multiple sizes / repeated launches exercise the kernelEnd counter-reset + # path across launches that reuse the same (host-zeroed-once) Synchronizer. + # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0) + # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (remainder != 0, steal fires) + # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) + - ProblemSizes: + - Exact: [512, 512, 1, 512] + - Exact: [640, 640, 1, 512] + - Exact: [384, 384, 3, 256] + +BenchmarkProblems: + + - # SGEMM NT - StreamK=4 dynamic with single-hop work stealing + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # SK4 dynamic - small MI matrix to keep kernel-object size bounded + <<: *sk_dynamic_bpsg + ForkParameters: + - DepthU: [16] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [1] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + - [32, 32, 2, 1, 1, 2,2, 2,2] + - [16, 16, 4, 1, 1, 2,2, 2,2] + - PrefetchLocalRead: [1] + - VectorWidthA: [1] + - VectorWidthB: [1] + + - # HGEMM NT - StreamK=4 dynamic with single-hop work stealing + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + + - # SK4 dynamic - small MI matrix + <<: *sk_dynamic_bpsg + ForkParameters: + - DepthU: [64] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - PrefetchLocalRead: [1] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml new file mode 100644 index 000000000000..805c912aa4a1 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml @@ -0,0 +1,110 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201, skip-gfx1250] # SK5 requires MFMA + gfx94x/950 + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + MaxWorkspaceSize: 134217728 + NumWarmups: 0 + EnqueuesPerSync: 1 + SleepPercent: 50 + # Exercise the two deterministic SK5 hybrid-mode paths in one run: + # 0 -> static work assignment (SK3 sub-path) + # 1 -> dynamic per-queue work-queue path (SK4 sub-path; work stealing lives here) + # Crossed with StreamKWorkStealing [0,1] below this validates: + # static + steal-off, static + steal-on (steal is a no-op / harmless reset), + # dynamic + steal-off, dynamic + steal-on (steal actually fires). + StreamKHybridMode: [0, 1] + +# Shared SK5-hybrid fork entries (single-value) plus common benchmark params. +# StreamKWorkStealing is forked over [0, 1] so reference-vs-kernel validation +# runs for both stealing-off and stealing-on across both hybrid sub-paths. +.SkHybridCommonFork: &sk_hybrid_common_fork + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + - 1LDSBuffer: [1] + - ExpandPointerSwap: [False] + - GlobalSplitU: [0] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + - StreamK: [5] + - StreamKWorkStealing: [0, 1] + - TransposeLDS: [0] + - WorkGroupMapping: [1] + +.SkHybridBpsg: &sk_hybrid_bpsg + InitialSolutionParameters: + BenchmarkCommonParameters: *sk_hybrid_common_fork + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + # Multiple sizes / repeated launches exercise the kernelEnd counter-reset + # path across launches that reuse the same (host-zeroed-once) Synchronizer. + # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0) + # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (remainder != 0, steal fires) + # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) + - ProblemSizes: + - Exact: [512, 512, 1, 512] + - Exact: [640, 640, 1, 512] + - Exact: [384, 384, 3, 256] + +BenchmarkProblems: + + - # SGEMM NT - StreamK=5 hybrid with single-hop work stealing (dynamic sub-path) + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # SK5 hybrid - small MI matrix to keep kernel-object size bounded + <<: *sk_hybrid_bpsg + ForkParameters: + - DepthU: [16] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [1] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + - [32, 32, 2, 1, 1, 2,2, 2,2] + - [16, 16, 4, 1, 1, 2,2, 2,2] + - PrefetchLocalRead: [1] + - VectorWidthA: [1] + - VectorWidthB: [1] + + - # HGEMM NT - StreamK=5 hybrid with single-hop work stealing + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + + - # SK5 hybrid - small MI matrix + <<: *sk_hybrid_bpsg + ForkParameters: + - DepthU: [64] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - PrefetchLocalRead: [1] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py new file mode 100644 index 000000000000..6535327e6619 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -0,0 +1,409 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Unit tests for the single-hop StreamK work-stealing codegen. + +These tests assert that the new work-stealing assembly is emitted by the +helper methods on the ``StreamK`` base class, and -- crucially -- that those +helpers are only ever reached behind the codegen-time ``StreamKWorkStealing`` +toggle. Following the StreamK=5 hybrid tests, they import rocisa instructions +and inspect emitted modules rather than matching source text; the toggle gating +and the Solution-level validation are verified by executing the *real* source +(via the AST) so the assertions track the actual code, not a copy of it. +""" + +import ast +import inspect +import textwrap + +import pytest + +# Prime the component registry before StreamK imports (avoids circular import). +from Tensile.KernelWriterAssembly import KernelWriterAssembly # noqa: F401 + +from rocisa.code import Module, Label +from rocisa.instruction import ( + SAddU32, + SAndB32, + SAtomicInc, + SBarrier, + SCBranchSCC0, + SCBranchSCC1, + SCmpGeU32, + SCmpLtU32, + SMovB32, + SStoreB32, +) + +from Tensile.Common.ValidParameters import validParameters +from Tensile.Components.StreamK import ( + StreamK, + StreamKDynamic, + StreamKHybrid, + streamKVariantClass, +) +from Tensile.SolutionStructs import Solution +from Tensile.SolutionStructs.Utilities import reject + + +# --------------------------------------------------------------------------- +# Fakes: just enough of a "writer" for the standalone helper methods. +# +# The three helpers only touch ``writer.sgprPool`` (checkOut / checkOutAligned +# / checkIn) and emit rocisa instructions via free functions (sgpr/vgpr), so a +# tiny pool that hands out monotonically increasing register indices is all +# that is required -- no KernelWriter, no GPU. +# --------------------------------------------------------------------------- +class _FakeSgprPool: + def __init__(self, start: int = 100): + self._next = start + + def checkOut(self, n: int, name: str = "", *args, **kwargs) -> int: + reg = self._next + self._next += n + return reg + + def checkOutAligned(self, n: int, align: int, name: str = "", *args, **kwargs) -> int: + if self._next % align: + self._next += align - (self._next % align) + reg = self._next + self._next += n + return reg + + def checkIn(self, *args, **kwargs): + return None + + +class _FakeWriter: + def __init__(self): + self.sgprPool = _FakeSgprPool() + + +def _mk_label(base: str) -> Label: + return Label(base, "") + + +def _stream_k_instance(streamk: int) -> StreamK: + """A concrete StreamK variant (helpers live on the base class).""" + return streamKVariantClass(streamk)() + + +def _imm_in(inst, value: int) -> bool: + """True if *inst* carries *value* as an immediate operand. + + rocisa renders immediates inconsistently -- ints passed straight through + print as decimal ("7"), while values passed as ``hex(...)`` print as + "0x..." -- so normalise every param through ``int(p, 0)`` and compare. + """ + for p in inst.getParams(): + try: + if int(str(p), 0) == value: + return True + except (TypeError, ValueError): + continue + return False + + +def _flat(module: Module) -> list: + return list(module.flatitems()) + + +# --------------------------------------------------------------------------- +# AST helpers: read the *real* StreamK / Solution source and reason about it. +# --------------------------------------------------------------------------- +def _const_slice(subscript: ast.Subscript): + s = subscript.slice + if isinstance(s, ast.Constant): + return s.value + return None + + +def _is_subscript_on(node, name: str, key: str) -> bool: + return ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == name + and _const_slice(node) == key + ) + + +def _ws_guarded_calls(func) -> set: + """Names of ``self.streamKWorkStealing*`` calls that sit inside an + ``if kernel["StreamKWorkStealing"]:`` block in *func* (recursing into + nested closures).""" + tree = ast.parse(textwrap.dedent(inspect.getsource(func))) + guarded = set() + for node in ast.walk(tree): + if isinstance(node, ast.If) and _is_subscript_on( + node.test, "kernel", "StreamKWorkStealing" + ): + for sub in ast.walk(node): + if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Attribute): + if sub.func.attr.startswith("streamKWorkStealing"): + guarded.add(sub.func.attr) + return guarded + + +def _all_ws_calls(func) -> set: + """Every ``self.streamKWorkStealing*`` call in *func*, guarded or not.""" + tree = ast.parse(textwrap.dedent(inspect.getsource(func))) + calls = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr.startswith("streamKWorkStealing"): + calls.add(node.func.attr) + return calls + + +def _extract_ws_validation(): + """Compile the *real* ``if state["StreamKWorkStealing"]:`` block out of + ``Solution.assignDerivedParameters`` into a standalone callable so the + actual rejection logic can be exercised without a full Solution state.""" + tree = ast.parse( + textwrap.dedent(inspect.getsource(Solution.assignDerivedParameters)) + ) + target = None + for node in ast.walk(tree): + if isinstance(node, ast.If) and _is_subscript_on( + node.test, "state", "StreamKWorkStealing" + ): + target = node + break + assert target is not None, "could not find StreamKWorkStealing validation block" + + func = ast.FunctionDef( + name="_validate", + args=ast.arguments( + posonlyargs=[], + args=[ast.arg("state"), ast.arg("printRejectionReason"), ast.arg("reject")], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ), + body=[target], + decorator_list=[], + returns=None, + type_params=[], + ) + mod = ast.Module(body=[func], type_ignores=[]) + ast.fix_missing_locations(mod) + ns: dict = {} + exec(compile(mod, "", "exec"), ns) + return ns["_validate"] + + +# =========================================================================== +# 1. ValidParameters: the codegen-time toggle exists and is boolean. +# =========================================================================== +class TestValidParameters: + def test_work_stealing_param_exists(self): + assert "StreamKWorkStealing" in validParameters + + def test_work_stealing_param_is_zero_one(self): + assert validParameters["StreamKWorkStealing"] == [0, 1] + + +# =========================================================================== +# 2. The three new StreamK helper methods exist and are callable. +# =========================================================================== +class TestHelperMethodsExist: + @pytest.mark.parametrize( + "name", + [ + "streamKWorkStealingHomeNoReset", + "streamKWorkStealingSteal", + "streamKWorkStealingKernelEndReset", + ], + ) + def test_method_is_defined_on_base(self, name): + assert callable(getattr(StreamK, name)) + + +# =========================================================================== +# 3a. Presence: the helpers actually emit the work-stealing assembly. +# =========================================================================== +class TestHomeNoResetEmission: + """Disabling the home auto-reset must mask TotalItems with 0x7 and, when a + remainder exists, move 0xFFFFFFFF into the auto-reset bound register.""" + + def _emit(self): + sk = _stream_k_instance(4) + writer = _FakeWriter() + module = Module("home-no-reset") + sBound = writer.sgprPool.checkOut(1, "bound") + sk.streamKWorkStealingHomeNoReset(writer, module, {}, sBound, _mk_label) + return _flat(module) + + def test_masks_total_items_with_queue_mask(self): + items = self._emit() + masks = [ + i for i in items + if isinstance(i, SAndB32) and _imm_in(i, 0x7) + ] + assert masks, "expected an s_and_b32 with the 0x7 queue mask" + + def test_disables_auto_reset_with_all_ones(self): + items = self._emit() + movs = [ + i for i in items + if isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) + ] + assert movs, "expected 0xFFFFFFFF mov to disable the home auto-reset" + + +class TestStealEmission: + """One single-hop NEXT steal: walk to (queueIdx+1) & 0x7, guard against the + neighbor having no structural extra, then a single auto-reset-disabled + atomic increment against the stolen queue.""" + + def _emit(self): + sk = _stream_k_instance(4) + writer = _FakeWriter() + module = Module("steal") + sQueueIdx = writer.sgprPool.checkOut(1, "queueIdx") + sWorkItemIdx = writer.sgprPool.checkOut(1, "workItemIdx") + sk.streamKWorkStealingSteal( + writer, module, {}, sQueueIdx, sWorkItemIdx, _mk_label + ) + return _flat(module) + + def test_neighbor_walk_is_plus_one_then_wrap(self): + items = self._emit() + # +1 to advance to the next neighbor ... + assert any( + isinstance(i, SAddU32) and _imm_in(i, 1) for i in items + ), "expected +1 advance to the next queue" + # ... wrapped within the 8-queue ring via & 0x7. + assert any( + isinstance(i, SAndB32) and _imm_in(i, 0x7) for i in items + ), "expected (queueIdx+1) & 0x7 wrap" + + def test_skips_when_neighbor_has_no_extra(self): + items = self._emit() + assert any(isinstance(i, SCmpGeU32) for i in items), ( + "expected a >= remainder guard so a neighbor without a structural " + "extra is not robbed" + ) + + def test_exactly_one_atomic_increment(self): + items = self._emit() + atomics = [i for i in items if isinstance(i, SAtomicInc)] + assert len(atomics) == 1, "single-hop steal must emit exactly one atomic" + + def test_atomic_uses_auto_reset_disabled_bound(self): + items = self._emit() + assert any( + isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) + for i in items + ), "the stolen atomic must run with auto-reset disabled (0xFFFFFFFF)" + + def test_guards_on_valid_home_fetch(self): + # A valid home fetch (index < TotalItems) must short-circuit the steal. + items = self._emit() + assert any(isinstance(i, SCmpLtU32) for i in items) + assert any(isinstance(i, SCBranchSCC1) for i in items) + + +class TestKernelEndResetEmission: + """The last WG zeroes the 8 per-queue counters plus the completion counter, + behind a barrier + wave-0 completion count.""" + + def _emit(self): + sk = _stream_k_instance(4) + writer = _FakeWriter() + module = sk.streamKWorkStealingKernelEndReset(writer, {}, "skGrid", _mk_label) + return _flat(module) + + def test_starts_with_barrier(self): + items = self._emit() + assert any(isinstance(i, SBarrier) for i in items) + + def test_resets_eight_queues_plus_completion_counter(self): + items = self._emit() + stores = [i for i in items if isinstance(i, SStoreB32)] + assert len(stores) == StreamK._WS_NUM_QUEUES + 1 == 9, ( + "expected 8 per-queue counter resets + 1 completion counter reset" + ) + + def test_completion_count_uses_atomic_inc(self): + items = self._emit() + assert any(isinstance(i, SAtomicInc) for i in items), ( + "wave 0 counts completed WGs via an atomic increment" + ) + + def test_only_last_wg_resets(self): + # SCBranchSCC0 guards (a) wave-0-only and (b) last-WG-only. + items = self._emit() + assert sum(isinstance(i, SCBranchSCC0) for i in items) >= 2 + + +# =========================================================================== +# 3b. Absence-by-toggle: the helpers are only reached behind the +# ``kernel["StreamKWorkStealing"]`` gate at every callsite. Verified +# against the real source so "off" provably emits nothing extra. +# =========================================================================== +class TestCallsitesAreToggleGated: + def test_sk4_grawg_steal_calls_are_all_gated(self): + guarded = _ws_guarded_calls(StreamKDynamic.graWorkGroup) + allcalls = _all_ws_calls(StreamKDynamic.graWorkGroup) + assert {"streamKWorkStealingHomeNoReset", "streamKWorkStealingSteal"} <= guarded + # Nothing slips through ungated. + assert allcalls == guarded + + def test_sk4_kernelend_reset_is_gated(self): + guarded = _ws_guarded_calls(StreamKDynamic.kernelEnd) + allcalls = _all_ws_calls(StreamKDynamic.kernelEnd) + assert "streamKWorkStealingKernelEndReset" in guarded + assert allcalls == guarded + + def test_sk5_grawg_steal_calls_are_all_gated(self): + guarded = _ws_guarded_calls(StreamKHybrid.graWorkGroup) + allcalls = _all_ws_calls(StreamKHybrid.graWorkGroup) + assert {"streamKWorkStealingHomeNoReset", "streamKWorkStealingSteal"} <= guarded + assert allcalls == guarded + + def test_sk5_kernelend_reset_is_gated(self): + guarded = _ws_guarded_calls(StreamKHybrid.kernelEnd) + allcalls = _all_ws_calls(StreamKHybrid.kernelEnd) + assert "streamKWorkStealingKernelEndReset" in guarded + assert allcalls == guarded + + +# =========================================================================== +# 4. Solution validation: the real rejection logic from +# assignDerivedParameters, executed in isolation. +# =========================================================================== +class TestSolutionValidation: + def setup_method(self): + self.validate = _extract_ws_validation() + + def _run(self, *, streamk, atomic, work_stealing=1): + state = { + "StreamKWorkStealing": work_stealing, + "StreamK": streamk, + "StreamKAtomic": atomic, + } + self.validate(state, False, reject) + return state + + @pytest.mark.parametrize("streamk", [0, 1, 2, 3]) + def test_rejected_when_streamk_not_4_or_5(self, streamk): + state = self._run(streamk=streamk, atomic=0) + assert state["Valid"] is False + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_accepted_for_dynamic_and_hybrid_without_atomic(self, streamk): + state = self._run(streamk=streamk, atomic=0) + assert state.get("Valid", True) is True + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_rejected_with_atomic(self, streamk): + state = self._run(streamk=streamk, atomic=1) + assert state["Valid"] is False + + def test_off_toggle_is_inert_even_for_bad_combo(self): + # With the toggle off the guard must not fire, even for a combination + # that would otherwise be rejected. + state = self._run(streamk=3, atomic=1, work_stealing=0) + assert "Valid" not in state From 327120e30126425053b6b28ea627b4a2ce2bb9f6 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 24 Jun 2026 17:49:11 +0000 Subject: [PATCH 03/20] [tensilelite] Fix parameter types in StreamK work-stealing test YAMLs Correct bool/int type mismatches flagged by the strict corpus-clean validator: BoundsCheck and PrefetchLocalRead must be int, MIArchVgpr must be bool. Resolves the 2 corpus-clean failures for the new sk_dynamic_work_stealing / sk_hybrid_work_stealing tests. --- .../Tests/common/streamk/sk_dynamic_work_stealing.yaml | 6 +++--- .../Tests/common/streamk/sk_hybrid_work_stealing.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml index 687c2933e854..a8e98c149f70 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml @@ -6,7 +6,7 @@ TestParameters: GlobalParameters: SyncsPerBenchmark: 0 NumElementsToValidate: -1 - BoundsCheck: False + BoundsCheck: 0 KernelTime: False DataInitTypeAlpha: 1 DataInitTypeBeta: 1 @@ -24,11 +24,11 @@ GlobalParameters: # paths in a single invocation. .SkDynamicCommonFork: &sk_dynamic_common_fork - KernelLanguage: ["Assembly"] - - PrefetchLocalRead: [True] + - PrefetchLocalRead: [1] - 1LDSBuffer: [1] - ExpandPointerSwap: [False] - GlobalSplitU: [0] - - MIArchVgpr: [0] + - MIArchVgpr: [False] - PrefetchGlobalRead: [2] - ScheduleIterAlg: [3] - SourceSwap: [True] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml index 805c912aa4a1..cde7c9c5d335 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml @@ -6,7 +6,7 @@ TestParameters: GlobalParameters: SyncsPerBenchmark: 0 NumElementsToValidate: -1 - BoundsCheck: False + BoundsCheck: 0 KernelTime: False DataInitTypeAlpha: 1 DataInitTypeBeta: 1 @@ -30,11 +30,11 @@ GlobalParameters: # runs for both stealing-off and stealing-on across both hybrid sub-paths. .SkHybridCommonFork: &sk_hybrid_common_fork - KernelLanguage: ["Assembly"] - - PrefetchLocalRead: [True] + - PrefetchLocalRead: [1] - 1LDSBuffer: [1] - ExpandPointerSwap: [False] - GlobalSplitU: [0] - - MIArchVgpr: [0] + - MIArchVgpr: [False] - PrefetchGlobalRead: [2] - ScheduleIterAlg: [3] - SourceSwap: [True] From 189649ec2c05b0ea6b92549badca98ed130fd214 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 24 Jun 2026 19:57:05 +0000 Subject: [PATCH 04/20] [tensilelite] Regenerate characterization snapshots for StreamKWorkStealing Adding StreamKWorkStealing to the required (min naming) parameter set inserts the SKWS0 token into the verbose solution name and perturbs the kernel identity-hash basenames, consistent with the rest of the StreamK family (SKA/SKFTR/SKFDPO/SKXCCM). Regenerate the syrupy golden snapshots to match. Diff is pure naming/identity churn -- no err-code, instruction count, or emitted-assembly changes. --- .../test_solution_class_char.ambr | 9 +- .../__snapshots__/test_builders_char.ambr | 9 ++ .../test_emit_bigfiles_char.ambr | 110 +++++++++--------- .../__snapshots__/test_emit_gfx1100_char.ambr | 22 ++-- .../__snapshots__/test_emit_gfx1201_char.ambr | 8 +- .../__snapshots__/test_emit_gfx1250_char.ambr | 8 +- .../__snapshots__/test_emit_gfx908_char.ambr | 8 +- .../__snapshots__/test_emit_gfx90a_char.ambr | 12 +- .../__snapshots__/test_emit_gfx942_char.ambr | 36 +++--- .../__snapshots__/test_emit_gfx950_char.ambr | 28 ++--- .../__snapshots__/test_harness_smoke.ambr | 2 +- .../test_r2_activation_char.ambr | 2 +- .../__snapshots__/test_r2_gsu_char.ambr | 4 +- .../__snapshots__/test_r2_kwconv_char.ambr | 2 +- .../__snapshots__/test_r2_lra_char.ambr | 4 +- .../__snapshots__/test_r2_mac_char.ambr | 12 +- .../test_r2_shiftvector_char.ambr | 6 +- .../__snapshots__/test_r2_solution_char.ambr | 8 +- .../__snapshots__/test_r2_store_char.ambr | 8 +- .../__snapshots__/test_r2_streamk_char.ambr | 12 +- .../__snapshots__/test_r2_subtile_char.ambr | 16 +-- .../__snapshots__/test_r2_wgm_char.ambr | 10 +- .../__snapshots__/test_r3_addrstore_char.ambr | 4 +- .../__snapshots__/test_r3_datamover_char.ambr | 4 +- .../test_r3_globalwrite_char.ambr | 4 +- .../__snapshots__/test_r3_gsu_on_char.ambr | 4 +- .../__snapshots__/test_r3_kwafeat_char.ambr | 4 +- .../__snapshots__/test_r3_kwfeat_char.ambr | 4 +- .../__snapshots__/test_r3_lra_tr_char.ambr | 4 +- .../test_r3_streamk_gfx1250_char.ambr | 6 +- .../test_r3_streamk_xccm_char.ambr | 2 +- .../test_r3_subtile_lr_fp8_char.ambr | 4 +- .../__snapshots__/test_r4_asmaddr2_char.ambr | 4 +- .../__snapshots__/test_r4_fp8mx_char.ambr | 8 +- .../test_r4_shiftvector2_char.ambr | 6 +- .../__snapshots__/test_seed_gfx90a_char.ambr | 4 +- .../test_seed_gfx90a_db_char.ambr | 2 +- .../test_seed_gfx90a_hhs_char.ambr | 4 +- .../__snapshots__/test_seed_gfx942_char.ambr | 8 +- .../test_seed_gfx942_db_char.ambr | 2 +- .../test_seed_gfx942_f8n_char.ambr | 16 +-- .../test_seed_gfx942_gg_char.ambr | 8 +- .../test_seed_gfx942_grad_char.ambr | 2 +- .../test_seed_gfx942_hss_char.ambr | 2 +- .../test_seed_gfx950_bbs_char.ambr | 4 +- .../__snapshots__/test_seed_gfx950_char.ambr | 4 +- .../test_seed_gfx950_dtl_char.ambr | 8 +- .../test_seed_gfx950_hhs_char.ambr | 4 +- .../test_seed_gfx950_hss_char.ambr | 2 +- .../test_seed_gfx950_i8gsu_char.ambr | 2 +- 50 files changed, 238 insertions(+), 228 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 50afa3cfe781..078d867fe33f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -3,7 +3,7 @@ dict({ 'count': 1, 'names': list([ - 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', ]), }) # --- @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 334, + 'len': 335, }) # --- # name: test_solution_construction @@ -293,6 +293,7 @@ 'StreamKAtomic', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', + 'StreamKWorkStealing', 'StreamKXCCMapping', 'SubGroup0', 'SubGroup1', @@ -396,8 +397,8 @@ 'tailLoopOptA', 'tailLoopOptB', ]), - 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 334, + 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'num_keys': 335, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr index 942a877d4767..620974fee4fa 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr @@ -1480,6 +1480,7 @@ 'StreamKAtomic', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', + 'StreamKWorkStealing', 'StreamKXCCMapping', 'SuppressNoLoadLoop', 'SwInstructionPrefetch', @@ -2575,6 +2576,14 @@ 'len': 2, 'type': 'list', }), + 'StreamKWorkStealing': dict({ + 'head': list([ + 0, + 1, + ]), + 'len': 2, + 'type': 'list', + }), 'StreamKXCCMapping': dict({ 'head': list([ 0, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr index 9cbdfb79b7d5..84055e57aac9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr @@ -2,27 +2,27 @@ # name: test_bigfile_capped_emit[aqua_FreeSize_GSU9] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x128_MI16x16xeK9asiZMrbCMdM7BN8Y8VPs4X9l_t-4POmnLY85J1GU=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x128_MI16x16xwAdtYQWUqjgXrBtLtqNrtj6jQajcSpcvAZbFrSAKkmU=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x32_MI16x16x1TCbsX_BfMv2CNCw0TNUm7efYm228hG0wrRy67XVON8Y=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x32_MI16x16x1LMApYeEBdxkQRzGKfDsm0L2yV5PVJeOJzZv3o2LQOjo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x64_MI16x16x1rhlefxteZ8hwtuVdx2EC_Bamr_OdAScfMLInAH-J8yk=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x64_MI16x16x1o42mEe6LDQj-PBhmHCwOdM1Qfdb6LDS-qRoI5pRMJbQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x128_MI16x16xz33xcoSkfqgK9CoqGBMprGOyPIfdtiiEEztdqCitWsY=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x128_MI16x16xMh8e0XQbctaben66JJxuGnPi7ZcoEZ6UC-Vxdjk3BZ0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x32_MI16x16x1PbY0aqQr_Je8mk-a--fGNxHP0bI3-EXiM69kP2ii240=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x32_MI16x16x1TG3NC4UM08VgpGSenjVZIL-FFIjL1y-R5DhOeoi55X0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x64_MI16x16x1aEJ3PBUeEVd3l9MWVaQ_avWrHAv3OHvF76f0WFx6iXg=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x64_MI16x16x1z6h9Jhghr0Gov7ldUPnwKp3KTwZIaZc5AmA3D4-lPnI=', 'err': 0, }), ]) @@ -30,27 +30,27 @@ # name: test_bigfile_capped_emit[equality_gfx950_HSS_big] list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1sKhCC6r13jWT-trhAumMCbTwRmkCpSw-B1fP1F1QnJM=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1yx4TpoOVcetzEHBqlf2V_ySyBAfOnqMd_2Okf4Xi0AU=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT20QpE-CCweOA1rnZYgi3cS6FfAB6bK7KLB2fV6PlUM2U=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT24i53Y_LSFgOzB9sDPGjQGThrZo3hFH0nKlhQjY7_7Xo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT23JaUbTkbyw2UM-8PDFN_RCN7o1yz_sSB2NkW8k9QIlg=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2AGUSydOLlSW-QpdVW6_l88JgoH9J_L5PaNAuMr5B3PQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT24KLeeoDv4J_ZuX0_QJF-g3mVdR8Zx5K7SgUQOkJ1OdM=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2H4XR7PFMiLXg14CMUPWthg6iVl1DssPpPjw0mBOrWXs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT27P08B54pBVnckJfm4VUMqK9i0FBqWDMkSlXNob2XeU8=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2TFVW0ZJ_iVtJ4bW8RCR5g8LCE5jbW-sSnzFXwFLl03c=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT29zv8BeExU8VlaBwThmkQchpVnwFIpLCXL6Q_Bvhqetk=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2ThmvE5k1prsDoB3vCW95ULKJfhr4itCwuU9YU95FF7U=', 'err': 0, }), ]) @@ -58,27 +58,27 @@ # name: test_bigfile_capped_emit[freesize_gfx942_F8NH_GSU] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1-t3FQK7Z8LL3jy3Pp3omUmbQ-tRicZjZ7bbCuEzV7IY=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT10qLbIPFj521VmphAr_ABWov6_t_NfHJOjsASOE6WyLs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1FWpPKPGArlwoZKHp6iQGxXWZa1aBfzCyHAnGisUjD0M=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT11KU2JctRnNMBU88dJGCN_RPOHM4EBfuhGk5LLAUiPpk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1aBEhkjymwy8hULkPqfBluOWbl84h6ZYUPdnvvJobwSI=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT145WRf49nA1MqvaI0JDPg5OhHqTMCQOfV8O9G_A8xTcA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1cV8Kd1xGPzeggCSIZ_E1_hecsIy4F12UTlho2023Dbo=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1NqpWdfPkT8Zdwma94LncAPJ9Itz9Unb9Au7aJRq0c5I=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1s8MeKyqkrPsc5L0qnzk8Mdp_5RagY33obELLr54lnK4=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1TZfJtgG1E0VsX0BHA_XunVcxHuDBrAbxzueilV1fqGg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1vG8fC7NFOsF-5KltJLm7ozhwuk0ZfFLMwyfLNB6RBqI=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1mHINYmfrGnjC6BmJX5AwkaBGc_a7zn7TfK-L8bLNxKQ=', 'err': 0, }), ]) @@ -86,27 +86,27 @@ # name: test_bigfile_capped_emit[gfx1201_I8II] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT128x1bLA5DrrHpK4phunWs3Dgy8CXXdiflp17CbDdKYq_qis=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT128x1osnqWm6bfBmbpCU6ctZlJ2p4pbR0saQ3yR-mcIr3hmM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64SJkGLhwv5KGI3MIQEVI84Yg7-ppfv4gJHOOz87d4Rf4=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x645IOegC5osH5beI8STcJA2HhxPLbc3LGI8uNV6Q1j8Nw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64i6cjEXLaas83zDDB4-iQ-xQ7xuxuvwS7iBVS581xFhA=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x646rSkX5E_1b0qwkFG-iXnNEixph3rlhNtHYnoVGNU4EQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x1-qqJllfWr2gY_Tvk-vPSXy5ZX1VnkCJu5P18cgsbmE4=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x17LD0QJxi8zZV6dc-w1cZUNgSLbJkn0HZ6gIN8zIdre4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x19f4Vss0fpNXBJniCJzSLSsoLAcB1t-6L-tdyZB-bf6k=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x1AckpWHsCo7XlN9733l_tftYKIUO0JG8woa5mxHqTdu8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x1IBM57wuT6zq1-ub0Va4egHmyFqBjROFzU9YZ5I265AI=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x1Dw4SRGDufbz7Do1jCccOz4XrjULqyIIrAz5o6yVIo1A=', 'err': 0, }), ]) @@ -114,15 +114,15 @@ # name: test_bigfile_capped_emit[gfx1250_GG] list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2DrKzk7e_b4rHrEBWQYZ4-TEitrra3ekErYv13Or5HGs=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2JAv7dWb6NZfFzC2fc7xVRfXtNdNFlc1WHwSQaNf_iiU=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2WkQ20AiCPo6MjM2uB_z25otK0DC-X6vzUxXiCMvHBmk=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2Zd-ZBHzqY4BG4DQrq30OpFGU4FVOu1o-jeeVDlGqcgQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3rdKajoNdAwaZAZbvIIyoXVBv1C4p22Piqo6FQM4MB5U=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3L-TqCWdxF1w5yZmlNOPEuoL9Hii4EaMf6_wQBcrI9nE=', 'err': 0, }), ]) @@ -130,27 +130,27 @@ # name: test_bigfile_capped_emit[gfx90a_HSS_big] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI325T4M6UoWWS9hk7r4ZSdqTzSrK9P6f_rbnowaKAL75oA=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI324U0ncfK8GVBNOftbfM1YpL4toRwSi_se6SOf3E6ZNqA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32Uxuad6XehxIlG1YKFxtvfGCdPoOgAO7lXstVnwAMkqc=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32N8GrCvk0TR7mmlbFz0-P18fZkycCmq2O0wZ7-eCbIWM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32g2osxjOvFszd2pb2IHdGhX3pVMIdSerUE8MJ9maCGrI=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32pPQhMhe7ddiPGUiTZuF2Quhb1nib5ZQXthFe00Jj1nE=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x192x64_MI32e287lawo5gN_Ee9HUSfwjso6g13kOnl5fl_X97qRlL0=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x192x64_MI32ZWBKXS0OLU_Agzjz6IX-4j6ELNkG6N509hqo6ogA6vs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32xKhbFvclEtkPtcP8_GwkcAnO25xKruYLteLOHyHfsNKY=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32xJNSDniEPOjGcMN_m_Pnj1PuFApe1WVfh2Tcr2D0QA18=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32xenl7qAoQzmmc9t_s4bk8k5xxbyFPDDSZHfVxDhnzHDk=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32xpSWw4VCBWhMK8fXVogpAEJ9l5v2Sl5wWWiC-bX6rdHE=', 'err': 0, }), ]) @@ -158,27 +158,27 @@ # name: test_bigfile_capped_emit[gfx950_origami_MX] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT102aixOSinF6v08jhBMBDJrgdH0e41sV7vzPc4vGl-Ws=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT13aqCQ9Bvcq4zIZBSEZyx5VNPZ7JMSx4gCRvrIJJBS98=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT12xn86BHWBZzXx5K2qtDfegqBfKDYmcbEsHk7Jdyxfhw=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT17CIFoQIHXCyVOHfDRUEE3iCRhKi_gffFIVFezEM5g-8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1FtG5zmUtIDQG3mjcJydTxEJ7uLBQuAEHWINz2KrEVNk=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT19j7RhvmH-91oAq3EV54lt-Ff9tY7M1IW_XB6OeMDEIA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1Kh7MNMCQyKqsFZNRaEWbi6ijFKF7xqJUqKd7VpmkjUI=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1AprcDQ0jxgcAx-NIulTcZJbNaPEZphajiES9cRGYAUs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1Lqn_APGAiB_Aktw6Km9CNeKavYVvVQm0ZZoByh4vvv4=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1BpZEJDSCa1AreNuDaYOeZ3KD9fZcIYcy3jryH9u-l9c=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1Us957z5co-i-cuSkQJv6Did70Xnm18RLnByy5Zdmj8c=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1H4G5TZhAj7JbSGCdI8O78Brn8GAy34o6iTBT5ZsBpmM=', 'err': 0, }), ]) @@ -186,27 +186,27 @@ # name: test_bigfile_capped_emit[navi31_HSS] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT15YMySe0pkCl1rEXn5rje7T7bStrWpqwZAmn8OjFgSTw=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT16X7zEPRZMCbUjR1RgvHLUDNeF9Q6b0sRnKLM6AYYbJc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1SfSSoeXnRMghfBncEqYXlNEvOl9-df7cU0KDvXnV308=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1NCaOTpPvrGy4kJjAgz-SGK0X35lwbtPCqB73ezHa9CM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3AM0cLd15_DpmUFuG1N2DsQ9pERQmXv3p7RO_xJ6Pa3w=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT33-XvK4PpUYUA-8j-rftSqwV6JAv0yNhXKz5E4BF-UBU=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3gT4ZgN0syyAe69YiiySUR3iDn-lV3neSVPZ0JWTeFhg=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3KubVhIYO_biKTfzb4xhCR_sDPWv_qLyJeyyZDQOs_rI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4LmwoKywiN3PnolOCSt-7Yr-SpLARQPks53aGKqwZFz4=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4OYiQYkPZWh5Xx7dGmK79SBWSsxyUQ2yCmLoPdodLbKo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4iAVh8oFvnA22gJHFFODPzDeOEh3oypsNNH31uwOewC8=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4ofZN61nr_2ZoYg1O1uPLuuOnmRCcZMBxKCx-HyfVCmk=', 'err': 0, }), ]) @@ -214,19 +214,19 @@ # name: test_bigfile_capped_emit[streamk_gfx942_Ailk] list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16ROFXdYSCGxQC0-TRbS7cIaNzooaCCM1-keBVZqsqDDY=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16BrrCjvPfKg9660yB-PHBk-cO-HXNmLmbeWsdQrCLwnY=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x32_MI16x166n4yvq8IdEoXIWwFOd9ldMKg7ay91qyBvNBwOu0TI-4=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16w1zcpudFGJ_1Ckpoif4PbXypr-kOiPy6Ydu4GzTEiMk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16DHiYqAVnm26bNsI2naL35KiCFhqEoqXZ4_LOG-0qJuA=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16lFYG2BdYwwAO9YRMZTF9xZkVmoYRunRfklrkVY6yH7A=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16dcqsLNFLXM4qrV9hG0vKv-tv8HQQ7kVFr9UhkosjGPQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16ob3XO15L3ejJ_N_uF1GwyR6ZYIxPRTwrdcucCQxRPew=', 'err': 0, }), ]) @@ -234,27 +234,27 @@ # name: test_bigfile_capped_emit[streamk_gfx942_S] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16pRpQ4qqKgHyAhbcMQ8pF4pITdRb8i4iaU75GWIwtwAE=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16hnnrn5MzutRXIFWXmBJRUI7Qc2_-42IHawMZdmtdEfI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16lMdC7VkYDf-TMjm8KLvZPSatdzyNTc9HTHOrbUiHfeA=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16708UrMUcQiZVD1ngGbBjGSj8mPyc7TIw_oz-RiOLTZk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16ZoqZzcO71yQSlTfh3C2qLJhwkkS0gLiZUzjx8tMTtIM=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16MAv5r2IbYUyHnI3XjPahcm4qvZhLh8a_CFS7ds-_fGc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16G-O3RykTaOBVoudGBKELh7RzN8F9QtdqbzLKQGAyRco=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x32_MI16x162V2WfqwYUW0BBSeAEUvQBfQSKKSLi_GJOovJmy9K2eI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x16_MI16x1641ng0cO7YU21RU5iEfI86tT6s2Cz0fVK7bR06eea9Po=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x16_MI16x16Q2HTlDYwCkPP4ttFGcJdaMDUBmIP4J0mYTlEgDjjnA4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x32_MI16x16fU9Qs7CiuHchxJZu6VFf3JhF85pRK6ziGk171iPWZls=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x32_MI16x16zmXL5dIzLic8uKgpZM4rTothKEbNTxkUo-tIh0Pj2os=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1100_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1100_char.ambr index c5a2d3a607b3..7769784e42b7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1100_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1100_char.ambr @@ -5,19 +5,19 @@ 'file': 'DB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x16_SN_LDSB0_cwuCvvUuC8kdadYqDARrCIuBkWJmEtgfqfVscwwPgvA=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x16_SN_LDSB0__C12YwAGM1VHgL80q56Yhm2AYEb2JI_q7QO7pJTSKxQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x8_SN_LDSB0_AJ8zPH-zLP26s2FnwvsvdDJjjBnYhHu0pzVFZr26oopM=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x8_SN_LDSB0_AlXjArD08fjjCj9mAQS5V8vWPAgo-YEdTN1eucXYehSA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT8x8x16_SN_LDSB0_AFgXFhfhzVQjiU9MHCopQkMt-L7anZDpgJgrSBOzPWgDo=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT8x8x16_SN_LDSB0_AF2g5uq_WciN-Dxif2eM6mU7P5qtJ3AQgAau63tchIxAk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT8x8x8_SN_LDSB0_AFCiFOcsL1pdDasnbfcIwNOZMTIiuTwGs7vtwFu16-8y-Y=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT8x8x8_SN_LDSB0_AFCfFuTx36QlJwxB6qbv7CxqIOnxu9qP-TUIfjUIoTAA6o=', 'err': 0, }), ]), @@ -26,15 +26,15 @@ 'file': 'HSS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT348dwD9Fhz7Ppe8sdn5T_PeluDd4qmD_a3Xca2t1kJVk=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT39-1I9WZ4syol_ags9jr8p8uU7DwSXt5JvVpgaRXyqjk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT9mjyBxX8VJho0_8xtzdbiT7KFR5_s6gJFPEsYu84e_XU=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT9A784Mc2D6qfh0QhivT33l_c_CBy14ESSUHhdbZiXJxY=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT9szjK8ctnVYWAIbytBwDnER0AaMJvaHoW8s8qE6Bn__Q=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT9faC01eTPZhlYUZw89TMyEoerjALxmLDmUXd7dH6t1FM=', 'err': 0, }), ]), @@ -43,19 +43,19 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT16x1LGEpVNETqMEdtR6DYcG-VXBfLvW2ixx3_Dzm6fX157Y=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT16x1p5kyxL0LvcmTmmV3SdRDpzuFtYCQaKRAI6nkua4_80w=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT16x1rx5gP59xPm-ojvEAu1sNgZirNC4TgTnJjmkOnTARc4s=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT16x1zNkweXO-JcvMRxFs_g1IAL-vku-WvxWfHXWKOK6xKo8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT8x8xN3OszkBfrgLDt9wuYB8qEZivi0OInX6cIQ22ByJrCVs=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT8x8xXscQChS35ytwTPhKJxowtVvXdf3H4oSwVC6H4N3kkGI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT8x8xXy-Dqx4IK6Bo_AWjYLOR7aIRtKREc38wIvsPfeoU1wI=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT8x8xntVV1mGSIss29ezbnFVVGranAIX1Ia9r6NZXJHEMU_I=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1201_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1201_char.ambr index deecaccbf846..8501e4c95e29 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1201_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1201_char.ambr @@ -5,7 +5,7 @@ 'file': 'B8B8S.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_B8B8S_BH_Bias_HA_S_SAV_UserArgs_MQ_k-8G45_t7fkZw_OF2V6IoksILpJQox35Q2eKnM7HM=', + 'basename': 'Cijk_Alik_Bljk_B8B8S_BH_Bias_HA_S_SAV_UserArgs_MAm-6BsJmpL_hW9N6DMWyouZ7ya5ddeSCm8hTV_I3Xr0=', 'err': 0, }), ]), @@ -14,7 +14,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SH_HA_S_SAV_UserArgs_JL96849B4LxAXiAds_4FC-h9n25bepu2yj6abMcP7SY=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SH_HA_S_SAV_UserArgs_VjbOj7pCfiEyQHNv5j1gvBLrIr9iVlWMWAnEjJCJXAI=', 'err': 0, }), ]), @@ -23,7 +23,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT1oEzh0VaU3wEEPIYq_6TXfzghtL4sRFCt2_qER-i57Kc=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT1zbdjNvISWXx8AFCcKxgHz0YcMjJjmVSjuhwC_cVuHtc=', 'err': 0, }), ]), @@ -32,7 +32,7 @@ 'file': 'HSS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1oEzh0VaU3wEEPIYq_6TXfzghtL4sRFCt2_qER-i57Kc=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1zbdjNvISWXx8AFCcKxgHz0YcMjJjmVSjuhwC_cVuHtc=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr index e6021c39ff34..07b4f51c6168 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_BBS_BH_Bias_A_UserArgs_MT16x16x32Jigdht1N3VAzzdESguhTae427xj4MkB7fXwj1Lzq0VI=', + 'basename': 'Cijk_Ailk_Bjlk_BBS_BH_Bias_A_UserArgs_MT16x16x32JAFITuzagwii6J5Ns7PrVhwz6Ag4Y624g9d23Hu1VbQ=', 'err': 0, }), ]), @@ -14,7 +14,7 @@ 'file': 'F4_MX.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_MXAE8B32_MXBE8B32_BH_Bias_HADKO1EMMlfOMg7uQJz95AUvsMAmAQri95FZNYQqQbvh0=', + 'basename': 'Cijk_Alik_Bljk_F4BS_MXAE8B32_MXBE8B32_BH_Bias_HAAcG-PxwT20sZRbFgY_9U-EKrBl0JCf5TJdkElrRECAw=', 'err': 0, }), ]), @@ -23,7 +23,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_A_UserArgs_MT16x16x32Jigdht1N3VAzzdESguhTae427xj4MkB7fXwj1Lzq0VI=', + 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_A_UserArgs_MT16x16x32JAFITuzagwii6J5Ns7PrVhwz6Ag4Y624g9d23Hu1VbQ=', 'err': 0, }), ]), @@ -32,7 +32,7 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_MX_B_Bias_SB_HA_S_SAB_SAV_UserA0N6EjB5KANz4kF9c5noAdWTSEHduQJsxhU_a9iEkMvo=', + 'basename': 'Cijk_Ailk_Bjlk_S_MX_B_Bias_SB_HA_S_SAB_SAV_UserA_AiaAuGw6gzJKwOmSU_6n4_Td1SoyMdO50o_TVkL7hc=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx908_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx908_char.ambr index d4ce155cb09f..8d5233b3320c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx908_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx908_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_BBS_BH_Bias_HA_S_SAV_MT64x16x32_M07orm7M12qwyas_uyrCbEhsQJapqhmh7o1flS6B724g=', + 'basename': 'Cijk_Ailk_Bljk_BBS_BH_Bias_HA_S_SAV_MT64x16x32_M0MWgN6lNCygPmK2AH0Xnb8z6TKcT0M1Eb3QzygAq2Fc=', 'err': 0, }), ]), @@ -14,7 +14,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT6rNyMoPPOtvh1FNxIAjrXsNEhOIvKj0POlyvZidFVtcU=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT6ea-87O5WCccgDk1WdT6ah1bDWoNpMQXHsrCK5gSONyw=', 'err': 0, }), ]), @@ -23,7 +23,7 @@ 'file': 'HSS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1MivaD2EzLLlP9dcsdyhD9uEsn5tw0S9uz4GGPmcHKQM=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1udWqdnBgEb3_0OSM55WAVKcc2FWAIZtJf3Tn1h5J_rg=', 'err': 0, }), ]), @@ -32,7 +32,7 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT32x3V6-LfcHTQCPQMARJM_rsOYUC-4FLABwVSIuPqk6zN9k=', + 'basename': 'Cijk_Alik_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT32x38xW9KOjr0ZNQ6cuF1WiOw-x0gzP7uaG9cP_n8RLEO0k=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx90a_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx90a_char.ambr index 956709359cd4..851d2750fe7b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx90a_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx90a_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT3p5ofdCzoUiH1htPmnbXnCC7RlRV4miq8I7WxVwuKmB4=', + 'basename': 'Cijk_Alik_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT3PWZ_Q4ztiOQm48q_jVxZIlKshKafNh8VUJCe11XbDdQ=', 'err': 0, }), ]), @@ -14,11 +14,11 @@ 'file': 'DB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x16_MI16x16x1sZCjdfYhuNo2qr2apFqBmR0KAeyfzhUjTrxW2zagRrE=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x16_MI16x16x1hSqel2456Jfbx70191CFPnVzdVaaugkdmKPLVK5PDRs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x8_MI16x16x1_WKSeqOFFwNUHqnmMzJZXkSygd-E38lokedk_TmS2gvI=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT16x16x8_MI16x16x1_LTIqruYpHyaNvm-tjwEPZW9oHwR1UaurGDqlQey-ysc=', 'err': 0, }), ]), @@ -27,7 +27,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT3p5ofdCzoUiH1htPmnbXnCC7RlRV4miq8I7WxVwuKmB4=', + 'basename': 'Cijk_Alik_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT3PWZ_Q4ztiOQm48q_jVxZIlKshKafNh8VUJCe11XbDdQ=', 'err': 0, }), ]), @@ -36,7 +36,7 @@ 'file': 'HSS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3p5ofdCzoUiH1htPmnbXnCC7RlRV4miq8I7WxVwuKmB4=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3PWZ_Q4ztiOQm48q_jVxZIlKshKafNh8VUJCe11XbDdQ=', 'err': 0, }), ]), @@ -45,7 +45,7 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT32x34ZsRbJyNWEJksIrfKSTimDjbBKTEjyeclza8lBw3kTw=', + 'basename': 'Cijk_Alik_Bjlk_S_B_Bias_HA_S_SAV_UserArgs_MT32x39KHjuoD9ft446YV8jJ0Xdi30ImFZgcLWeSF6osFEnZw=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx942_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx942_char.ambr index fed74c84f229..77c5ae0add08 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx942_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx942_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS_BH_Bias_Act.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT256x9jwtf5xp5GYLmQPQiDq7UlOkMemtKvijGW8Q05Mu7iGk=', + 'basename': 'Cijk_Ailk_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT256x9GbJ5bUi3pNo8K_ylBPLYfCv5eMB8v6Hlf-IZqjQIlcY=', 'err': 0, }), ]), @@ -14,11 +14,11 @@ 'file': 'DB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x16_MI16x16x16CVnYc7h1dLNJG-9-SfujrXqhIKv88DUJ1TS93jGw7g=', + 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x16_MI16x16x11397V8DF6OIzsDlEZPyYVwqD2ZM5y5z1iuFD9CHWYJM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_OQOELZqZVIL-hKggC30K0Igg3xCxKDu-nmT-4Lden7I=', + 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_O5z4CYRMDQ7OSzFx9wHAeJlbhDkYJySuXYsyEQr5r0Q=', 'err': 0, }), ]), @@ -27,7 +27,7 @@ 'file': 'DTV.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_STA_BH_UserArgs_MT512x128x64_SYKpGncTXjDp7Vs1FQL7wTPZGDQF3VLEzdOfdktlfsk=', + 'basename': 'Cijk_Alik_Bljk_BBS_STA_BH_UserArgs_MT512x128x64_RBAEInewdWucrYVq4_02laIzfUIBhut3Of2bl-AYJ0A=', 'err': 0, }), ]), @@ -36,19 +36,19 @@ 'file': 'F8N_multi.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_S3qzQQ6gu9LOF4zLehg5WN_-R03KyUnelMowUxX4AUz0=', + 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SO7zsxuBzrqavKXLZL_fXAvpVjMTsrX23JNDUYFMdiBo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SW6nMe6T7dOUAyZa_WMaWCyYq6ATDMxF9R4Q63u1CLvg=', + 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_S_LUYNUoeivsyy-KSdckWHjLK4VQ7m4rnWKIGpsEpuws=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SanVCadW45snzzYhXgMKOyvG6BOWCm81BtkH3rBgVFRc=', + 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SemT01D7qbARcXfcvv-L5hHwd4NwevH5X_nZe51Yasto=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SjM6Ox-xkWO1O_6X1EJJu-wjG2yv6q5xLtTMrXqoCFDs=', + 'basename': 'Cijk_Alik_Bljk_F8NF8NS_BH_Bias_H_AuxH_AmaxD_HA_SiY-W1Hz-qiHEY1LXIrL_OKA86MEZUOebmOH6p1wlcnE=', 'err': 0, }), ]), @@ -57,7 +57,7 @@ 'file': 'GG.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1rDz9kgx5cc8BTW6HAS7WvXtlwE20aPuTKg7NUq-qnj8=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1j2qKTQML63oW36kPp4HbXRmlIf4FCv_rhaduDr8xnMI=', 'err': 0, }), ]), @@ -66,11 +66,11 @@ 'file': 'GSU.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x16_MI16x16x16CVnYc7h1dLNJG-9-SfujrXqhIKv88DUJ1TS93jGw7g=', + 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x16_MI16x16x11397V8DF6OIzsDlEZPyYVwqD2ZM5y5z1iuFD9CHWYJM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_OQOELZqZVIL-hKggC30K0Igg3xCxKDu-nmT-4Lden7I=', + 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_O5z4CYRMDQ7OSzFx9wHAeJlbhDkYJySuXYsyEQr5r0Q=', 'err': 0, }), ]), @@ -79,7 +79,7 @@ 'file': 'Grad.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_D_GradS_HA_S_SAV_UserArgPAE27qZ7J1uFrvElgcBal8PiYqy7-faRXW5KGtHsSck=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_D_GradS_HA_S_SAV_UserArg3v43_ljAdLhQtiEhC_vah8awd66m36A-KOA3s9AVrnI=', 'err': 0, }), ]), @@ -88,7 +88,7 @@ 'file': 'HHS_BH_Bias_GG.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT14OtgOV2tmLObARViwOfAwOcH5ycC8TR2fKaz7YD3QgY=', + 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT1FfUbKddKdBaPMVsDjHp5ipwGWj60PTZJ6TefS5jVBTo=', 'err': 0, }), ]), @@ -97,7 +97,7 @@ 'file': 'HSS_BH_Bias.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3yCM7xliPHhZoNd6Txg8HL32dOyB0vApUHbVWqoqhB74=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3OS2t9wtkZ7mCFn0OigBWTdwacIIVVPcisSuJ0Np-VXc=', 'err': 0, }), ]), @@ -106,11 +106,11 @@ 'file': 'LSU_MX.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT113exoKh_lnZ1fuLgV0bL501ZV6eUPaTUx2_Vl2IpKBE=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1huCMIzwGsyZf0IRhx_p0MvFnpSTlH4ZdJAZ8_G1brXo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1p6Du3eIRRrkQgS1U1fDqot-_PCfAlLl6-qfedYA84uw=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1jCwV5Db0ek10kjso6AdclklPJUiTu7E9PE3i_-zeZlA=', 'err': 0, }), ]), @@ -119,7 +119,7 @@ 'file': 'MX.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT3PUXNgnMAlNlHO4LrNZ4Ug69LNrRC1zuBqbbvFEaIyvg=', + 'basename': 'Cijk_Ailk_Bljk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT3g-cNYQjOafZVVbrQmiYMFf5y7bqOYUsT9mh9CT7WpZI=', 'err': 0, }), ]), @@ -128,7 +128,7 @@ 'file': 'SB_Bias_Aux.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_AuxS_HA_S_SAV_UserArgs_MpcehK3aQcfBkIMpK-RPxzYcgLEmLk5CW-2y99WNHzbA=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_AuxS_HA_S_SAV_UserArgs_MjlSMlDx1y-_7H7ASXVIRBvTGcxUwSZaO5erFlCj5qfM=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx950_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx950_char.ambr index f60d8b653c5e..641ed23138de 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx950_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx950_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_BBS_BH_Bias_AuxB_HA_S_SAV_UserArgHVqqgaXTZJ4wy7kz5WvmEBAaTsF9Z8-ciHUvxiBCOQY=', + 'basename': 'Cijk_Alik_Bjlk_BBS_BH_Bias_AuxB_HA_S_SAV_UserArg-nP1CWMnE9Ks9JJ3Yy1xf5NGy4hvqn67_GdIP2rzGQs=', 'err': 0, }), ]), @@ -40,7 +40,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT1UZann1SR_ZA8f_rATa6rSr8WgvAWVVgroVlVVGtAaWc=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT1AcBHzRHyeTl5hqpFqP9U9Mj34JwxKQp4WcHFpIk41ng=', 'err': 0, }), ]), @@ -49,7 +49,7 @@ 'file': 'HSS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3BTq-dKH2NL6oykDZRuzgocTqfKCODbjkuef3cPjvQDY=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3Q5HSG8-eTOAejq747YNm5jRTDQAsXd8WIn6MmCzFzJ4=', 'err': 0, }), ]), @@ -58,7 +58,7 @@ 'file': 'I8_GSU.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_I8I8S_BH_Bias_HA_S_SAV_UserArgs_M89XLHNvqDVfwxelZ5AzJ7HbBu3TJzAuqzqAhX2H3LT0=', + 'basename': 'Cijk_Ailk_Bjlk_I8I8S_BH_Bias_HA_S_SAV_UserArgs_MJsuzF89o27YfEh-ZQvQ3GQ2oWVV2Om9WMcccDMbxTMk=', 'err': 0, }), ]), @@ -67,7 +67,7 @@ 'file': 'MX.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT4O_AU5KzdqJQdDO-PoCQbFsdq_pR31QYGOnBYNi59Tc0=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT4daf4InBnipSdeN_9C4sY5-xZjaLWULdOI6KiVF0H_Wo=', 'err': 0, }), ]), @@ -76,7 +76,7 @@ 'file': 'MX_StreamK.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT4O_AU5KzdqJQdDO-PoCQbFsdq_pR31QYGOnBYNi59Tc0=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT4daf4InBnipSdeN_9C4sY5-xZjaLWULdOI6KiVF0H_Wo=', 'err': 0, }), ]), @@ -85,7 +85,7 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_AuxS_HA_S_SAV_UserArgs_MD-Tl5nv-FTkc5TJP1h_jP7EGbfCab2aWF9Ne9Eh_p7M=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_Bias_AuxS_HA_S_SAV_UserArgs_M6Kafvz_1rdbL8TQi_Sq_GXJV4Rqd7b-Mwwjf555mV7M=', 'err': 0, }), ]), @@ -94,23 +94,23 @@ 'file': 'StreamK_B8F8.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArg51-gJAv5FMPAnxTohGd6wulRSW9lX4FUpaNA3Bnl_W4=', + 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgA_2Azc6RdjIuDI0vMHWIeqocP1kp_IvpIw3pYgRxua0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArg6A-TkGXDDD201999hpgcoPVbEpsADFwV04-mo-lzhsM=', + 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgDMpqHt2jt4_iwnI8N2VUKwGNQdXV8YS90dVSTwB5kAE=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgTr3goefVd8jUBfgpXBRWUPfNuVqY2SJZp--7wuEyIxE=', + 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgJpfR99OSqjZCCbPZnUOJipkNN2Jc24trzoFCelkpziQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgW1hZ4jX8ry6kgoguGwpP_f6Gv49mv3k4Q-UPho7xdLk=', + 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgKU4TbD-J_6u1tHUAG69LUq9RlWJF1Et9x2OIEsigeGw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgwXnuUxCnYewA9j3JWommM7rEv5na3mxfgjs81lx9Vdo=', + 'basename': 'Cijk_Alik_Bljk_B8F8_BH_Bias_HA_S_SAB_SAV_UserArgc09k187NJfMD4v6_EGQrIX-_40yyUwfYW-trF5z0v7Y=', 'err': 0, }), ]), @@ -119,7 +119,7 @@ 'file': 'StreamK_F8F8S.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8F8S_BH_Bias_HA_S_SAV_UserArgs_MJGDcmoogPYBbyro4EZto_gZnoVZpc11RstwaMV3994o=', + 'basename': 'Cijk_Alik_Bljk_F8F8S_BH_Bias_HA_S_SAV_UserArgs_MOsgQ6xnWmyRm3RCHfRM6kEp42v5HY46aDkCUfoqtdsQ=', 'err': 0, }), ]), @@ -128,7 +128,7 @@ 'file': 'WaveSplitK.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT3BTq-dKH2NL6oykDZRuzgocTqfKCODbjkuef3cPjvQDY=', + 'basename': 'Cijk_Alik_Bjlk_HHS_BH_Bias_HA_S_SAV_UserArgs_MT3Q5HSG8-eTOAejq747YNm5jRTDQAsXd8WIn6MmCzFzJ4=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_harness_smoke.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_harness_smoke.ambr index de8b25d3d17d..3b8bfd973157 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_harness_smoke.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_harness_smoke.ambr @@ -2,7 +2,7 @@ # name: test_emit_golden_digest list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3yCM7xliPHhZoNd6Txg8HL32dOyB0vApUHbVWqoqhB74=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3OS2t9wtkZ7mCFn0OigBWTdwacIIVVPcisSuJ0Np-VXc=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_activation_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_activation_char.ambr index 28ea61abd0b9..19f048c90976 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_activation_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_activation_char.ambr @@ -2,7 +2,7 @@ # name: test_r2_activation_gfx942_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_D_GradH_A_S_UserArgs_iP-FE0efdW0QcVXVD30BALXuNhBzyKKXttYRsCoOVt4=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_D_GradH_A_S_UserArgs_mRJ_boSxcuSiF89N5NfOMSbRNPGVw3QSq0oIA4MMD4w=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_gsu_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_gsu_char.ambr index a48f03e02979..8894cf7ec97e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_gsu_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_gsu_char.ambr @@ -2,11 +2,11 @@ # name: test_r2_gsu_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xCz71mPdwgIEXyy_2fHMRT8cASwdMi-kYXdJ0o5xpMcs=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xRbED8bvp70e3BYG_wAoght234ShX6KybYidyHvdeWyY=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xlL0CUbs9uueyP65OKDJ_sdo3P8njgxw4Cj7tNFo9TMk=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xYai5bTAKL7uvZKtRNRP_cK_FzgdTauHEQvUi5bvdI2I=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_kwconv_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_kwconv_char.ambr index 47a757f655aa..4da35e4b192b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_kwconv_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_kwconv_char.ambr @@ -2,7 +2,7 @@ # name: test_kwconv_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_B_A_S_UserArgs_MT64x1HrNQfJvUo7SLqxrLvQbkEQnfzZW2vwQ4zpwDSosAdxo=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_B_A_S_UserArgs_MT64x1t1kBsmVqssg9d1kxfo-KSZgntc8tK6Kyuq404BqBb2U=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_lra_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_lra_char.ambr index cbcd1ef3576f..ad7fa72fccf3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_lra_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_lra_char.ambr @@ -2,11 +2,11 @@ # name: test_r2_lra_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HHS_BH_UserArgs_MT16x8x64_SN_LDSBhlGF7R4TRS7TQTIbSmmIuWcPJ7XVF6uR1k5hgAVMpW4=', + 'basename': 'Cijk_Alik_Bjlk_HHS_BH_UserArgs_MT16x8x64_SN_LDSB9NtGZHsT5bQSN2l8S3_3gC-56oaeB4XHL9t-qrhyAwA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HHS_BH_UserArgs_MT64x4x64_SN_LDSBf4bJU8uoGzL8V1B56ycsvHxjIg8cZHxjIoBDOZ7ilPM=', + 'basename': 'Cijk_Alik_Bjlk_HHS_BH_UserArgs_MT64x4x64_SN_LDSBFOHV0TiKpPf-MC4Mpb5tAJ8dPGBKf1WnY991FRMN9BY=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_mac_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_mac_char.ambr index 03c11bdc7f6d..7dc64e5589df 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_mac_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_mac_char.ambr @@ -2,27 +2,27 @@ # name: test_r2_mac_hhs_dot2_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT16x32x16_SN_LDSYCT9sUMJRDLmOAr8jMI86K-XrD5zzrKkADyRd5UseZY=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT16x32x16_SN_LDSBCr7i42KDZVbCdnQPG-YILiOu47Yv_OcfQnZPlE0Vj4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT16x64x16_SN_LDS1awJC3yINVxbuixIaw3moSOfoyLCelIVIdxwM0DsqbY=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT16x64x16_SN_LDSDn55ZsRfptCv84YEqBKK9UTt_ZPvogrVbHyqQ8eNYfw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x16_SN_LDSsSJF66hHydBQ7Za8wN8VMKkqCOhJzsMoPZ1oPKQumKQ=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x16_SN_LDS7iD6h9FtvYEjXXEtJRCHV69u7fyO-mQpRXldUC9jmBE=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x64x16_SN_LDSutC2uBnEpemdcyuSpZJf8cF_UimKus_QJsi5sTJBM70=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x64x16_SN_LDSJ6vGuGh1LoZJBUIsiuxfCXqhYp5rWtzmKOdFJyPXIRw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT64x32x16_SN_LDSHbSNtcsNhLoQKmex4LOT_MjNYfTW2-mcSzPxQuABnwg=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT64x32x16_SN_LDS3Naon1svCBXSFVUJntwWKQ_zWz9Zeuo1UBcTt5-BWIg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT8x64x16_SN_LDSB4_IWR0uYBXQ8A4Z3Mxj82uA_fwWzXJFkMDdL2H-7Tos=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT8x64x16_SN_LDSBpWjNRD91iNji1jZMgBFUFGsAY2i37QGidKD8YtqELgw=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_shiftvector_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_shiftvector_char.ambr index e5ca7b6611d8..694af4798965 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_shiftvector_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_shiftvector_char.ambr @@ -2,15 +2,15 @@ # name: test_r2_shiftvector_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x64x16_SN_LDSB0lCzWGOCTQBMzSpC20-Xz7LqLXAF6PlZ7qJnbsdlJPZA=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x64x16_SN_LDSB0TLk-ETSWIIM3my_9RE3-E5u8AvCDEeKlhnNuXxOxc6w=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x128x16_SN_LDSB0znVp7vUMphvaaSgudxTQfONUfZdMkhTj5pYlWftXwK8=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x128x16_SN_LDSB00AcRGM7ul6Ft5nqHg1d4SR-oz6IDXumvF2EVtRX0nok=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x64x16_SN_LDSB0_DQ0NfvivhGArv9GT2lUninx-ZyPoWenWs64rV4kjqxc=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x64x16_SN_LDSB0_tMqKirfGdajiO4OeORPrYvMSc3yslGVrLGoRZUpc1-Y=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_solution_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_solution_char.ambr index 554b5f2cbe88..be4366300f08 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_solution_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_solution_char.ambr @@ -2,19 +2,19 @@ # name: test_r2_solution_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16x7pfJ1JPmuBozvaI6Zxr42M5blFDahr9b63Q_Qv_cVn4=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xJJc0Bm0ysjxh4eYy_nYqc4Lmdgd7J2MTCgK9HLGReDo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xQnB10PUAg-Mtx_Lkg5GVYB-a651ogiGWeye39obkMPE=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xYai5bTAKL7uvZKtRNRP_cK_FzgdTauHEQvUi5bvdI2I=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xWC600cVq-6gCYBVajFA4dK5CsIIYJPODit9bXRUpRxw=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xrq9lqUqaf16rs9EFWYUwhzdlwJWsPLJzoaIGRNRYToM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xlL0CUbs9uueyP65OKDJ_sdo3P8njgxw4Cj7tNFo9TMk=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xvGUJE3Rq2swpMGQVw9eSvMpyxKrKWFGlIyHdjFwfhY0=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_store_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_store_char.ambr index 4bb2c9395133..8bb2be069887 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_store_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_store_char.ambr @@ -2,19 +2,19 @@ # name: test_r2_store_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16x26vYmyeIgbEpdyfVMSW7YorYqda6-e1VABv0xfM1Szk=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16x2tfjkPG8TwIQt-YTCmYpIK7-bPWCjdCMXVZltPMkZr0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xp4jyL8lKYzT-u0re_iZtaxZhgwXVJjDow6ugkaP4mmk=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xLEfMBYjzCzPjWLsqA9ag_spDZU1OsMop-JtjEFJuJtg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xqGBRMtsQDWzoBj2XXshxbiTrWfM06nmCrdKoO22asFo=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xdcIlkSNFVrcoMND2lmvOH7vQY_V0iUp2Fd8UMzMHhCc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xzsTQJ64sAKoZolA6tTjlzKZ6J-hBc5A8J65Lc1FTKu0=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_SAV_UserArgs_MT64x16xuSYPTvEV6XCd9ghUTvwTNblr_Z_adiQkZUFZBM2Zf7I=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_streamk_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_streamk_char.ambr index 333e90b3d883..0a69b2f0c68c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_streamk_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_streamk_char.ambr @@ -2,27 +2,27 @@ # name: test_r2_streamk_gfx942_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x160xHV58gsi_NgQol7NPmKeoTkTmPBC-8D3iWk8zmF_ho=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16CrUoO-hS_mEvn8Te6sZTc7Kdbp5WOWVXHj41Fnvejkk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x160xHV58gsi_NgQol7NPmKeoTkTmPBC-8D3iWk8zmF_ho=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16CrUoO-hS_mEvn8Te6sZTc7Kdbp5WOWVXHj41Fnvejkk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16K9xN6qexZjSVlhbT37m-0oTcvEF1ZbKeSVvP4sPLMxQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16G_SYkEzW0MVzqLfdmE2kLcOWsFJa5k_-bZB0NvfTD18=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16K9xN6qexZjSVlhbT37m-0oTcvEF1ZbKeSVvP4sPLMxQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16G_SYkEzW0MVzqLfdmE2kLcOWsFJa5k_-bZB0NvfTD18=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16qjZAWgHA0xsa8UW2z7oDWta6knEbfzs3018uitNq9GQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16wvSymRfYZyG7HT1B776W9Lbb4nBLYiQzSm03Ys3LLLw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16qjZAWgHA0xsa8UW2z7oDWta6knEbfzs3018uitNq9GQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16wvSymRfYZyG7HT1B776W9Lbb4nBLYiQzSm03Ys3LLLw=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_subtile_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_subtile_char.ambr index 279a7a0cc799..d87e5d1ab5d1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_subtile_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_subtile_char.ambr @@ -2,35 +2,35 @@ # name: test_r2_subtile_gfx950_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x128x64_MI16LCenN916YyoZQKGs8uD-BgiB-kL0mBeIWKJHwTYOx3k=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x128x64_MI16DfHP7x2dbuLsSz6XRy4-mEgBu-ARl6yDMhVQCRIe3XQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x128x64_MI16onPA7ib259tec6Om51okIFyZQ2SLOH9JWQabiRD_DK8=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x128x64_MI16gb5zDNOcfXADGDVz2ZBOsjFretLjHznvCWnE85w7atM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x64x64_MI16xRkeiDxD1aqZMwe0AyjSUW9VUpep7YFn6yqtyCwN0ceU=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x64x64_MI16x6wy9iLYe9W6e7bjQrB3Pvjmz_E9GgI89JPIZGiRI6mk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x64x64_MI16xoRrU5bLZT7jz7XfFIrhiXQ4FHTHnTU-Ad3e0B4anPBw=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT128x64x64_MI16xqTWeYrfNYo9R_1norW3qZwDmUC2Bn73Eku_xfPmFyD4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT32x32x64_MI16x1dse8W36jxnMPbyGrCkilTD_nZFFdzcEYbTkeNZBfKyY=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT32x32x64_MI16x13cH0W89R4c4ssSJbbgrqfeD89WfenUJ_QIshO9BWrm8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT32x32x64_MI16x1sKkDLxCUWCktekIQ0mgz0jPE6_YGS8jytGtkQCtdcNM=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT32x32x64_MI16x167ythlWtJ-498OWs_ekeNmO72t5Ki-48nZ3ECgSpJCs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x128x64_MI16xBUEWS8QPwPnZQ6Kx51eLCXTDKFjvyeb4Zj7hnTObcrw=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x128x64_MI16x4yYoYCL_jaIsgdHq7B8jRo0h6oTgWoDEmAt4Iy8Dsn8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x128x64_MI16xIXuTX_6rbZdIgfYzHaJuW875w26_u2haEjO1-W3ruPc=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x128x64_MI16x8YxiaviCKKtrFfcefgVVy-zzBtTK2lHrnEqXuYf0kg4=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_wgm_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_wgm_char.ambr index 07f73f926613..8a75517714b5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_wgm_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r2_wgm_char.ambr @@ -2,23 +2,23 @@ # name: test_r2_wgm_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16x6H0OcEtL-IOK-W8nALcy6L7x0QtGgXHmt9QGeUi75Bc=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16x7jPba_5IpZpGwjvDAc32sxvGVWrJVjOm8bhlERkWWzw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16x7UCGTiIhCFk1_i5iWJhghnqYGoseh3DD7MDQ1C2WCSI=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xLudqSYpN4x95jtNnLtVqQjWt6-YrSj5VfcYccrUUlNY=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xD8SIIoS9xU2yV7YeYCPtoUmNPLfC_39WGRTkHzTnJPY=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xYOcvOt09Wch_lYj__S2HpxVOW91UAu0vbqPIvJu295g=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xNCsFVlqFDrJo-vASyfBgVZJvBSbzlbrrF37iSuJDimE=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xrBD6p64ajH9qQvTIPcokQM4tK5g_4pBSJU-fPIjRoXA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xbyaJoPojmuM32u6yd2JVoI0b1tAs343zsWt11V2pJP8=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xxcZZ7o3RrfPcFrFnLnf7V4I_GvcNjz3GeYAVHO4nI_A=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_addrstore_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_addrstore_char.ambr index 264f85a5e540..dfe96b1505c9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_addrstore_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_addrstore_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_addrstore_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8NSS_BH_SABV_SAV_UserArgs_MT128xiZdBAHqfXxeEzfE-pAaRGRDvxRBneEX3fMDFgeBaF40=', + 'basename': 'Cijk_Alik_Bljk_F8NSS_BH_SABV_SAV_UserArgs_MT128xi3HGqz8x78XkoSl2HCnW8BWXLUue_raNyIb9pKfnpqc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8NSS_BH_SABV_SAV_UserArgs_MT256xO1rmKN66uCw0-Mzf2qb00m20w7xQxlAHuI1AWERug5Y=', + 'basename': 'Cijk_Alik_Bljk_F8NSS_BH_SABV_SAV_UserArgs_MT256xmo3aArwwcD2eXxZ5E2NSKqH21qQnCteSGQQquBdNmVY=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr index aecb2d0b2b46..145a2f1b6a4c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_datamover_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT16x16x128_MI16bhpL8wcLasPHoBxlw80_42sUESuCuHGXJBbFfUipn3U=', + 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT16x16x128_MI16J_ZHuBHctbApWvtkY-iphiZX4gQ_SpIDxqXNdhhsyO4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT32x32x128_MI16gpItLOZI1sYnFD_KvQHmVk-mb6MopAJLEu41BfJfLy0=', + 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT32x32x128_MI16hupMA4b55t2_4crovZWlG1oaPxc45CLwlKlFoM0GD5g=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_globalwrite_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_globalwrite_char.ambr index e79f66702927..aaa00c226ed5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_globalwrite_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_globalwrite_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_globalwrite_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8NBS_BH_SABV_SAV_UserArgs_MT64x6Fb6HFu5K3-qVvbV4cb9kRGg3kEOyL21-eNUuLpkibOw=', + 'basename': 'Cijk_Alik_Bljk_F8NBS_BH_SABV_SAV_UserArgs_MT64x6B93QxtqgLV5sFfZu84pLkDis2yq3TlVEWPVhMLdYDiA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8NBS_BH_SABV_SAV_UserArgs_MT64x6KW_JS4LDrVoLrqwh-DYAON-e3ZbU3hT6liw-yw1auXc=', + 'basename': 'Cijk_Alik_Bljk_F8NBS_BH_SABV_SAV_UserArgs_MT64x6WrN3bAoT9kDhebyo1-lB3YGMYmf1XWr0FQc2O6dVtq8=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_gsu_on_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_gsu_on_char.ambr index 94f09dc84f08..77eed41dc102 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_gsu_on_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_gsu_on_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_gsu_on_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1-t0s2RupdR8-9OW60HuNQCohY04DUb8dLAMex8180Qo=', + 'basename': 'Cijk_Ailk_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x183IIdF9Fiph3ATKiaufQVyPjYQxxBG5sjY5trEfWHwI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1A_ogBMuzemvbPLFlBIeajI1Z3G8SdnCW7NuH5z3NAvU=', + 'basename': 'Cijk_Ailk_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1WCPXfr6jEK5gZDJUErHDCWYaKFknjKVb0EJPAqJt1nM=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwafeat_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwafeat_char.ambr index ba4e4fcab6c7..236dff8d78e8 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwafeat_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwafeat_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_kwafeat_gfx942_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x164i7uAs8fdNMzaQPXAYedJ9EMpZiIqbY_TlfutTS3CkQ=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16-dOcFFs5eX09CgCZdnDMaxbKhMYm_x6sp4bYH7LPNC4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16wCtZlWtxgjS5__ax2RcrecFL4se8v7wNrDCsrYnWWqw=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x161Z2jkqCvCW6I5GQ5AzsSAJK1FlxiHXLJ6mSiLVwbyKU=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwfeat_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwfeat_char.ambr index d052a4172671..8d0ef02a6a31 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwfeat_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_kwfeat_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_kwfeat_gfx942_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x16x32_MI16x1B4rW9O1-7W_M2CNfx6NuqEqUcUgx69ACJ_2tdNOS_As=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x16x32_MI16x10lVf55swqO3Cws0KLpdEuvFpwrnVjoHkplbsqEBrLW0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x16x32_MI16x1f8ucR7V1cF7-JbNNqICxjZTfVoc8lpYabR645jPg6dc=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_UserArgs_MT64x16x32_MI16x1G5EAWc5B1EEGH4dsGaxPc2IPFo0tL3X8bqlZ4CwHIWA=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_lra_tr_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_lra_tr_char.ambr index c4b399b7e669..f1b72f55a087 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_lra_tr_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_lra_tr_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_lra_tr_gfx950_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_BBS_BH_UserArgs_MT128x128x32_MI163JBhqMkgYeYH_c3Xy44ur3LGLzosXKvWXpw1AgzB8U0=', + 'basename': 'Cijk_Ailk_Bljk_BBS_BH_UserArgs_MT128x128x32_MI16LSCpdr09NGQgG9HL96-UKDs8jKNMfsjwEu8kT9F0H4U=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_BBS_BH_UserArgs_MT128x128x32_MI16XOAcIukvu3yUjB7ZO5WwTimcvIUXkhfut8WmAyzK9aE=', + 'basename': 'Cijk_Ailk_Bljk_BBS_BH_UserArgs_MT128x128x32_MI16eVocLfHkoyrGJhUCLg5WH8qWsI3GCl132oAXUFsENvo=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr index 70f41cd70fba..625cb3aa027c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr @@ -2,15 +2,15 @@ # name: test_r3_streamk_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgERE3TwlyDMsvuIer9R5NRmknAzt5uI1JUl39HC3v0p8=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg4fwNNtnjP214bRKwmg05OmaFNOmNJUp_X3-9iZip5X8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgZNYYx4JszIzTbs85275vCiS-l4OVnFf-dkY8yp8qEgQ=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg6fVhyCvIselcsGGSBDys2ZdfOD1ZkFL8Q3XUww0NZJs=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgqo3oMEiheKtQz1ehFT9CkmQHZYz7xqbcycfWQWqrXu0=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgh5ubNXx_pUxTub2_WuYpzPwRYNgneH328u9XiXZh3hk=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_xccm_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_xccm_char.ambr index c72b77038972..1aab6e105656 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_xccm_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_xccm_char.ambr @@ -2,7 +2,7 @@ # name: test_r3_streamk_xccm_gfx942_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16BDKtwJjnkfPbR294Jz7lZSalFYcqGiMbbxOHzAxY_98=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16rU9HIWb2ImZ3D7BIRc775qg0CmqWscEpM-fJ1TaUWlY=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_subtile_lr_fp8_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_subtile_lr_fp8_char.ambr index 7457061294d8..3cf735958fe7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_subtile_lr_fp8_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_subtile_lr_fp8_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_subtile_lr_fp8_gfx950_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_BH_UserArgs_MT128x128x256_MIe8a1FKEVFO51ZR73Klfcd8YdmVMncnMcXkyvdtaTJZk=', + 'basename': 'Cijk_Alik_Bljk_F8SS_BH_UserArgs_MT128x128x256_MIoDxgl1aaBul3_12uagjRBk74pae13ZptOXQuPvTBQM0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_BH_UserArgs_MT64x64x256_MI16EfbOffhey2UkUTOrYZJUNiHGbj-2DCiZXgcv1BlbY-Q=', + 'basename': 'Cijk_Alik_Bljk_F8SS_BH_UserArgs_MT64x64x256_MI166BsEZUZyLE5ZC-6lhVdLOOd_xl1L_fzXhBF0ChGBPw8=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_asmaddr2_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_asmaddr2_char.ambr index 12e99076ede4..cf8f7f118134 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_asmaddr2_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_asmaddr2_char.ambr @@ -2,7 +2,7 @@ # name: test_r4_asmaddr2_bf16_srvw_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_UserArgs_MT128x128x16PFyxw_qyKhjeq-I_1RrRLFXkTfVpfnlIELFNPeGyBuM=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_UserArgs_MT128x128x161W7NdMHxeBsz9ec8T2KlDepFRPFBf11LZgPZ7lxJ3pA=', 'err': 0, }), ]) @@ -10,7 +10,7 @@ # name: test_r4_asmaddr2_fp32_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x256x16_MI16x16xqo6YNjRkFZp2b46oKCZQw2NaUJTdDk1smSif9QtB4Gk=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x256x16_MI16x16xHWel8pi7RV3P3ERIGIgVg301oZEiNMFHYNZkKxDr1o0=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_fp8mx_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_fp8mx_char.ambr index b9718ad05753..508818eaac4a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_fp8mx_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_fp8mx_char.ambr @@ -2,19 +2,19 @@ # name: test_r4_fp8mx_gfx950_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg8L3sqJBSVtOxLApTt8ns4lG2vp07pgtwTsXZkRI7PsU=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgAwIswm2k9JtrDkjHsh6ohnvPyCC0L9jBBPJIPqja4uY=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgsfpfmMeSkkCxx8FyItttkvW5cKVIP7-Upms3QYnVQ2w=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgpXG1f-uYO9z6h5SLsK1zRZif82ypb1wCrO_1uhlk4lI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgwuQRk8XNwNAvyG7neAMuNGJPI6A5lsHIdMJoSjsGvnw=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgr_ZGdqQv9oJsdIa0XP8GAHQjN6mY33dAo9hwhVEoXi8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgxVzl5H9-zRKcCFf2tXNa1n4Z1PdIEORajKzZgCdHLKY=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArguyEAeU9IjfzRhYOA3SLm5tRCOUroroBQqgEIRm-H5hw=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_shiftvector2_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_shiftvector2_char.ambr index 498fe24debc4..5d3f4aad0f4b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_shiftvector2_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r4_shiftvector2_char.ambr @@ -2,15 +2,15 @@ # name: test_r4_shiftvector2_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x64x16_SN_LDSB0_m8KnuJM8bxs0ZxEdGO9CdkXUnaJ6PI_iwU01_zYQLs=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x64x16_SN_LDSB0jE9CzAXMhC1g_gIMizS1JsSSwiVF-x3wsjXCsvRuuEA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x128x16_SN_LDSB0ZipCF56AgYIiW2_828YIqqhyG94FqTPSZvaxiBtrGEM=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x128x16_SN_LDSB0JZxUHB9vuWyLfjGMQGAS_8O3l4gR9yxU1GPrfS5K71M=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x64x16_SN_LDSB0_CiXkti3-vriks8cKWXcsByJsB8xH-uIy0eOEnh04r7c=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT64x64x16_SN_LDSB0_fp3wANJpuuDc9WqnM4hjBUDQAijVD5Xm8dpK97BxaJs=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_char.ambr index cf7340aaa039..83ce16ba17ff 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_char.ambr @@ -2,11 +2,11 @@ # name: test_gfx90a_dominant_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT128x3Y0I7CiZ4nC498BtHZ6cEOnv3tbSko7S3Ff3eMOPr-cg=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT128x3UIP6KzpUcosxo964LvOLQC3nASVpzLBQd2_KT_4YSso=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x16K9QZAGckF7NodlXkqKz8nrTsXrkTkbF66OiV9n073lY=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x16agmIL6y10cZtrXKFT7E3gAP82c9yWMUusXkKcJk7ht4=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_db_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_db_char.ambr index f19f786b12c1..376030e180f3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_db_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_db_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx90a_db_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT4x16x16_MI4x4x4_SNjyD3aAwTJ2eNaSp2jxa4Obq4sukgCwVMjbsIemgEY9M=', + 'basename': 'Cijk_Ailk_Bjlk_D_B_UserArgs_MT4x16x16_MI4x4x4_SNUc5_pkvU1SKMUxxGFQkvghrHrJivG1xl5AER7XlkuNQ=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_hhs_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_hhs_char.ambr index 000a9d1cf2dd..1344d0ca8cb6 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_hhs_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx90a_hhs_char.ambr @@ -2,11 +2,11 @@ # name: test_gfx90a_hhs_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xHRuhvBViicT3noFtBDwnpjtYQ3_CJM7ztKFqfEPBfKw=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xdIEqp-2AFSpbWYGkFRZ1FRFGRsevfaU8SShXwJkdXtM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xMlKNDXKjpLB9WgTjbGePxpIlPeTmAzug3fKSlRO8ouk=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xmGiE8qk8K8rOoqyKPmnCETwgPw4z13YmjhA8QlzCpv8=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_char.ambr index 7500a179f19b..fe2ff6f7afe4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_char.ambr @@ -2,19 +2,19 @@ # name: test_gfx942_dominant_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT128x32G6y_zoH0amPj5mOXPtke7tBwRDrOq6odQHhskq1Ybl8=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT128x320JIwE5NeelTJhMAoNuBh6US1oTv-W1Bybo7E_hd9CQ0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT128x32oTwfDt5dPtAY1Y9ns-3PxlSglcrvGnRq88d-lSZ6eR4=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT128x32ioeZvUPBhzvY_EA_tMi4dwxnlPblbct4ywwQytTSjY0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xlL0CUbs9uueyP65OKDJ_sdo3P8njgxw4Cj7tNFo9TMk=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xYai5bTAKL7uvZKtRNRP_cK_FzgdTauHEQvUi5bvdI2I=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xvwFrc_jgkjXOIgw9z-djpQp7rXqFKxVqRCXHwAXzXrs=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_A_S_UserArgs_MT64x16xd5ynrlbjTw80bt2f4rOd0uFZ5ZESO6eWp_q_upItI7M=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_db_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_db_char.ambr index 9c358f2dc608..185e19a5b8f0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_db_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_db_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx942_db_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_qvUhbHfvb0RJnShrX3FG1KTbJ84KB3MUi1HA5yOVPBY=', + 'basename': 'Cijk_Ailk_Bljk_D_B_UserArgs_MT16x16x8_MI16x16x1_vNagNvbj0poeflJ8cvxZQMbLgNXrHi0ui48LR42swJQ=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_f8n_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_f8n_char.ambr index eb2b5e444e69..6d561ed1f1cf 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_f8n_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_f8n_char.ambr @@ -2,35 +2,35 @@ # name: test_gfx942_f8n_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_56uBj1j0YpVWr60itf4YisFPAKFBySqS6Lg3JYCRDfU=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_2ALACyNMglfwK_tpUchmlD1_Y11EGJFvH2Wm7HAHvk4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_CdnrmkWdpDY-zo3utmqPdM29BFiv3sKLkfzi19U4bCQ=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_Mj1Wx_DH8KsYeKSTBWwtx5e1AH59EZddjM2OViFqUcg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_Okqk9rha3n2-k5W-zeVuFBEBHbXQ8Bev-T4OIb7d2u0=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_PQzYOC5Y-CAiJBRNhZJv_1PEdtoGu3tFQJQSKwKlzvk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_RtXhlpeg-2XneycahQkWBBPsyqs_IiizaOo1VDc0SXY=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_SZ3Sn5yB9GEr-t-TZR74_KSu5nOFC6qBUsjrJtkR4YA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_dwYF7XMW9wx-jVu_6Hk7ZtMv7yh-Alx6hGbywNqmyJg=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_XD-nM1GqZJd8f-rzMfaABp0ZPsfKozu5Ox9g-TOa9Rw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_eeyOykHfZMjg6O8pMY2MWkVWQLA19fwp_JtwPForVT0=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_k0uSUu6szNRh0phhETuGmgXcVb6cKpjcVYLwGDGbHKg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_uf_XRGjhBEnCBVo9sRJ_2iItq03ozYyf2JW8Bt5AZNI=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_sR1zcBsdhtXxlGcB8Bby68r34DxLpML5JCgxZK2XBwA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_x3OlLcLiJSS3-3bmJ892aUBTnuBwfeefXCSxa7xfGE4=', + 'basename': 'Cijk_Ailk_Bljk_F8NHS_BH_Bias_S_HA_S_SAB_SCD_SAV_tDxRX6EcWIUfOKtT-LQATURjDuKgRLOAFHP0pwbWqB8=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_gg_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_gg_char.ambr index 1475874c1c6e..1d813a246b88 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_gg_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_gg_char.ambr @@ -2,19 +2,19 @@ # name: test_gfx942_gg_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x10UqAgYlA342ZCvgHnUXtd6-pmkCbH1x2oLDW8sdZivo=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x18cEyiVUWLX0B-XLXHMI_RDyOY_KRqqUEm9q9Ux1axhI=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1iZcuH-fyOxqb5-rSqSGoLxvtWVQw4pnzqrSEYiDqTAE=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1AizA477ccYymmHSeBfXJK-6U8JVi33ksO846nj7d0Dg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1k5e6LAts6xhYiyHgrW13xJ23IY7MJMWb2P0KU4fMuLI=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1gVZ9XL2HGvJ41QuykF_0C03QGc9nu6EiP4TXI_woQNo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1lYTnor3-NYWo-QrMHaJ6XQnR9F9d1MYQxeEVJxFNsHU=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_HA_S_UserArgs_MT128x1mKblct5xvrhTEgc6hbSR0lc9jzQ8uvOamJi2mT-e5uU=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_grad_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_grad_char.ambr index cfe09fd956ed..3a39c3fe972c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_grad_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_grad_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx942_grad_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_D_GradH_A_S_UserArgs_c5YoO_NdNcopCC02ED2mqLlFGmAFxwzWkTLrTjyJuBk=', + 'basename': 'Cijk_Ailk_Bljk_HHS_BH_Bias_D_GradH_A_S_UserArgs_goEFq2CcwZWVz3-2_6oCx4e4CR8--1OSwXRWruaXLyQ=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_hss_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_hss_char.ambr index 1f9f0d86b841..cbfdcb028e3f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_hss_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx942_hss_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx942_hss_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_S_UserArgs_MT64x16x32TLJYBDBklDSArFgO7MxaEnjkKxYTbisvgeV4Hod2Yks=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_S_UserArgs_MT64x16x322zIYz1_KygY0369WUx7KLN6yXwo4f-pydiYvicEERCI=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_bbs_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_bbs_char.ambr index cd52f8588723..89f3b45c1de0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_bbs_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_bbs_char.ambr @@ -2,11 +2,11 @@ # name: test_gfx950_bbs_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x16Gh-5_v4fCd2gY8w0-v1MEoWCKxyoWoRHIwbwymdWcK8=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x161m0jTQefkqSAv3OZYxyLzSiWVndKlObrnPeyG6j-66I=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x16qSxWn6OS9r0aE26SFeHN_V6kzab00FsHImCK8QCa3Ag=', + 'basename': 'Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_UserArgs_MT64x16o_GJOcaIRREt9zcrDBGHFa-HoJJ1_mUSKPnQ27bwgto=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_char.ambr index 0da43c7260ad..335721131a80 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_char.ambr @@ -2,11 +2,11 @@ # name: test_gfx950_dominant_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8HS_BH_UserArgs_MT64x64x128_MI16ngo06J1wnPqA1TFl0YlpUkRMBU_8KaYrQg5oblfoFMs=', + 'basename': 'Cijk_Alik_Bljk_F8HS_BH_UserArgs_MT64x64x128_MI163KwMd4yCCfSEdlQ5Q9Km4KySjxJGTtLr9z5mO7AXzIo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8HS_BH_UserArgs_MT64x64x128_MI327yJ1bAZOUn4_mQ0IzxOyl1wQC7PCHBatXG23hySqKWI=', + 'basename': 'Cijk_Alik_Bljk_F8HS_BH_UserArgs_MT64x64x128_MI32k3Ou03nJpF1A4gaILG_QPDWaCmOlFWmJaebbqhH3smc=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_dtl_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_dtl_char.ambr index 4f7083b8130a..f8a3e08b1497 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_dtl_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_dtl_char.ambr @@ -2,19 +2,19 @@ # name: test_gfx950_dtl_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x17ZfH8sIXDpHWSCqCIlgFw3HiH3uOs9GM8L39cC9Y4Z8=', + 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1A9SiADaoeP82UR0uhsyijTk9u4yN9pUYAMaof1LB-U0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1ANWiHbil0rE6VJVArfGep9h4i-pyN0eBGpGzfYTVNvU=', + 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1Dq9DxStO4c65Ov6MGaKHo1Ji0TLLD8hiK8jUT8D9-6o=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1Cu7-kHJoBw4kmauDCwX22jebH067L3c4hLuy5kQgjHc=', + 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1SCgL-wjRrvmjPqpX3elXMywNejRXdEUbwXA1ltrqajM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1uRBy80JjSkd0CA9vRmQrGl5r-YX8xMLyNM_h5Y8Rpek=', + 'basename': 'Cijk_Alik_Bljk_S_B_UserArgs_MT16x16x32_MI16x16x1xPExKy9_NZ0rN7s7vvsf9t7YUauZkDxhdsgH4MKvt7o=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hhs_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hhs_char.ambr index d2833f8d2d5e..22187ed45ea3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hhs_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hhs_char.ambr @@ -2,11 +2,11 @@ # name: test_gfx950_hhs_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xNGrRN8Ixd2LmpfLrYeJ60I0abw9xakxdLe4E4Mq8cCc=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16x47toi3U3krG93xNLSPOl3FZunggcGIJH4G9UpE6mSZQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xNPAx4Kxny0LKRn2o2e5XbVBhw3pj5kWPX5cQUr2I9To=', + 'basename': 'Cijk_Alik_Bljk_HHS_BH_Bias_A_S_UserArgs_MT64x16xBuGz-IOiSPSkO43EvdPHCZy-uaKhmAYuiwqFJv0tThk=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hss_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hss_char.ambr index bb8d88175f68..6bdc9d1c7efc 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hss_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_hss_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx950_hss_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_SH_HHS_BH_Bias_UserArgs_MT64x64x33trODkxQLBnPwNqNjeelqiAn2LRFJ6orFUDRQdKexJ0=', + 'basename': 'Cijk_Ailk_Bljk_SH_HHS_BH_Bias_UserArgs_MT64x64x3xgEOizjJ9YFvxkZkRZpMDYlmFWyoF_9m7RP7Dan9rvo=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_i8gsu_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_i8gsu_char.ambr index c5caef5399e4..a2d078cec6b4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_i8gsu_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_seed_gfx950_i8gsu_char.ambr @@ -2,7 +2,7 @@ # name: test_gfx950_i8gsu_golden list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_I8I8S_BH_HA_S_UserArgs_MT16x16x644mdrWH88zAGgUlStBTuDVecmXIeDQaHbnRucu8vYzEw=', + 'basename': 'Cijk_Ailk_Bljk_I8I8S_BH_HA_S_UserArgs_MT16x16x64Rk90Q2KBm8gQk5P0oDNjHW-wK4-lVjNW9PrOyvwi2lk=', 'err': 0, }), ]) From d09e44a374f0984c2ac77a26c185650c65bef122 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Thu, 25 Jun 2026 16:34:14 -0500 Subject: [PATCH 05/20] [tensilelite] Reset StreamK work-stealing counters on deferred-store kernel end StreamK work-stealing (WS) kernels coordinate through device-global work-queue and completion counters. On the deferred-store / K-split epilogue path the kernel ended via SEndpgm without emitting the StreamK end-of-kernel reset (only the non-deferred functionEnd path emitted it). The counters therefore kept their drained terminal value across launches, so the first launch did the real work and launches 2..N found the queue exhausted and exited as no-ops. This produced physically impossible benchmark numbers (per-launch time collapsing below the arithmetic floor, us < min_us, super-peak Gflops) and would silently no-op repeated real-world calls until the buffer was re-zeroed. - Fix 1 (root cause, KernelWriter.py): emit skComponent.kernelEnd() in the hasAnyDeferred epilogue branch before SEndpgm, mirroring functionEnd. WS gating lives inside kernelEnd, so non-WS kernels are unaffected. - Fix 2 (defense-in-depth, tensile_host.cpp, default OFF): optional per-launch hipMemsetAsync of the Synchronizer for StreamK-WS solutions, gated by env TENSILE_STREAMK_WS_RESET_SYNCHRONIZER (off by default; Fix 1 is sufficient). Validated on gfx950/MI355X with a --streamk-ws-mode only library: per-launch time is now flat at the physical ~4.56 ms across iters (was collapsing toward sub-us), adaptive timing is sub-peak with us >= min_us, and --verify at iters=10 passes (norm_error ~1e-5) on WS-selecting shapes. (cherry picked from commit 58f9489a7494e508b8063c4abb6d8da86dea394c) --- .../amd_detail/rocblaslt/src/tensile_host.cpp | 18 ++++++++++++++++++ .../tensilelite/Tensile/KernelWriter.py | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp b/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp index 323e802910b2..cacad1405a0b 100644 --- a/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp +++ b/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp @@ -3445,6 +3445,24 @@ rocblaslt_status runContractionProblem(rocblaslt_handle handle } isPreloaded = true; } + // Defense-in-depth (default OFF): optionally zero the StreamK + // work-stealing Synchronizer region before each launch so stale + // work-queue/completion counters from a prior launch cannot make + // this launch no-op. The primary fix is kernel-side (the WS + // kernel-end reset); enable this only for debugging via + // TENSILE_STREAMK_WS_RESET_SYNCHRONIZER=1 (or "true"). + static const bool resetStreamKWSSync = []() { + const char* env = getenv("TENSILE_STREAMK_WS_RESET_SYNCHRONIZER"); + return env && (std::string(env) == "1" || std::string(env) == "true"); + }(); + if(resetStreamKWSSync && solution->sizeMapping.streamK > 0 + && solution->sizeMapping.streamKWorkStealing && prob.Synchronizer) + { + // Size mirrors the one-time allocation in hipblaslt.cpp + // (16 * 409600 ints); no symbolic constant exists for it. + static_cast(hipMemsetAsync( + prob.Synchronizer, 0, 16 * 409600 * sizeof(int), prob.stream)); + } status = hip2RocStatus( adapter->launchKernels(kernels, prob.stream, nullptr, nullptr, isPreloaded)); if(rocblaslt::Debug::Instance().printLogAsMarker()) diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index a35a05cd09c1..4edd71409743 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -5178,6 +5178,11 @@ def kernelBodySubtile(self, kernel, tensorParametersA, tensorParametersB): module.add(kernelEndLabel) if kernel["ProblemType"]["OutputAmaxD"]: module.add(self.insertAmaxD(kernel)) + # Mirror functionEnd's StreamK kernel-end reset so the deferred-blocks + # epilogue also restores the WS work-queue/completion counters between + # launches (the reset is internally gated on StreamKWorkStealing). + skComponent = Component.StreamK.find(self) + module.add(skComponent.kernelEnd(self, kernel)) module.add(SEndpgm(comment="Kernel End")) else: # If activation was deferred but no other deferred blocks exist, emit it before functionEnd From c457dd2e3bd60b6a4a6f01465dac7e8513d55e02 Mon Sep 17 00:00:00 2001 From: Alex Vasile <48962821+Alex-Vasile@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:18:43 -0400 Subject: [PATCH 06/20] Remove MergeFiles (cherry picked from commit 2c5a52c5d472ce268943b3dc1ea6553222936df7) --- .../common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml | 1 - .../Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml | 1 - .../Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml | 1 - .../common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml | 1 - ...subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml | 1 - ...subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml | 1 - ...subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml | 1 - ...subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml | 1 - .../gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml | 1 - .../Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml | 1 - .../Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml | 1 - 11 files changed, 11 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml index b6efe0dd5d9c..227b789a06e0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml @@ -54,7 +54,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml index 651804584d46..8974de3cdce9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml @@ -82,7 +82,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml index ce66d2087dd7..1d5a220acb90 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml @@ -28,7 +28,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml index 1db7acd1a76b..a821264eae8e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml @@ -86,7 +86,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml index 218f7e6324cc..359f88fcd102 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml @@ -59,7 +59,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml index 17293c1b1150..f5c8667bb192 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml @@ -61,7 +61,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml index f2069a451f78..f307063f1941 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml @@ -80,7 +80,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml index 68aa16f6e2e1..943dcb1c7ffe 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml @@ -88,7 +88,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml index 8d754fcaa9cc..701b23e69327 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml @@ -23,7 +23,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml index effae31a7944..bdbff23d600e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml @@ -24,7 +24,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml index 3df838197f8f..f2f427cc43dc 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml @@ -26,7 +26,6 @@ GlobalParameters: PrintLevel: 3 Device: 0 CMakeBuildType: Release - MergeFiles: False KernelTime: True MaxWorkspaceSize: 13421772800 DataInitTypeA: 3 From 1796e313580dee60434008e074a3a03e096782fe Mon Sep 17 00:00:00 2001 From: Alex Vasile <48962821+Alex-Vasile@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:49:45 +0000 Subject: [PATCH 07/20] Further fixes (cherry picked from commit 65c65bb3c8280dc34d0662d3930194c3cf1859f4) --- .../common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml | 2 -- .../Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml | 2 -- .../Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml | 2 -- .../gemm/gfx950/subtile_mxfp8_correctness_regression.yaml | 2 -- ...ubtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml | 2 -- ...ubtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml | 2 -- ...ubtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml | 2 -- ...ubtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml | 2 -- .../gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml | 2 -- .../Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml | 2 -- .../Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml | 2 -- 11 files changed, 22 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml index 227b789a06e0..975cbb9066fc 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_residual_fail_classes_repro.yaml @@ -66,8 +66,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 32 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml index 8974de3cdce9..81e157ee242d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/mxfp8_verify_fail_repro.yaml @@ -94,8 +94,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 32 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml index 1d5a220acb90..8efe67b4e156 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_bias_sav.yaml @@ -42,8 +42,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml index a821264eae8e..be33e9ae4caf 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression.yaml @@ -98,8 +98,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml index 359f88fcd102..76adfd5e7761 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt192x256_plr1_wgm4.yaml @@ -71,8 +71,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml index f5c8667bb192..27e7aed96127 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x192_plr1_wgm1.yaml @@ -73,8 +73,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml index f307063f1941..2b7f55179a49 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt256x320_plr0_wgm1.yaml @@ -92,8 +92,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml index 943dcb1c7ffe..babf86a76fa4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_correctness_regression_mt320x256_plr1_wgm4.yaml @@ -100,8 +100,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml index 701b23e69327..7023ee784f16 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_multi_du_multi_partition.yaml @@ -35,8 +35,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml index bdbff23d600e..bdeae6934675 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop.yaml @@ -36,8 +36,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml index f2f427cc43dc..8a108858e1a9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/gemm/gfx950/subtile_mxfp8_tail_loop_smoke.yaml @@ -38,8 +38,6 @@ GlobalParameters: MXScaleFormat: 1 BoundsCheck: 0 KeepBuildTmp: True - DeviceLDS: 163840 - MaxLDS: 163840 CpuThreads: 1 RotatingBufferSize: 0 PrintSolutionRejectionReason: True From e56cb2462f6d84aaddf3413eb9a91a4908c5185a Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Thu, 25 Jun 2026 17:19:12 -0500 Subject: [PATCH 08/20] [tensilelite] Drop host-side StreamK WS synchronizer reset on this branch The cherry-picked WS counter-reset fix included an optional host-side per-launch Synchronizer memset gated on solution->sizeMapping.streamKWorkStealing. That SizeMapping field does not exist on this branch (work-stealing is not exposed via the hipBLASLt API / to clients here), so the host block failed to compile (no member named 'streamKWorkStealing' in 'TensileLite::SizeMapping'). Remove the host-side block entirely. The kernel-side reset (KernelWriter.py, emitting the StreamK kernel-end reset on the deferred-store epilogue path) is the actual fix and is retained; it is sufficient on its own. --- .../amd_detail/rocblaslt/src/tensile_host.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp b/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp index cacad1405a0b..323e802910b2 100644 --- a/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp +++ b/projects/hipblaslt/library/src/amd_detail/rocblaslt/src/tensile_host.cpp @@ -3445,24 +3445,6 @@ rocblaslt_status runContractionProblem(rocblaslt_handle handle } isPreloaded = true; } - // Defense-in-depth (default OFF): optionally zero the StreamK - // work-stealing Synchronizer region before each launch so stale - // work-queue/completion counters from a prior launch cannot make - // this launch no-op. The primary fix is kernel-side (the WS - // kernel-end reset); enable this only for debugging via - // TENSILE_STREAMK_WS_RESET_SYNCHRONIZER=1 (or "true"). - static const bool resetStreamKWSSync = []() { - const char* env = getenv("TENSILE_STREAMK_WS_RESET_SYNCHRONIZER"); - return env && (std::string(env) == "1" || std::string(env) == "true"); - }(); - if(resetStreamKWSSync && solution->sizeMapping.streamK > 0 - && solution->sizeMapping.streamKWorkStealing && prob.Synchronizer) - { - // Size mirrors the one-time allocation in hipblaslt.cpp - // (16 * 409600 ints); no symbolic constant exists for it. - static_cast(hipMemsetAsync( - prob.Synchronizer, 0, 16 * 409600 * sizeof(int), prob.stream)); - } status = hip2RocStatus( adapter->launchKernels(kernels, prob.stream, nullptr, nullptr, isPreloaded)); if(rocblaslt::Debug::Instance().printLogAsMarker()) From 8971162832cece9aff63c064b8f6a783cdc7d208 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 6 Jul 2026 16:35:58 +0000 Subject: [PATCH 09/20] [tensilelite] Document StreamKWorkStealing required-param golden change (D17) Add a characterization DECISIONS.md entry classifying the .ambr golden churn from promoting StreamKWorkStealing to the required (min-naming) parameter set as an intended behavior change (SKWS0 name token + all-arch identity-hash basename churn, no assembly/err/kernel-count change), mirroring D16. Satisfies the hipblaslt-pr-quality overlay's characterization-snapshot discipline. --- .../Tests/unit/characterization/DECISIONS.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md index 505ca066d3c0..b5c5b8d187c3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md @@ -346,4 +346,31 @@ No other equivalent mutants are accepted yet; widening rounds append their accepted equivalents/pragmas here, each with its one-line reason. ## D16 — BufferLoad/BufferStore promoted to Required Parameters -**Context** kernel basename hash changes across all archs; assembly verified unchanged/correct; no err or kernel-count changes." \ No newline at end of file +**Context** kernel basename hash changes across all archs; assembly verified unchanged/correct; no err or kernel-count changes." + +## D17 — StreamKWorkStealing added to the required (min-naming) parameter set +**Decision:** Accept the regenerated `_codegen` / SolutionClass / ValidParameters goldens after +adding the new codegen parameter `StreamKWorkStealing` to `Common/RequiredParameters.py` +(`getRequiredParametersMin`). Regenerate surgically via the characterization suite; do not +hand-edit goldens. +**Classification:** intended behavior change (overlay class (a)), not a bug and not test fragility. +Adding the parameter to the min-naming set inserts the `SKWS0` token into the verbose solution +name (next to its StreamK siblings `SKA`/`SKFTR`/`SKFDPO`/`SKXCCM`) and perturbs the kernel +identity-hash `basename` on **every** kernel across **all** archs — `num_keys` 334→335. This is +the same required-parameter-promotion footprint as D16 (BufferLoad/BufferStore). +**Why the diff spans all arch nodes (not blanket masking):** the change is a single new required +parameter, so by construction it touches every kernel's identity hash; the wide golden diff is the +faithful, minimal consequence of that one source change, not an unrelated multi-node re-record. +Verified: only `basename` hashes + the `SKWS0` name token + the added roster/valid-values entry +change — **no `err`, instruction-count, or emitted-assembly changes**. `minNaming` still drops the +constant param from the *short* visible kernel name, so no `_WS0` token pollutes production kernel +names. +**Stable-arch signal (gfx908/gfx90a/gfx942):** the digest changes on the stable archs are this +intended naming/identity change, **not** a compiler/codegen regression — root cause is the +required-param addition above; assembly is byte-for-byte unchanged with `StreamKWorkStealing=0`. +**Re-run:** goldens are byte-identical on two further `--snapshot-update`-free runs and the full +`-m unit` suite stays green. +**Alternatives rejected:** (a) keep the param out of `RequiredParameters` — rejected: it would +drop the param from the kernel identity, so two solutions differing only in `StreamKWorkStealing` +would collide on the same name/hash; (b) hand-edit the goldens — rejected: derived state + name +hashes are computed, hand-editing desynchronizes the serialized state from real codegen output. \ No newline at end of file From 9e67bbde5fdd2c60c37ae81a20ce9be29a357b76 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 13 Jul 2026 18:03:12 +0000 Subject: [PATCH 10/20] feat(tensilelite): gate StreamK work stealing to power-of-two XCD counts Derive the StreamK dynamic-queue count per architecture (power-of-two lookup) instead of hardcoding a literal 8. Reject and warn -- rather than silently degrading to a tree-reduction fallback -- on non-power-of-two NUM_XCD devices (e.g. MI300A's 6 XCDs), since the shift/AND fast masking that maps StreamKIdx->queue->cache-line is only valid for a power-of-two queue count. The rejection is enforced host-side (ContractionSolution.cpp softwarePredicate) because MI300A and MI300X both compile as gfx942 and codegen cannot tell them apart; when a solution is excluded a non-work- stealing solution serves the GEMM. --- .../tensilelite/Tensile/Components/StreamK.py | 38 ++++ .../Tensile/SolutionStructs/Solution.py | 9 + .../include/Tensile/ContractionSolution.hpp | 11 ++ .../include/Tensile/SolutionLibrary.hpp | 8 +- .../tensilelite/src/ContractionSolution.cpp | 120 +++++++++++- .../tensilelite/tests/CuCount_test.cpp | 172 ++++++++++++++++++ 6 files changed, 352 insertions(+), 6 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 0d3acac08940..d48c1105ad05 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -355,6 +355,44 @@ def _skv(self, writer, name): _WS_QUEUE_LOG2 = 3 # log2(8) _WS_COMPLETION_COUNTER_OFFSET = 0x80 + # Per-architecture work-stealing / dynamic-queue count. + # + # The dynamic-queue fetch partitions work across one queue per XCD and maps + # StreamKIdx -> queueIdx (StreamKIdx & (numQueues-1)) and queueIdx -> cache + # line (queueIdx << log2(256)) with shift/AND fast masking. That masking is + # only valid when the queue count is a power of two, so the count is keyed + # by architecture here instead of hardcoding a literal 8. + # + # NOTE: MI300A and MI300X BOTH report gfx942; only the physical XCD count + # differs (MI300A=6, MI300X=8). Codegen cannot tell them apart, so gfx942 is + # generated for 8 queues and the host runtime (ContractionSolution.cpp) + # rejects the dynamic-queue / work-stealing path when the device does not + # expose the assumed power-of-two queue count (e.g. MI300A's 6 XCDs). + _WS_NUM_QUEUES_BY_ISA = { + (9, 4): 8, # gfx942 (MI300X) + (9, 5): 8, # gfx950 + } + _WS_DEFAULT_NUM_QUEUES = 8 + _WS_CACHE_LINE_LOG2 = 8 # log2(256): per-queue counters are one per 256B line + + def _wsQueueConstants(self, kernel): + """Return (numQueues, mask, log2Queues, cacheLineLog2) for this arch. + + Centralizes the dynamic-queue / work-stealing fast-mask constants so the + formerly hardcoded "8" lives in exactly one place. ``numQueues`` is + looked up per architecture from ``kernel["ISA"]`` and must be a power of + two for the shift/AND masking to be valid; the host guards the same + assumption at runtime (rejecting non-power-of-two XCD counts like + MI300A's 6). + """ + isa = tuple(kernel["ISA"][:2]) + numQueues = self._WS_NUM_QUEUES_BY_ISA.get(isa, self._WS_DEFAULT_NUM_QUEUES) + assert numQueues > 0 and (numQueues & (numQueues - 1)) == 0, ( + "StreamK dynamic-queue fast masking requires a power-of-two queue " + "count (got %d for ISA %s)" % (numQueues, isa) + ) + return numQueues, numQueues - 1, log2(numQueues), self._WS_CACHE_LINE_LOG2 + def streamKWorkStealingHomeNoReset(self, writer, mod, kernel, sBound, mkLabel): """Disable the home queue's atomic auto-reset when a neighbor could steal. diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index f6e5c2995319..a62709e6fbc4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1663,6 +1663,15 @@ def assignDerivedParameters( "DebugPersistentKernelLoopForever requires StreamK=3 (got %d)" % state["StreamK"]) if state["StreamKWorkStealing"]: + # NOTE: these are codegen-time rejections only; there is no hardware + # context here (MI300A and MI300X both compile as gfx942). The kernel + # hardcodes a power-of-two queue count (8), so the check that the device + # actually exposes a power-of-two XCD count -- and the explicit + # rejection of the dynamic-queue / work-stealing path on non-power-of-two + # devices like MI300A's 6 XCDs (a non-work-stealing solution serves the + # GEMM instead, and the user is warned) -- lives host-side in + # ContractionSolution.cpp (streamKDynamicQueueSupported / + # streamKDynamicQueueUnsupported, wired into softwarePredicate). # Work stealing only exists in the dynamic-queue fetch (auto-mode # SK4 and the SK4 sub-path of SK5). if state["StreamK"] not in (4, 5): diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index e85995673b0f..09dd5c89b2ce 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -382,6 +382,17 @@ namespace TensileLite // the launch grid and the packed args can never disagree. bool streamK5EffectiveDynamic(Problem const& problem, Hardware const& hardware) const; + // Selection-time predicate for the StreamK dynamic-queue / work-stealing + // path. The SK4 and dynamic sub-path of SK5 kernels hardcode a + // power-of-two per-XCD queue count and mask indices with (Q-1); that fast + // masking is only valid when the device exposes a power-of-two number of + // XCDs. Returns false (and warns once) when this solution would take the + // dynamic-queue path but the hardware's NUM_XCD is not a power of two + // (e.g. MI300A = 6), so the solution is EXCLUDED from selection rather + // than silently degraded to tree reduction. All other solutions return + // true. Wired into softwarePredicate() (SolutionLibrary.hpp). + bool streamKDynamicQueueSupported(Problem const& problem, + Hardware const& hardware) const; size_t partialTileSize(size_t skGrid) const; static float computeGranularity(float x); diff --git a/projects/hipblaslt/tensilelite/include/Tensile/SolutionLibrary.hpp b/projects/hipblaslt/tensilelite/include/Tensile/SolutionLibrary.hpp index 689785b7b612..1e94054251aa 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/SolutionLibrary.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/SolutionLibrary.hpp @@ -80,7 +80,13 @@ namespace TensileLite switch(searchType) { case SolutionLibrarySearchType::DEFAULT: - return (*solutions.problemPredicate)(problem) && (*solutions.taskPredicate)(task); + // streamKDynamicQueueSupported() excludes StreamK dynamic-queue / + // work-stealing solutions (SK4 and the dynamic sub-path of SK5) on + // devices whose XCD count is not a power of two, warning the user + // once. This is reject-and-continue: selection falls through to + // another (SK3-static / non-StreamK) solution for the GEMM. + return (*solutions.problemPredicate)(problem) && (*solutions.taskPredicate)(task) + && solutions.streamKDynamicQueueSupported(problem, hardware); break; case SolutionLibrarySearchType::GEMM_TYPE_ONLY: return isGemmTypeSame(solutions, problem); diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 41a41b539cc4..584dd342ab92 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -56,6 +57,56 @@ namespace TensileLite { + namespace + { + // The dynamic-queue StreamK kernels (SK4 and the SK4 sub-path of SK5) + // hardcode a fixed, power-of-two per-XCD queue count for fast index + // masking (StreamKIdx & (Q-1), queueIdx << log2(256)). Codegen emits + // Q = 8 for gfx942/gfx950 (see StreamK.py _WS_NUM_QUEUES_BY_ISA), so the + // per-XCD work-queue region reserved in the Synchronizer workspace is + // always 8 cache lines (256 B each) wide. + constexpr size_t StreamKDynamicQueueCount = 8; + + // The dynamic-queue fetch / work stealing is only correct when the + // device exposes a power-of-two number of XCDs (one queue per XCD), + // because the kernel masks with (Q-1). MI300A and MI300X BOTH report + // gfx942, but MI300A has 6 XCDs (not a power of two), so the fast-mask + // assumption breaks and the dynamic-queue path is NOT supported there. + // Returns true when NUM_XCD is known and is NOT a power of two; returns + // false when the XCD count is unknown so historic behavior is preserved. + // The numeric power-of-two classification is intentionally isolated here + // so it stays trivially unit-testable (see CuCount_test.cpp). + inline bool streamKDynamicQueueUnsupported(Hardware const& hardware) + { + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) + return false; + size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; + return numXCD == 0 || (numXCD & (numXCD - 1)) != 0; + } + + // Emit a single, user-visible warning (not once-per-call spam) when a + // StreamK dynamic-queue / work-stealing solution is excluded from + // selection because the device's XCD count is not a power of two. This + // is what surfaces the reject to the user instead of silently degrading + // to tree reduction. + void warnStreamKDynamicQueueUnsupportedOnce(Hardware const& hardware) + { + static std::once_flag warnedFlag; + std::call_once(warnedFlag, [&]() { + size_t numXCD = 0; + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU != nullptr && hipAMDGPU->analyticalHardware != nullptr) + numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; + std::cerr << "hipBLASLt Warning: StreamK dynamic-queue (work-stealing) solutions " + "require a power-of-two XCD count; this device reports NUM_XCD=" + << numXCD + << ", so those solutions are excluded from selection and a " + "non-work-stealing solution will be used instead.\n"; + }); + } + } + enum class KERNELARGTYPE { NORMAL = 0, @@ -3145,6 +3196,26 @@ namespace TensileLite const bool effectiveDynamic = (sizeMapping.streamK == 5) ? streamK5EffectiveDynamic(problem, hardware) : false; + // Defensive: dynamic-queue / work-stealing StreamK solutions are + // excluded from selection on devices whose XCD count is not a power + // of two (see streamKDynamicQueueSupported() wired into + // softwarePredicate). The normal path therefore never reaches solve() + // for such a solution; a different (SK3-static / non-StreamK) + // solution serves the GEMM instead. If we DO get here it means the + // software predicate was bypassed (e.g. an explicit select-by-index), + // so reject EXPLICITLY rather than silently running the fixed-mask + // kernel with a mismatched queue count (which would corrupt results). + const bool dynamicQueuePath + = (sizeMapping.streamK == 4) + || (sizeMapping.streamK == 5 && effectiveDynamic); + if(dynamicQueuePath && streamKDynamicQueueUnsupported(hardware)) + { + warnStreamKDynamicQueueUnsupportedOnce(hardware); + throw std::runtime_error( + "hipBLASLt Error: StreamK dynamic-queue (work-stealing) solution selected on a " + "device whose XCD count is not a power of two; this kernel is unsupported here. " + "Select a non-work-stealing solution instead."); + } if(sizeMapping.streamK == 4) sk.reduction = origami::reduction_t::tree; else if(sizeMapping.streamK == 5) @@ -3169,11 +3240,12 @@ namespace TensileLite size_t idealWorkspace = partialTileSize(sk.grid); // SK4 and SK5-dynamic need the per-XCD work-queue region; SK5-static // sizes like standalone SK3. - if(sizeMapping.streamK == 4 - || (sizeMapping.streamK == 5 && effectiveDynamic)) - idealWorkspace += 256 * 8; + if(dynamicQueuePath) + idealWorkspace += 256 * StreamKDynamicQueueCount; // If given workspace is less than ideal, we can fall back to DP mode - // Performance will likely be lower, but the kernel can run if workspace is unavailable + // Performance will likely be lower, but the kernel can run if workspace is unavailable. + // (The non-power-of-two XCD case is handled earlier by explicit + // rejection, not by a silent fall back to tree reduction.) if(idealWorkspace > problem.workspaceSize()) { sk.reduction = origami::reduction_t::tree; @@ -3599,9 +3671,13 @@ namespace TensileLite else if(skGrid > 0 && (tiles % skGrid != 0 && !streamKDP && !forceDPOnly)) { size_t idealWorkspace = partialTileSize(skGrid); + // Reserve the per-XCD work-queue region for the dynamic-queue + // path. Sized to the kernel's fixed queue count (8); this may + // slightly over-report on a device that falls back to tree + // reduction (e.g. MI300A), which is safe (never under-sized). if(sizeMapping.streamK == 4 || (sizeMapping.streamK == 5 && effectiveDynamic)) - idealWorkspace += 256 * 8; + idealWorkspace += 256 * StreamKDynamicQueueCount; // If given workspace is less than ideal, we can fall back to DP mode // Performance will likely be lower, but the kernel can run if workspace is unavailable if(idealWorkspace <= problem.workspaceSize()) @@ -3889,6 +3965,40 @@ namespace TensileLite return effectiveDynamic; } + bool ContractionSolution::streamKDynamicQueueSupported(Problem const& problem, + Hardware const& hardware) const + { + // Only StreamK solutions can ever take the dynamic-queue / work-stealing + // path; everything else (SK3-static, non-StreamK) is always selectable. + if(sizeMapping.streamK != 4 && sizeMapping.streamK != 5) + return true; + + // Fast/common path: on hardware whose XCD count IS a power of two the + // fixed power-of-two queue masking is valid, so nothing is excluded. + // This also covers the "unknown XCD" case (predicate returns false), + // preserving historic behavior. Checked before streamK5EffectiveDynamic() + // so the mainline gfx942(MI300X)/gfx950 path stays cheap. + if(!streamKDynamicQueueUnsupported(hardware)) + return true; + + // XCD count is not a power of two. Only the dynamic-queue sub-path is + // affected: an SK5 solution that resolves to the static (SK3) sub-path + // for this problem stays valid and selectable. + const bool dynamicQueue + = (sizeMapping.streamK == 4) + || (sizeMapping.streamK == 5 && streamK5EffectiveDynamic(problem, hardware)); + if(!dynamicQueue) + return true; + + // Reject-and-continue: exclude this dynamic-queue / work-stealing + // solution from selection (return false) and warn the user ONCE so they + // are informed rather than silently degraded to tree reduction. Because + // this is a selection-time predicate, other solutions (SK3-static, + // non-StreamK) remain available to serve the GEMM. + warnStreamKDynamicQueueUnsupportedOnce(hardware); + return false; + } + namespace { size_t getSKGridImpl(ContractionSolution const& self, diff --git a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp index 3307bad84089..2cadc57a3e84 100644 --- a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp @@ -734,3 +734,175 @@ TEST(Sk3Sk5OffPartition512Test, NativeSk3MatchesSk5OffHostPack) EXPECT_NE(sk3Pack.grid, sk5OnPack.grid) << "512^3 static path oversubscribes; dynamic path should not match"; } + +// =========================================================================== +// StreamKDynamicQueueXcdGateTest -- MI300A (NUM_XCD=6) reject-and-continue. +// +// The SK4 / SK5-dynamic work-stealing kernels use a fixed, power-of-two per-XCD +// queue count (8) and mask queue/tile indices with (Q-1). That fast masking is +// only valid when the device exposes a power-of-two number of XCDs. MI300A and +// MI300X both report gfx942, but MI300A has 6 XCDs (not a power of two). +// +// Rather than SILENTLY degrading such a solution to tree reduction, the host +// now EXCLUDES the dynamic-queue / work-stealing solution from selection and +// warns the user once, so a different (SK3-static / non-StreamK) solution +// serves the GEMM. This is enforced by ContractionSolution:: +// streamKDynamicQueueSupported(problem, hardware), which is wired into +// softwarePredicate() (SolutionLibrary.hpp) so returning false drops the +// solution from findBestSolution/findAllSolutions. It builds on the file-local +// numeric predicate streamKDynamicQueueUnsupported(hardware): dynamic_cast to +// hip::HipAMDGPU, read analyticalHardware->NUM_XCD, return true when NUM_XCD is +// 0 (defensive) or not a power of two, false otherwise (incl. unknown/no +// analytical hardware -> historic behavior preserved). +// +// Both production predicates live in anonymous namespaces / a .cpp translation +// unit and cannot be linked from here, so -- following the same convention as +// computeStreamKHostPack above (which mirrors solve()'s internal reduction +// decision for host-only testing) -- this test mirrors them and drives them +// through a real hip::HipAMDGPU mock. It validates (a) the Hardware& -> +// HipAMDGPU -> analyticalHardware -> NUM_XCD accessor chain, (b) the power-of- +// two classification (6 -> reject, 8 -> allow, unknown -> allow), and (c) the +// selection-predicate composition (dynamic-queue on 6 XCD -> excluded; other +// StreamK modes / non-power-of-two-agnostic solutions -> selectable). It is NOT +// executed on real MI300A silicon. +// =========================================================================== +namespace +{ + // Byte-for-byte mirror of the anonymous-namespace numeric predicate in + // ContractionSolution.cpp (streamKDynamicQueueUnsupported). Kept in lockstep + // with the production code; if that predicate changes, update this too. + inline bool streamKDynamicQueueUnsupportedRef(Hardware const& hardware) + { + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) + return false; + size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; + return numXCD == 0 || (numXCD & (numXCD - 1)) != 0; + } + + // Mirror of ContractionSolution::streamKDynamicQueueSupported(). Returns + // true when the solution is SELECTABLE, false when it must be EXCLUDED + // (dynamic-queue / work-stealing on a non-power-of-two XCD device). streamK + // is sizeMapping.streamK; effectiveDynamic is the SK5 sub-mode result + // (ignored for streamK != 5). Kept in lockstep with the production member. + inline bool streamKDynamicQueueSupportedRef(int streamK, + bool effectiveDynamic, + Hardware const& hardware) + { + if(streamK != 4 && streamK != 5) + return true; + if(!streamKDynamicQueueUnsupportedRef(hardware)) + return true; + const bool dynamicQueue = (streamK == 4) || (streamK == 5 && effectiveDynamic); + return !dynamicQueue; // dynamic-queue on non-pow2 XCD -> excluded + } + + // gfx942 analytical hardware with a caller-chosen XCD count. NUM_XCD is the + // 5th positional arg of origami::hardware_t (see origami/hardware.hpp). + origami::hardware_t makeGfx942HardwareWithXcd(size_t numXCD) + { + using arch_t = origami::hardware_t::architecture_t; + return origami::hardware_t(arch_t::gfx942, + 304, // N_CU (MI300X SPX) + 163840, + 262144, + numXCD, + 1.0, + 1.0, + 1.0, + 4000000, + 1.2, + 1, + std::make_tuple(0.0, 0.008, 0.0)); + } + + hip::HipAMDGPU makeGfx942DeviceWithXcd(size_t numXCD) + { + hip::HipAMDGPU device; + device.processor = AMDGPU::Processor::gfx942; + device.computeUnitCount = 304; + device.deviceName = "test-gfx942-xcd"; + device.analyticalHardware = std::make_shared( + makeGfx942HardwareWithXcd(numXCD)); + return device; + } +} // namespace + +TEST(StreamKDynamicQueueXcdGateTest, RejectsMi300aSixXcd) +{ + hip::HipAMDGPU mi300a = makeGfx942DeviceWithXcd(6); + Hardware const& hw = mi300a; + EXPECT_TRUE(streamKDynamicQueueUnsupportedRef(hw)) + << "MI300A (NUM_XCD=6, not a power of two) must flag the dynamic-queue " + "work-stealing path as unsupported"; +} + +TEST(StreamKDynamicQueueXcdGateTest, AllowsMi300xEightXcd) +{ + hip::HipAMDGPU mi300x = makeGfx942DeviceWithXcd(8); + Hardware const& hw = mi300x; + EXPECT_FALSE(streamKDynamicQueueUnsupportedRef(hw)) + << "MI300X (NUM_XCD=8, power of two) must keep the dynamic-queue path"; +} + +TEST(StreamKDynamicQueueXcdGateTest, AllowsGfx950EightXcd) +{ + // gfx950 (local MI355X) analytical hardware advertises 8 XCDs. + hip::HipAMDGPU gfx950 = makeHipDeviceWithAnalytical(makeGfx950AnalyticalHardware()); + Hardware const& hw = gfx950; + EXPECT_FALSE(streamKDynamicQueueUnsupportedRef(hw)) + << "gfx950 (NUM_XCD=8) must keep the dynamic-queue work-stealing path"; +} + +TEST(StreamKDynamicQueueXcdGateTest, MissingAnalyticalHardwarePreservesHistoricBehavior) +{ + // The real "unknown -> preserve historic behavior (allow)" path in the + // production predicate is a null analyticalHardware: it returns false so a + // device without analytical info keeps the pre-gate dynamic-queue path. + // (The predicate's separate NUM_XCD==0 guard returns true as a defensive + // fallback, but that state is not constructible here: origami::hardware_t + // divides by NUM_XCD, so a real analytical hardware can never carry 0 XCDs.) + hip::HipAMDGPU noAnalytical; + noAnalytical.processor = AMDGPU::Processor::gfx942; + noAnalytical.deviceName = "test-gfx942-no-analytical"; + Hardware const& hwNoAnalyt = noAnalytical; + ASSERT_EQ(noAnalytical.analyticalHardware, nullptr); + EXPECT_FALSE(streamKDynamicQueueUnsupportedRef(hwNoAnalyt)) + << "Missing analyticalHardware must fall through to historic behavior (allow)"; +} + +// Selection-predicate contract: on MI300A (6 XCD) the dynamic-queue solution is +// EXCLUDED from selection (supported == false) so a different solution serves +// the GEMM, while on MI300X (8 XCD) the identical solution stays selectable. +TEST(StreamKDynamicQueueXcdGateTest, ExcludesDynamicQueueSolutionOnMi300a) +{ + hip::HipAMDGPU mi300a = makeGfx942DeviceWithXcd(6); + hip::HipAMDGPU mi300x = makeGfx942DeviceWithXcd(8); + Hardware const& hwA = mi300a; + Hardware const& hwX = mi300x; + + // SK4 is always dynamic-queue. + EXPECT_FALSE(streamKDynamicQueueSupportedRef(4, /*effectiveDynamic=*/false, hwA)) + << "SK4 work-stealing solution must be excluded from selection on MI300A"; + EXPECT_TRUE(streamKDynamicQueueSupportedRef(4, /*effectiveDynamic=*/false, hwX)) + << "SK4 work-stealing solution must remain selectable on MI300X"; + + // SK5 only takes the dynamic-queue path when it resolves to the dynamic + // sub-mode; the static (SK3) sub-mode stays selectable even on MI300A. + EXPECT_FALSE(streamKDynamicQueueSupportedRef(5, /*effectiveDynamic=*/true, hwA)) + << "SK5-dynamic must be excluded from selection on MI300A"; + EXPECT_TRUE(streamKDynamicQueueSupportedRef(5, /*effectiveDynamic=*/false, hwA)) + << "SK5-static (SK3 sub-path) must remain selectable on MI300A"; +} + +// Non-dynamic-queue solutions must never be excluded, so the GEMM still runs. +TEST(StreamKDynamicQueueXcdGateTest, KeepsNonDynamicQueueSolutionsOnMi300a) +{ + hip::HipAMDGPU mi300a = makeGfx942DeviceWithXcd(6); + Hardware const& hwA = mi300a; + + EXPECT_TRUE(streamKDynamicQueueSupportedRef(0, /*effectiveDynamic=*/false, hwA)) + << "Non-StreamK solution must remain selectable on MI300A"; + EXPECT_TRUE(streamKDynamicQueueSupportedRef(3, /*effectiveDynamic=*/false, hwA)) + << "SK3-static solution must remain selectable on MI300A"; +} From eaf679dfcd495b02399ac8e3e9540042a1334057 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 13 Jul 2026 18:03:35 +0000 Subject: [PATCH 11/20] feat(tensilelite): reset-free single-hop next-neighbor StreamK work stealing Replace the guarded structural-extra steal with an unconditional single-hop next-neighbor steal: each queue q steals one tile from its immediate next neighbor s = (q+1) & mask whenever its home queue drains, dropping the remainder/structural-extra guards. A per-WG sticky-home flag (StreamKStickyEmpty) touches the home counter for valid dispenses plus exactly one empty fetch, then steals only. Because each queue is stolen from by exactly one predecessor, the atomic_inc auto-reset bound becomes predecessor-inclusive (home = tiles_q + W_q + W_(q-1) - 1, steal = tiles_s + W_s + W_q - 1) so every counter self-zeroes each launch. This removes the explicit end-of-kernel reset and the 0x80 completion counter; the host-zeroed-once Synchronizer stays clean for back-to-back launches. Solution validation rejects DebugStreamK overrides that could leave a tile-owning queue with no home workgroup (W_q>=1 when tiles_q>=1) and thus break the wrap. --- .../tensilelite/Tensile/Components/StreamK.py | 358 ++++++++++-------- .../tensilelite/Tensile/KernelWriter.py | 15 +- .../Tensile/SolutionStructs/Solution.py | 17 + .../streamk/sk_dynamic_work_stealing.yaml | 26 +- .../streamk/sk_hybrid_work_stealing.yaml | 26 +- .../Tests/unit/test_streamk_work_stealing.py | 251 ++++++++---- 6 files changed, 443 insertions(+), 250 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index d48c1105ad05..d86dfa45508b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -332,29 +332,28 @@ def _skv(self, writer, name): # Single-hop next-neighbor work stealing (codegen-time, off by default) # # Topology: the dynamic-queue fetch (auto-mode SK4 / SK5-dynamic) uses 8 - # hardcoded queues. queueIdx = StreamKIdx & 0x7; per-queue counters live one - # per cache line at AddressFlags + (queueIdx << 8); the global tile index is - # (perQueueRaw << 3) + queueIdx; remainder tiles = TotalItems & 0x7. + # hardcoded queues in a wraparound next-neighbor order. queueIdx = StreamKIdx + # & 0x7; per-queue counters live one per cache line at AddressFlags + + # (queueIdx << 8); the global tile index is (perQueueRaw << 3) + queueIdx. + # + # Every queue q steals from its immediate next neighbor s = (q+1) & 0x7 once + # its own home queue drains (there is no remainder/structural-extra guard: an + # empty home fetch always attempts one steal from that single next neighbor). + # A per-WG STICKY-HOME flag (StreamKStickyEmpty) makes each WG touch its home + # counter for its valid dispenses plus EXACTLY ONE empty fetch, then only + # steal thereafter. Because the extra atomics against a queue are now a static + # function of the next-neighbor topology (each queue is stolen from by exactly + # one predecessor p = (q-1) & 0x7), every counter self-resets to 0 via a per-queue auto-reset + # bound that folds in the predecessor's workgroup count. No explicit + # end-of-kernel reset is needed: the Synchronizer, host-zeroed once at handle + # creation, is left zeroed for the next (back-to-back) launch by the bounds + # alone. # # AddressFlags buffer layout (per problem), for reference: # [0x000 .. 0x800) 8 queue counters, one per 256B cache line # (only the first word of each line is used). # [0x800 .. ) partials/fixup ready flags (one word per partial tile, # see storeBranches: offset = (partialIdx << 2) + 256*8). - # - # The work-stealing completion counter is placed at offset 0x80. This is - # collision-free: it is NOT a queue-counter offset (those are multiples of - # 256) and it sits below the flags region that starts at 256*8 = 0x800, so - # it reuses already-reserved, host-zeroed padding inside queue 0's cache-line - # slot. Picking 0x80 (rather than the prior art's numXCDs*256 = 0x800, which - # would collide with the first partials flag) needs no host workspace growth: - # ContractionSolution.cpp already reserves the full 256*8 queue region and - # the Synchronizer buffer is zeroed once at handle creation. - _WS_NUM_QUEUES = 8 - _WS_QUEUE_MASK = 0x7 - _WS_QUEUE_LOG2 = 3 # log2(8) - _WS_COMPLETION_COUNTER_OFFSET = 0x80 - # Per-architecture work-stealing / dynamic-queue count. # # The dynamic-queue fetch partitions work across one queue per XCD and maps @@ -393,110 +392,122 @@ def _wsQueueConstants(self, kernel): ) return numQueues, numQueues - 1, log2(numQueues), self._WS_CACHE_LINE_LOG2 - def streamKWorkStealingHomeNoReset(self, writer, mod, kernel, sBound, mkLabel): - """Disable the home queue's atomic auto-reset when a neighbor could steal. + def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, comment): + """Emit sDst = (sTotal >> log2Queues) + [sQueue < (sTotal & mask)]. - When remainder (TotalItems & 0x7) != 0 a neighbor may steal this queue's - structural extra tile, which would throw off the auto-reset count, so the - home counter must run unbounded (kernelEnd resets it instead). Overrides - the precomputed auto-reset bound in sBound with 0xFFFFFFFF in that case; - remainder == 0 keeps the normal bound. Caller must gate on - kernel["StreamKWorkStealing"]. + Reuses the shift/and(mask)/cmp/cselect idiom already used for + tilesInQueue / workgroupsInQueue in graWorkGroup so the per-queue + structural share (tiles or workgroups) can be recomputed for an + arbitrary queue index. ``sTmp`` is a caller-owned scratch SGPR. """ - skKeepBound = mkLabel("SK_FetchHomeQueue") - sRemainder = writer.sgprPool.checkOut(1, "wsRemainder") - mod.add(SAndB32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=self._WS_QUEUE_MASK, comment="Remainder tiles")) - mod.add(SCmpEQU32(src0=sgpr(sRemainder), src1=0, comment="Stealing only helps when remainder != 0")) - mod.add(SCBranchSCC1(labelName=skKeepBound.getLabelName(), comment="No remainder; keep auto-reset bound")) - mod.add(SMovB32(dst=sgpr(sBound), src=hex(0xFFFFFFFF), comment="Disable home auto-reset (neighbor may steal)")) - mod.add(skKeepBound) - writer.sgprPool.checkIn(sRemainder) - - def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, mkLabel): - """Single-hop NEXT-neighbor steal on the 8-queue topology. - - On entry sQueueIdx holds the home queue index and sWorkItemIdx holds the - home global tile index; both must be live. If the home fetch came up - empty (index >= TotalItems) and the next queue (queueIdx+1)&0x7 owns a - structural extra tile that the home queue does not, make exactly ONE - atomic (auto-reset disabled) against that neighbor and recompute the - global index. A lost race leaves sWorkItemIdx >= TotalItems so the - existing downstream valid-index check turns this WG into a no-op. - sQueueIdx is clobbered. Caller must gate on kernel["StreamKWorkStealing"]. + mod.add(SLShiftRightB32(dst=sgpr(sDst), src=sgpr(sTotal), shiftHex=log2Queues, comment=comment)) + mod.add(SAndB32(dst=sgpr(sTmp), src0=sgpr(sTotal), src1=mask, comment="Remainder")) + mod.add(SCmpLtU32(src0=sgpr(sQueue), src1=sgpr(sTmp), comment="Queue gets a structural extra?")) + mod.add(SCSelectB32(dst=sgpr(sTmp), src0=1, src1=0)) + mod.add(SAddU32(dst=sgpr(sDst), src0=sgpr(sDst), src1=sgpr(sTmp))) + + def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, sGrid): + """Fold the predecessor's workgroup count into the home auto-reset bound. + + The non-stealing dynamic path relies on the atomic_inc auto-reset bound + ``tiles_q + W_q - 1`` so that after exactly ``tiles_q + W_q`` fetches (all + tiles dispensed + one empty per WG) the counter wraps back to 0. Under + single-hop next-neighbor stealing every queue q is also stolen from by EXACTLY ONE + predecessor p = (q-1) & (numQueues-1): once each of p's W_p workgroups + goes sticky-empty it makes one atomic against q per persistent iteration, + contributing W_p additional increments to q's counter. The self-resetting + bound therefore becomes ``tiles_q + W_q + W_p - 1``. This adds the W_p + term (W_p = (skGrid >> log2) + [p < (skGrid & mask)]) to the already + computed ``tiles_q + W_q - 1`` in ``sBound``. Caller must gate on + kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR + name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). ``sQueueIdx`` + is preserved. + + Precondition (see Solution validation): the fold is exact only when + W_q >= 1 whenever tiles_q >= 1, i.e. the grid assigns at least one + workgroup per queue (skGrid >= numQueues); the Solution layer rejects + debug overrides that could violate this. + """ + _, mask, log2Queues, _ = self._wsQueueConstants(kernel) + sPred = writer.sgprPool.checkOut(1, "wsPredQueue") + sWp = writer.sgprPool.checkOut(1, "wsPredWorkgroups") + sTmp = writer.sgprPool.checkOut(1, "wsPredTmp") + # p = (q - 1) & mask (wraps 0 -> numQueues-1 for unsigned subtract) + mod.add(SSubU32(dst=sgpr(sPred), src0=sgpr(sQueueIdx), src1=1, comment="Predecessor queue (q-1)")) + mod.add(SAndB32(dst=sgpr(sPred), src0=sgpr(sPred), src1=mask, comment="Wrap predecessor index")) + # W_p = (skGrid >> log2) + [p < (skGrid & mask)] + self._wsStructuralCount(mod, mask, log2Queues, sWp, sGrid, sPred, sTmp, + comment="Predecessor workgroups W_(q-1)") + mod.add(SAddU32(dst=sgpr(sBound), src0=sgpr(sBound), src1=sgpr(sWp), + comment="Home auto-reset bound += predecessor workgroups (next-neighbor steal)")) + writer.sgprPool.checkIn(sTmp) + writer.sgprPool.checkIn(sWp) + writer.sgprPool.checkIn(sPred) + + def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, sGrid, mkLabel): + """Single-hop next-neighbor steal on the per-XCD queue topology. + + On entry sQueueIdx holds the home queue index q and sWorkItemIdx holds + the home fetch result; both must be live. If the home fetch was valid + (index < TotalItems) this is a no-op. Otherwise the WG ALWAYS steals one + tile from its immediate next neighbor s = (q+1) & (numQueues-1): a single + s_atomic_inc against the neighbor's counter, then the global tile index + is recomputed from s. There is NO remainder / structural-extra guard -- + an empty home always attempts exactly one steal. A lost race leaves + sWorkItemIdx >= TotalItems so the downstream valid-index check turns this + WG into a no-op. sQueueIdx is clobbered (advanced to s). Caller must gate + on kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR + name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). + + The steal atomic runs with a STATIC auto-reset bound + ``tiles_s + W_s + W_q - 1`` (rather than a disabled 0xFFFFFFFF bound): + s is stolen from by exactly its predecessor q, so this is the same + predecessor-inclusive self-reset used on the home path, evaluated for the + stolen queue. This lets the neighbor counter wrap back to 0 on its own, + removing the need for an explicit end-of-kernel reset. """ + _, mask, log2Queues, cacheLineLog2 = self._wsQueueConstants(kernel) skFetchDone = mkLabel("SK_FetchDone") mod.add(SCmpLtU32(src0=sgpr(sWorkItemIdx), src1=sgpr("TotalItems"), comment="Home fetch valid?")) mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Valid work fetched; no steal")) - sRemainder = writer.sgprPool.checkOut(1, "wsRemainder") - mod.add(SAndB32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=self._WS_QUEUE_MASK, comment="Remainder tiles")) - mod.add(SCmpEQU32(src0=sgpr(sRemainder), src1=0, comment="Stealing only helps when remainder != 0")) - mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="No remainder; skip steal")) - mod.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Home already owns a structural extra?")) - mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Home has extra; no fair steal")) + # Build the steal auto-reset bound tiles_s + W_s + W_q - 1 into + # sWorkItemIdx (dead here). W_q is the stealer's own workgroup count and + # must be computed while sQueueIdx still holds q, before advancing to s. + sTmp = writer.sgprPool.checkOut(1, "wsStealTmp") + sWq = writer.sgprPool.checkOut(1, "wsStealerWorkgroups") + self._wsStructuralCount(mod, mask, log2Queues, sWq, sGrid, sQueueIdx, sTmp, + comment="Stealer workgroups W_q") - # Walk to the immediate next queue (wrap within the 8-queue ring). + # Walk to the immediate next queue (wrap within the per-XCD queues, single-hop next-neighbor). mod.add(SAddU32(dst=sgpr(sQueueIdx), src0=sgpr(sQueueIdx), src1=1, comment="Next queue")) - mod.add(SAndB32(dst=sgpr(sQueueIdx), src0=sgpr(sQueueIdx), src1=self._WS_QUEUE_MASK, comment="Wrap queue index")) - mod.add(SCmpGeU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Neighbor has no structural extra?")) - mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Neighbor has no extra; skip steal")) + mod.add(SAndB32(dst=sgpr(sQueueIdx), src0=sgpr(sQueueIdx), src1=mask, comment="Wrap queue index")) + + # tiles_s into sWorkItemIdx, then += W_s and += W_q, then -1. + self._wsStructuralCount(mod, mask, log2Queues, sWorkItemIdx, "TotalItems", sQueueIdx, sTmp, + comment="Stolen-queue tiles tiles_s") + sWs = writer.sgprPool.checkOut(1, "wsStolenWorkgroups") + self._wsStructuralCount(mod, mask, log2Queues, sWs, sGrid, sQueueIdx, sTmp, + comment="Stolen-queue workgroups W_s") + mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sWs), comment="tiles_s + W_s")) + writer.sgprPool.checkIn(sWs) + mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sWq), comment="+ W_q (stealer)")) + mod.add(SSubU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=1, comment="Steal auto-reset bound")) + writer.sgprPool.checkIn(sWq) + writer.sgprPool.checkIn(sTmp) - # One atomic on the neighbor's counter, auto-reset disabled. + # One atomic on the neighbor's counter with the static self-reset bound. sAddress = writer.sgprPool.checkOutAligned(2, 2, "wsStealAddress") - mod.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), shiftHex=log2(256), comment="Stride queues to cache lines (stolen queue)")) + mod.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), shiftHex=cacheLineLog2, comment="Stride queues to cache lines (stolen queue)")) mod.add(SAddU32(dst=sgpr(sAddress+0), src0=sgpr(sAddress+0), src1=sgpr("AddressFlags+0"))) mod.add(SAddCU32(dst=sgpr(sAddress+1), src0=0, src1=sgpr("AddressFlags+1"))) - mod.add(SMovB32(dst=sgpr(sWorkItemIdx), src=hex(0xFFFFFFFF), comment="Disable atomic auto-reset")) mod.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), comment="Fetch stolen work item index")) mod.add(SWaitCnt(kmcnt=0, comment="Wait for scalar memory op")) writer.sgprPool.checkIn(sAddress) # Recompute global tile index from the neighbor's queue. - mod.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=self._WS_QUEUE_LOG2)) + mod.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=log2Queues)) mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) mod.add(skFetchDone) - writer.sgprPool.checkIn(sRemainder) - - def streamKWorkStealingKernelEndReset(self, writer, kernel, sGridName, mkLabel): - """Last WG to finish zeroes the 8 queue counters + the completion counter. - - Required only when work stealing disabled the home auto-reset: the - Synchronizer buffer is zeroed by the host just once (at handle creation), - so the kernel must restore the counters to 0 itself for the next launch. - Wave 0 atomically counts completed WGs against the completion counter at - offset 0x80 (auto-reset disabled); the WG that observes the full grid - count (sGridName) performs the resets. Caller must gate on - kernel["StreamKWorkStealing"]. sGridName is "skGrid" (SK4) or "SKGrid" - (SK5-dynamic). - """ - module = Module("StreamK work-stealing kernelEnd reset") - completionOffset = self._WS_COMPLETION_COUNTER_OFFSET - skExit = mkLabel("SK_WSResetExit") - - module.add(SBarrier(comment="Wait for all waves before completion count")) - sWave = writer.sgprPool.checkOut(1, "wsWave") - module.add(VReadfirstlaneB32(dst=sgpr(sWave), src=vgpr("Serial"), comment="Wave 0 handles completion")) - module.add(SCmpEQU32(src0=sgpr(sWave), src1=0, comment="Check for wave 0")) - module.add(SCBranchSCC0(labelName=skExit.getLabelName(), comment="Only wave 0 counts completed WGs")) - writer.sgprPool.checkIn(sWave) - - sCompleted = writer.sgprPool.checkOut(1, "wsCompletedWGs") - module.add(SMovB32(dst=sgpr(sCompleted), src=hex(0xFFFFFFFF), comment="Disable completion counter auto-reset")) - module.add(SAtomicInc(dst=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=completionOffset, smem=SMEMModifiers(glc=True), comment="Count completed workgroups")) - module.add(SWaitCnt(kmcnt=0, comment="Wait for scalar memory op")) - module.add(SAddU32(dst=sgpr(sCompleted), src0=sgpr(sCompleted), src1=1, comment="Completed WGs including this WG")) - module.add(SCmpEQU32(src0=sgpr(sCompleted), src1=sgpr(sGridName), comment="All workgroups completed?")) - module.add(SCBranchSCC0(labelName=skExit.getLabelName(), comment="Not the last WG; skip reset")) - - module.add(SMovB32(dst=sgpr(sCompleted), src=0, comment="Zero value for synchronizer resets")) - # Per-queue counters are one-per-cache-line (256B apart) so they cannot - # be coalesced into a wide store; keep separate b32 stores. - for queueIdx in range(self._WS_NUM_QUEUES): - module.add(SStoreB32(src=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=queueIdx * 256, smem=SMEMModifiers(glc=True), comment="Reset queue counter")) - module.add(SStoreB32(src=sgpr(sCompleted), base=sgpr("AddressFlags", 2), soffset=completionOffset, smem=SMEMModifiers(glc=True), comment="Reset completion counter")) - module.add(SWaitCnt(kmcnt=0, comment="Wait for synchronizer reset")) - module.add(skExit) - writer.sgprPool.checkIn(sCompleted) - return module @abc.abstractmethod def preLoop(self, writer, kernel): @@ -3028,6 +3039,9 @@ def preLoop(self, writer, kernel): module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7", comment="workaround")) module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="Save original StreamK index")) + # Work stealing: this WG has not yet seen its home queue empty. + if kernel["StreamKWorkStealing"]: + module.add(SMovB32(dst=sgpr("StreamKStickyEmpty"), src=0, comment="WS: home not yet empty")) # Two-tile SK (DP first) # Do DP tiles before SK skInitDone = Label("SK_InitDone", "") @@ -3064,23 +3078,27 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): module.add(SCBranchSCC0(labelName=skSkipWorkItem.getLabelName(), comment="Skip work item")) writer.sgprPool.checkIn(sWave) + # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for the + # StreamKIdx/queue divisions, log2(256) for the cache-line stride). + _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(kernel) + # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") - module.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), shiftHex=log2(8))) - module.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), shiftHex=log2(8))) + module.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), shiftHex=wsLog2Queues)) + module.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), shiftHex=wsLog2Queues)) module.add(SSubU32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKIdx"), src1=sgpr(sQueueIdx), comment="Default queue index")) # Queue address sAddress = writer.sgprPool.checkOutAligned(2, 2, "Address") - module.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), shiftHex=log2(256), comment="Stride queues to different cache lines")) + module.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), shiftHex=wsCacheLineLog2, comment="Stride queues to different cache lines")) module.add(SAddU32(dst=sgpr(sAddress+0), src0=sgpr(sAddress+0), src1=sgpr("AddressFlags+0"))) module.add(SAddCU32(dst=sgpr(sAddress+1), src0=0, src1=sgpr("AddressFlags+1"))) # Tiles in queue sTilesInQueue = writer.sgprPool.checkOut(1, "tilesInQueue") - module.add(SLShiftRightB32(dst=sgpr(sTilesInQueue), src=sgpr("TotalItems"), shiftHex=log2(8))) + module.add(SLShiftRightB32(dst=sgpr(sTilesInQueue), src=sgpr("TotalItems"), shiftHex=wsLog2Queues)) sRemainder = writer.sgprPool.checkOut(1, "remainder tiles") - module.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sTilesInQueue), shiftHex=log2(8))) + module.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sTilesInQueue), shiftHex=wsLog2Queues)) module.add(SSubU32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=sgpr(sRemainder), comment="Remainder tiles")) module.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Check if queue gets an extra tile")) module.add(SCSelectB32(dst=sgpr(sRemainder), src0=1, src1=0)) @@ -3089,9 +3107,9 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): # Workgroups in queue sWorkgroupsInQueue = writer.sgprPool.checkOut(1, "workgroupsInQueue") - module.add(SLShiftRightB32(dst=sgpr(sWorkgroupsInQueue), src=sgpr("skGrid"), shiftHex=log2(8))) + module.add(SLShiftRightB32(dst=sgpr(sWorkgroupsInQueue), src=sgpr("skGrid"), shiftHex=wsLog2Queues)) sRemainder = writer.sgprPool.checkOut(1, "remainder workgroups") - module.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sWorkgroupsInQueue), shiftHex=log2(8))) + module.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sWorkgroupsInQueue), shiftHex=wsLog2Queues)) module.add(SSubU32(dst=sgpr(sRemainder), src0=sgpr("skGrid"), src1=sgpr(sRemainder), comment="Remainder workgroups")) module.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), comment="Check if queue gets an extra tile")) module.add(SCSelectB32(dst=sgpr(sRemainder), src0=1, src1=0)) @@ -3105,9 +3123,19 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): writer.sgprPool.checkIn(sTilesInQueue) writer.sgprPool.checkIn(sWorkgroupsInQueue) - # Work stealing: a stolen home queue must not auto-reset its counter. + # Work stealing: fold the predecessor's workgroup count into the home + # auto-reset bound so the counter still self-resets under next-neighbor stealing. if kernel["StreamKWorkStealing"]: - self.streamKWorkStealingHomeNoReset(writer, module, kernel, sWorkItemIdx, lambda base: Label(base, "")) + self.streamKWorkStealingHomeBound(writer, module, kernel, sWorkItemIdx, sQueueIdx, "skGrid") + + # Work stealing: once this WG has seen its home queue empty (sticky), it + # never touches the home counter again -- skip the home fetch and force + # the steal path with an invalid sentinel index (>= TotalItems). + if kernel["StreamKWorkStealing"]: + skStealOnly = Label(writer.labels.getNameInc("SK_StealOnly"), "") + skHomeFetched = Label(writer.labels.getNameInc("SK_HomeFetched"), "") + module.add(SCmpEQU32(src0=sgpr("StreamKStickyEmpty"), src1=0, comment="Home not yet empty?")) + module.add(SCBranchSCC0(labelName=skStealOnly.getLabelName(), comment="Sticky: skip home fetch, steal only")) # Fetch next work item module.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), comment="Fetch next work item index")) @@ -3116,13 +3144,20 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): writer.sgprPool.checkIn(sAddress) # Convert to global work item index - module.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=log2(8))) + module.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), shiftHex=wsLog2Queues)) module.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) - # Work stealing: if the home queue was empty, attempt one steal from the - # next neighbor before sharing the index with the rest of the waves. + # Work stealing: latch the sticky-empty flag on the first empty home + # fetch, then fall through to the steal (or, when already sticky, jump + # straight to the steal with the sentinel index). if kernel["StreamKWorkStealing"]: - self.streamKWorkStealingSteal(writer, module, kernel, sQueueIdx, sWorkItemIdx, lambda base: Label(base, "")) + module.add(SCmpGeU32(src0=sgpr(sWorkItemIdx), src1=sgpr("TotalItems"), comment="Home fetch empty?")) + module.add(SCSelectB32(dst=sgpr("StreamKStickyEmpty"), src0=1, src1=0, comment="Latch sticky-empty on empty home")) + module.add(SBranch(labelName=skHomeFetched.getLabelName(), comment="Home fetched; try one steal")) + module.add(skStealOnly) + module.add(SMovB32(dst=sgpr(sWorkItemIdx), src=sgpr("TotalItems"), comment="Sentinel index (>= TotalItems) forces steal")) + module.add(skHomeFetched) + self.streamKWorkStealingSteal(writer, module, kernel, sQueueIdx, sWorkItemIdx, "skGrid", lambda base: Label(writer.labels.getNameInc(base), "")) writer.sgprPool.checkIn(sQueueIdx) # Share work item index with all waves @@ -3401,16 +3436,13 @@ def routeToGeneralBatchedOrStridedBatched(self, stridedBatchedGemmLoad, generalB def kernelEnd(self, writer, kernel): module = Module("StreamK Dynamic kernelEnd") - # We don't need to track completed kernels if we know total tiles and grid size - # Reset is baked into the atomic_inc at the top of the loop - # TODO will need to reset the rest of the synchronizer if tiles were split - # Remaining reset can be done if workitem = grid + total - 1 - - # Work stealing disables the home auto-reset, so the last WG must restore - # the queue + completion counters for the next launch (the Synchronizer - # is zeroed by the host only once). - if kernel["StreamKWorkStealing"]: - module.add(self.streamKWorkStealingKernelEndReset(writer, kernel, "skGrid", lambda base: Label(base, ""))) + # We don't need to track completed kernels if we know total tiles and grid size. + # The reset is baked into the atomic_inc auto-reset bound at the top of the + # loop: under single-hop next-neighbor work stealing the per-queue bound folds in the + # predecessor's workgroup count (see streamKWorkStealingHomeBound / + # streamKWorkStealingSteal), so every counter wraps back to 0 on its own and + # leaves the (host-zeroed-once) Synchronizer clean for the next launch. No + # explicit end-of-kernel reset is required. return module @@ -3513,6 +3545,10 @@ def preLoop(self, writer, kernel): # so save directly to the StreamKIdx SGPR (no VGPR-cache path). module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="SK5: save original StreamK index")) + # Work stealing: this WG has not yet seen its home queue empty. + if kernel["StreamKWorkStealing"]: + module.add(SMovB32(dst=sgpr("StreamKStickyEmpty"), src=0, + comment="WS: home not yet empty")) # ----- Extract the mode bit once for the whole kernel ----- module.add(self._emitModeExtraction(writer, kernel)) @@ -3702,19 +3738,23 @@ def emitDynamicGRA(mod): comment="Skip work item")) writer.sgprPool.checkIn(sWave) + # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for + # the StreamKIdx/queue divisions, log2(256) for cache-line stride). + _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(kernel) + # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") mod.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) mod.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) mod.add(SSubU32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKIdx"), src1=sgpr(sQueueIdx), comment="Default queue index")) # Queue address sAddress = writer.sgprPool.checkOutAligned(2, 2, "Address") mod.add(SLShiftLeftB32(dst=sgpr(sAddress), src=sgpr(sQueueIdx), - shiftHex=log2(256), + shiftHex=wsCacheLineLog2, comment="Stride queues to different cache lines")) mod.add(SAddU32(dst=sgpr(sAddress+0), src0=sgpr(sAddress+0), src1=sgpr("AddressFlags+0"))) @@ -3723,10 +3763,10 @@ def emitDynamicGRA(mod): # Tiles in queue sTilesInQueue = writer.sgprPool.checkOut(1, "tilesInQueue") mod.add(SLShiftRightB32(dst=sgpr(sTilesInQueue), src=sgpr("TotalItems"), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) sRemainder = writer.sgprPool.checkOut(1, "remainder tiles") mod.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sTilesInQueue), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) mod.add(SSubU32(dst=sgpr(sRemainder), src0=sgpr("TotalItems"), src1=sgpr(sRemainder), comment="Remainder tiles")) mod.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), @@ -3740,10 +3780,10 @@ def emitDynamicGRA(mod): sWorkgroupsInQueue = writer.sgprPool.checkOut(1, "workgroupsInQueue") # SK5: SKGrid is the SK4-dedicated grid SGPR (uppercase). mod.add(SLShiftRightB32(dst=sgpr(sWorkgroupsInQueue), src=sgpr("SKGrid"), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) sRemainder = writer.sgprPool.checkOut(1, "remainder workgroups") mod.add(SLShiftLeftB32(dst=sgpr(sRemainder), src=sgpr(sWorkgroupsInQueue), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) mod.add(SSubU32(dst=sgpr(sRemainder), src0=sgpr("SKGrid"), src1=sgpr(sRemainder), comment="Remainder workgroups")) mod.add(SCmpLtU32(src0=sgpr(sQueueIdx), src1=sgpr(sRemainder), @@ -3761,10 +3801,23 @@ def emitDynamicGRA(mod): writer.sgprPool.checkIn(sTilesInQueue) writer.sgprPool.checkIn(sWorkgroupsInQueue) - # Work stealing: a stolen home queue must not auto-reset its counter. + # Work stealing: fold the predecessor's workgroup count into the + # home auto-reset bound so the counter still self-resets under + # next-neighbor stealing. + if kernel["StreamKWorkStealing"]: + self.streamKWorkStealingHomeBound(writer, mod, kernel, sWorkItemIdx, + sQueueIdx, "SKGrid") + + # Work stealing: once this WG has seen its home queue empty (sticky), + # never touch the home counter again -- skip the home fetch and force + # the steal path with an invalid sentinel index (>= TotalItems). if kernel["StreamKWorkStealing"]: - self.streamKWorkStealingHomeNoReset(writer, mod, kernel, sWorkItemIdx, - lambda base: Label(writer.labels.getNameInc(base), "")) + skStealOnly = Label(writer.labels.getNameInc("SK_StealOnly"), "") + skHomeFetched = Label(writer.labels.getNameInc("SK_HomeFetched"), "") + mod.add(SCmpEQU32(src0=sgpr("StreamKStickyEmpty"), src1=0, + comment="Home not yet empty?")) + mod.add(SCBranchSCC0(labelName=skStealOnly.getLabelName(), + comment="Sticky: skip home fetch, steal only")) mod.add(SAtomicInc(dst=sgpr(sWorkItemIdx), base=sgpr(sAddress, 2), soffset=0, smem=SMEMModifiers(glc=True), @@ -3774,14 +3827,26 @@ def emitDynamicGRA(mod): # Convert to global work item index mod.add(SLShiftLeftB32(dst=sgpr(sWorkItemIdx), src=sgpr(sWorkItemIdx), - shiftHex=log2(8))) + shiftHex=wsLog2Queues)) mod.add(SAddU32(dst=sgpr(sWorkItemIdx), src0=sgpr(sWorkItemIdx), src1=sgpr(sQueueIdx))) - # Work stealing: if the home queue was empty, attempt one steal from - # the next neighbor before sharing the index with the rest of the waves. + # Work stealing: latch the sticky-empty flag on the first empty home + # fetch, then fall through to one steal (or, when already sticky, + # jump straight to the steal with the sentinel index). if kernel["StreamKWorkStealing"]: + mod.add(SCmpGeU32(src0=sgpr(sWorkItemIdx), src1=sgpr("TotalItems"), + comment="Home fetch empty?")) + mod.add(SCSelectB32(dst=sgpr("StreamKStickyEmpty"), src0=1, src1=0, + comment="Latch sticky-empty on empty home")) + mod.add(SBranch(labelName=skHomeFetched.getLabelName(), + comment="Home fetched; try one steal")) + mod.add(skStealOnly) + mod.add(SMovB32(dst=sgpr(sWorkItemIdx), src=sgpr("TotalItems"), + comment="Sentinel index (>= TotalItems) forces steal")) + mod.add(skHomeFetched) self.streamKWorkStealingSteal(writer, mod, kernel, sQueueIdx, sWorkItemIdx, + "SKGrid", lambda base: Label(writer.labels.getNameInc(base), "")) writer.sgprPool.checkIn(sQueueIdx) @@ -4224,19 +4289,10 @@ def routeToGeneralBatchedOrStridedBatched(self, stridedBatchedGemmLoad, generalB def kernelEnd(self, writer, kernel): module = Module("StreamK Hybrid kernelEnd") - # Work stealing only exists on the dynamic (SK4) sub-path; the static - # (SK3) sub-path has no per-XCD queue counters to reset. Gate the reset - # on the runtime mode bit so it is harmless when SK5 runs in static mode. - if kernel["StreamKWorkStealing"]: - def emitDynamicReset(mod): - mod.add(self.streamKWorkStealingKernelEndReset( - writer, kernel, "SKGrid", - lambda base: Label(writer.labels.getNameInc(base), ""))) - - def emitStaticReset(mod): - pass - - self._emitSk3Sk4Branch(writer, module, "WSReset", emitDynamicReset, emitStaticReset) + # Single-hop next-neighbor work stealing self-resets every per-queue counter via the + # predecessor-inclusive atomic_inc auto-reset bound at the top of the loop + # (see streamKWorkStealingHomeBound / streamKWorkStealingSteal), so no + # explicit end-of-kernel reset is required on either SK5 sub-path. return module diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 0388537ffa11..ea9def08a03d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -5233,9 +5233,9 @@ def kernelBodySubtile(self, kernel, tensorParametersA, tensorParametersB): module.add(kernelEndLabel) if kernel["ProblemType"]["OutputAmaxD"]: module.add(self.insertAmaxD(kernel)) - # Mirror functionEnd's StreamK kernel-end reset so the deferred-blocks - # epilogue also restores the WS work-queue/completion counters between - # launches (the reset is internally gated on StreamKWorkStealing). + # Mirror functionEnd's StreamK kernelEnd on the deferred-blocks epilogue. + # Single-hop next-neighbor work stealing self-resets its per-queue counters via the + # atomic_inc auto-reset bounds, so kernelEnd emits no explicit reset. skComponent = Component.StreamK.find(self) module.add(skComponent.kernelEnd(self, kernel)) module.add(SEndpgm(comment="Kernel End")) @@ -9402,6 +9402,11 @@ def vgprAllocationImplSubtile(): "StreamKLocalStart", "StreamKLocalEnd", ] + # Work stealing: per-WG sticky-empty flag. Persists across the persistent + # loop back-edge (added to nonPostLoopSgpr below) so each WG only touches + # its home counter for its valid dispenses + exactly one empty fetch. + if kernel["StreamKWorkStealing"]: + requiredUnalignedSgprVar.append("StreamKStickyEmpty") if kernel["StreamKAtomic"] == 0: requiredAligned4SgprVar.append("SrdWS") elif kernel["StreamK"] == 5: @@ -9423,6 +9428,10 @@ def vgprAllocationImplSubtile(): "StreamKLocalEnd", "StreamKHybridMode", ] + # Work stealing: per-WG sticky-empty flag (see SK4 note above). Only the + # SK4 sub-path uses it, but it must persist for the whole persistent loop. + if kernel["StreamKWorkStealing"]: + requiredUnalignedSgprVar.append("StreamKStickyEmpty") if len(kernel["SpaceFillingAlgo"]): requiredUnalignedSgprVar.append("StreamKTileID") if kernel["StreamKAtomic"] == 0: diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index a62709e6fbc4..e56413e714ae 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1684,6 +1684,23 @@ def assignDerivedParameters( if state["StreamKAtomic"]: reject(state, printRejectionReason, "StreamKWorkStealing is not supported with StreamKAtomic") + # Auto-reset precondition (proof condition #1): the per-queue atomic_inc + # auto-reset bound is now predecessor-inclusive (home bound = + # tiles_q + W_q + W_(q-1) - 1, steal bound = tiles_s + W_s + W_q - 1). + # It self-resets each launch ONLY while every queue that owns tiles also + # owns at least one workgroup, i.e. W_q >= 1 whenever tiles_q >= 1 + # (equivalently skGrid >= numQueues, guaranteed by the CU-count-derived + # grid at runtime). DebugStreamK overrides the standard partials+fixup + # dispatch (1=no fixup, 2=no partials, 3=both) and can leave a queue + # owning tiles that are never dispensed by a home workgroup -- two + # consecutive zero-home queues owning tiles -- which breaks the wrap and + # would leave the Synchronizer non-zero for the next launch. Reject the + # combination so the invariant cannot be violated by a debug override. + if state["DebugStreamK"]: + reject(state, printRejectionReason, + "StreamKWorkStealing requires DebugStreamK=0 (the per-queue " + "auto-reset relies on W_q>=1 whenever tiles_q>=1); got %d" + % state["DebugStreamK"]) if not state["Valid"]: print2("in assignDerivedParameters, state['Valid'] = False") return diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml index a8e98c149f70..8beba20b7e05 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml @@ -14,8 +14,16 @@ GlobalParameters: DataInitTypeB: 13 DataInitTypeC: 12 MaxWorkspaceSize: 134217728 - NumWarmups: 0 - EnqueuesPerSync: 1 + # Repeated back-to-back enqueues per sync reuse the SAME (host-zeroed-once) + # Synchronizer workspace without re-zeroing between launches. This is exactly + # what the per-queue static auto-reset must guarantee: every counter wraps + # back to 0 on its own so the next launch starts clean (there is no explicit + # end-of-kernel reset anymore). + NumWarmups: 2 + EnqueuesPerSync: 4 + # SleepPercent injects a variable per-WG delay so tiles drain unevenly across + # queues, forcing the single-hop next-neighbor steal + sticky-home path to fire under + # contention rather than in lockstep. SleepPercent: 50 # Shared SK4-dynamic fork entries (single-value) plus common benchmark params. @@ -45,14 +53,20 @@ GlobalParameters: JoinParameters: BenchmarkJoinParameters: BenchmarkFinalParameters: - # Multiple sizes / repeated launches exercise the kernelEnd counter-reset - # path across launches that reuse the same (host-zeroed-once) Synchronizer. - # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0) - # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (remainder != 0, steal fires) + # Multiple sizes + repeated back-to-back launches exercise the per-queue + # static auto-reset across launches that reuse the same (host-zeroed-once) + # Synchronizer -- there is no longer an explicit kernelEnd reset. + # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0); + # with the next-neighbor steal this still exercises the neighbor steal + # + auto-reset (the old remainder==0 skip is gone). + # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (steal fires) + # 896x896 -> 49 (MT128) / 196 (MT64) tiles : unaligned, larger grid -> + # contention size (uneven drain crossed with SleepPercent) # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) - ProblemSizes: - Exact: [512, 512, 1, 512] - Exact: [640, 640, 1, 512] + - Exact: [896, 896, 1, 512] - Exact: [384, 384, 3, 256] BenchmarkProblems: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml index cde7c9c5d335..50d89739bcd9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml @@ -14,8 +14,16 @@ GlobalParameters: DataInitTypeB: 13 DataInitTypeC: 12 MaxWorkspaceSize: 134217728 - NumWarmups: 0 - EnqueuesPerSync: 1 + # Repeated back-to-back enqueues per sync reuse the SAME (host-zeroed-once) + # Synchronizer workspace without re-zeroing between launches. This is exactly + # what the per-queue static auto-reset must guarantee: every counter wraps + # back to 0 on its own so the next launch starts clean (there is no explicit + # end-of-kernel reset anymore). + NumWarmups: 2 + EnqueuesPerSync: 4 + # SleepPercent injects a variable per-WG delay so tiles drain unevenly across + # queues, forcing the single-hop next-neighbor steal + sticky-home path to fire under + # contention rather than in lockstep. SleepPercent: 50 # Exercise the two deterministic SK5 hybrid-mode paths in one run: # 0 -> static work assignment (SK3 sub-path) @@ -51,14 +59,20 @@ GlobalParameters: JoinParameters: BenchmarkJoinParameters: BenchmarkFinalParameters: - # Multiple sizes / repeated launches exercise the kernelEnd counter-reset - # path across launches that reuse the same (host-zeroed-once) Synchronizer. - # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0) - # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (remainder != 0, steal fires) + # Multiple sizes + repeated back-to-back launches exercise the per-queue + # static auto-reset across launches that reuse the same (host-zeroed-once) + # Synchronizer -- there is no longer an explicit kernelEnd reset. + # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0); + # with the next-neighbor steal this still exercises the neighbor steal + # + auto-reset (the old remainder==0 skip is gone). + # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (steal fires) + # 896x896 -> 49 (MT128) / 196 (MT64) tiles : unaligned, larger grid -> + # contention size (uneven drain crossed with SleepPercent) # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) - ProblemSizes: - Exact: [512, 512, 1, 512] - Exact: [640, 640, 1, 512] + - Exact: [896, 896, 1, 512] - Exact: [384, 384, 3, 256] BenchmarkProblems: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index 6535327e6619..01f215a96798 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -1,6 +1,6 @@ # Copyright © Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""Unit tests for the single-hop StreamK work-stealing codegen. +"""Unit tests for the single-hop next-neighbor StreamK work-stealing codegen. These tests assert that the new work-stealing assembly is emitted by the helper methods on the ``StreamK`` base class, and -- crucially -- that those @@ -9,6 +9,16 @@ and inspect emitted modules rather than matching source text; the toggle gating and the Solution-level validation are verified by executing the *real* source (via the AST) so the assertions track the actual code, not a copy of it. + +New emission contract (single-hop next-neighbor + sticky-home + static auto-reset): + * The steal always fires on an empty home fetch -- there is NO remainder / + structural-extra guard (no ``s_cmp_ge_u32`` neighbor guard, no + ``remainder == 0`` skip). + * The steal & home atomic bounds are the predecessor-inclusive self-reset + value, NOT the old 0xFFFFFFFF "disable auto-reset" sentinel. + * A per-WG sticky-empty SGPR gates the home fetch. + * There is NO ``streamKWorkStealingKernelEndReset`` / completion counter / + reset barrier anymore. """ import ast @@ -26,12 +36,11 @@ SAndB32, SAtomicInc, SBarrier, - SCBranchSCC0, SCBranchSCC1, SCmpGeU32, SCmpLtU32, SMovB32, - SStoreB32, + SSubU32, ) from Tensile.Common.ValidParameters import validParameters @@ -48,8 +57,8 @@ # --------------------------------------------------------------------------- # Fakes: just enough of a "writer" for the standalone helper methods. # -# The three helpers only touch ``writer.sgprPool`` (checkOut / checkOutAligned -# / checkIn) and emit rocisa instructions via free functions (sgpr/vgpr), so a +# The helpers only touch ``writer.sgprPool`` (checkOut / checkOutAligned / +# checkIn) and emit rocisa instructions via free functions (sgpr/vgpr), so a # tiny pool that hands out monotonically increasing register indices is all # that is required -- no KernelWriter, no GPU. # --------------------------------------------------------------------------- @@ -82,6 +91,11 @@ def _mk_label(base: str) -> Label: return Label(base, "") +# gfx942/gfx950 both map to 8 queues in the per-arch lookup; the helpers key +# their fast-mask constants off ``kernel["ISA"]``. +_WS_KERNEL = {"ISA": (9, 4, 0)} + + def _stream_k_instance(streamk: int) -> StreamK: """A concrete StreamK variant (helpers live on the base class).""" return streamKVariantClass(streamk)() @@ -154,6 +168,10 @@ def _all_ws_calls(func) -> set: return calls +def _source_of(func) -> str: + return inspect.getsource(func) + + def _extract_ws_validation(): """Compile the *real* ``if state["StreamKWorkStealing"]:`` block out of ``Solution.assignDerivedParameters`` into a standalone callable so the @@ -205,58 +223,84 @@ def test_work_stealing_param_is_zero_one(self): # =========================================================================== -# 2. The three new StreamK helper methods exist and are callable. +# 2. The work-stealing helper methods: the new steal/bound helpers exist and +# the removed single-hop-reset helpers are gone. # =========================================================================== class TestHelperMethodsExist: @pytest.mark.parametrize( "name", [ - "streamKWorkStealingHomeNoReset", + "streamKWorkStealingHomeBound", "streamKWorkStealingSteal", - "streamKWorkStealingKernelEndReset", ], ) def test_method_is_defined_on_base(self, name): assert callable(getattr(StreamK, name)) + @pytest.mark.parametrize( + "name", + [ + "streamKWorkStealingHomeNoReset", + "streamKWorkStealingKernelEndReset", + ], + ) + def test_removed_methods_are_gone(self, name): + assert not hasattr(StreamK, name), ( + "explicit-reset / no-reset helpers must be removed" + ) + + def test_completion_counter_constant_is_gone(self): + assert not hasattr(StreamK, "_WS_COMPLETION_COUNTER_OFFSET"), ( + "the 0x80 completion-counter offset must be removed" + ) + # =========================================================================== -# 3a. Presence: the helpers actually emit the work-stealing assembly. +# 3a. Home auto-reset bound: the predecessor's workgroup count is folded into +# the atomic_inc bound (NOT disabled with 0xFFFFFFFF). # =========================================================================== -class TestHomeNoResetEmission: - """Disabling the home auto-reset must mask TotalItems with 0x7 and, when a - remainder exists, move 0xFFFFFFFF into the auto-reset bound register.""" - +class TestHomeBoundEmission: def _emit(self): sk = _stream_k_instance(4) writer = _FakeWriter() - module = Module("home-no-reset") + module = Module("home-bound") sBound = writer.sgprPool.checkOut(1, "bound") - sk.streamKWorkStealingHomeNoReset(writer, module, {}, sBound, _mk_label) + sQueueIdx = writer.sgprPool.checkOut(1, "queueIdx") + sk.streamKWorkStealingHomeBound( + writer, module, _WS_KERNEL, sBound, sQueueIdx, "skGrid" + ) return _flat(module) - def test_masks_total_items_with_queue_mask(self): + def test_computes_predecessor_index(self): items = self._emit() - masks = [ - i for i in items - if isinstance(i, SAndB32) and _imm_in(i, 0x7) - ] - assert masks, "expected an s_and_b32 with the 0x7 queue mask" + # p = (q - 1) & 0x7 + assert any(isinstance(i, SSubU32) and _imm_in(i, 1) for i in items), ( + "expected q-1 to reach the predecessor queue" + ) + assert any(isinstance(i, SAndB32) and _imm_in(i, 0x7) for i in items), ( + "expected (q-1) & 0x7 wrap for the predecessor index" + ) - def test_disables_auto_reset_with_all_ones(self): + def test_adds_predecessor_workgroups_to_bound(self): items = self._emit() - movs = [ - i for i in items - if isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) - ] - assert movs, "expected 0xFFFFFFFF mov to disable the home auto-reset" + # The predecessor structural share plus the final fold-in add. + assert sum(isinstance(i, SAddU32) for i in items) >= 2, ( + "expected the W_(q-1) structural add and the bound fold-in" + ) + def test_does_not_disable_auto_reset(self): + items = self._emit() + assert not any( + isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) for i in items + ), "the home bound must be a finite predecessor-inclusive value" -class TestStealEmission: - """One single-hop NEXT steal: walk to (queueIdx+1) & 0x7, guard against the - neighbor having no structural extra, then a single auto-reset-disabled - atomic increment against the stolen queue.""" +# =========================================================================== +# 3b. Next-neighbor steal: one unconditional (past home-empty) NEXT-neighbor steal with +# the static predecessor-inclusive auto-reset bound. No remainder/extra +# guards. +# =========================================================================== +class TestStealEmission: def _emit(self): sk = _stream_k_instance(4) writer = _FakeWriter() @@ -264,7 +308,7 @@ def _emit(self): sQueueIdx = writer.sgprPool.checkOut(1, "queueIdx") sWorkItemIdx = writer.sgprPool.checkOut(1, "workItemIdx") sk.streamKWorkStealingSteal( - writer, module, {}, sQueueIdx, sWorkItemIdx, _mk_label + writer, module, _WS_KERNEL, sQueueIdx, sWorkItemIdx, "skGrid", _mk_label ) return _flat(module) @@ -274,29 +318,34 @@ def test_neighbor_walk_is_plus_one_then_wrap(self): assert any( isinstance(i, SAddU32) and _imm_in(i, 1) for i in items ), "expected +1 advance to the next queue" - # ... wrapped within the 8-queue ring via & 0x7. + # ... wrapped within the 8 queues (single-hop next-neighbor) via & 0x7. assert any( isinstance(i, SAndB32) and _imm_in(i, 0x7) for i in items ), "expected (queueIdx+1) & 0x7 wrap" - def test_skips_when_neighbor_has_no_extra(self): + def test_no_neighbor_extra_guard(self): + # The next-neighbor steal always fires on an empty home fetch: the old + # ">= remainder / neighbor-has-no-structural-extra" guard is removed. items = self._emit() - assert any(isinstance(i, SCmpGeU32) for i in items), ( - "expected a >= remainder guard so a neighbor without a structural " - "extra is not robbed" + assert not any(isinstance(i, SCmpGeU32) for i in items), ( + "the next-neighbor steal must NOT guard on a neighbor structural extra" ) def test_exactly_one_atomic_increment(self): items = self._emit() atomics = [i for i in items if isinstance(i, SAtomicInc)] - assert len(atomics) == 1, "single-hop steal must emit exactly one atomic" + assert len(atomics) == 1, "the steal must emit exactly one atomic" - def test_atomic_uses_auto_reset_disabled_bound(self): + def test_atomic_bound_is_predecessor_inclusive_not_all_ones(self): items = self._emit() + # The static bound tiles_s + W_s + W_q - 1 ends in a -1 (SSubU32 imm 1) + # and must never load the 0xFFFFFFFF disable-auto-reset sentinel. + assert not any( + isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) for i in items + ), "the stolen atomic must use a finite self-reset bound, not 0xFFFFFFFF" assert any( - isinstance(i, SMovB32) and _imm_in(i, 0xFFFFFFFF) - for i in items - ), "the stolen atomic must run with auto-reset disabled (0xFFFFFFFF)" + isinstance(i, SSubU32) and _imm_in(i, 1) for i in items + ), "expected the '- 1' that forms the atomic_inc auto-reset bound" def test_guards_on_valid_home_fetch(self): # A valid home fetch (index < TotalItems) must short-circuit the steal. @@ -304,42 +353,80 @@ def test_guards_on_valid_home_fetch(self): assert any(isinstance(i, SCmpLtU32) for i in items) assert any(isinstance(i, SCBranchSCC1) for i in items) + def test_no_reset_barrier(self): + items = self._emit() + assert not any(isinstance(i, SBarrier) for i in items), ( + "the steal helper must not emit a reset barrier" + ) -class TestKernelEndResetEmission: - """The last WG zeroes the 8 per-queue counters plus the completion counter, - behind a barrier + wave-0 completion count.""" - def _emit(self): - sk = _stream_k_instance(4) - writer = _FakeWriter() - module = sk.streamKWorkStealingKernelEndReset(writer, {}, "skGrid", _mk_label) - return _flat(module) +# =========================================================================== +# 3c. Sticky-home: the home fetch is gated by a persistent StreamKStickyEmpty +# SGPR, and the flag is latched on an empty home fetch. Verified against +# the real graWorkGroup source. +# =========================================================================== +class TestStickyHomeGate: + @pytest.mark.parametrize( + "func", [StreamKDynamic.graWorkGroup, StreamKHybrid.graWorkGroup] + ) + def test_home_fetch_is_gated_by_sticky_flag(self, func): + src = _source_of(func) + assert "StreamKStickyEmpty" in src, ( + "the home fetch must be gated by the persistent sticky-empty SGPR" + ) + # A steal-only skip label proves the home s_atomic_inc is bypassed once + # the WG has gone sticky. + assert "SK_StealOnly" in src, ( + "sticky WGs must skip the home fetch and steal only" + ) - def test_starts_with_barrier(self): - items = self._emit() - assert any(isinstance(i, SBarrier) for i in items) + @pytest.mark.parametrize( + "func", [StreamKDynamic.graWorkGroup, StreamKHybrid.graWorkGroup] + ) + def test_steal_passes_grid_sgpr(self, func): + # The steal now needs the mode-appropriate grid SGPR for its bound. + src = _source_of(func) + grid = "SKGrid" if func is StreamKHybrid.graWorkGroup else "skGrid" + assert "streamKWorkStealingSteal" in src + assert grid in src - def test_resets_eight_queues_plus_completion_counter(self): - items = self._emit() - stores = [i for i in items if isinstance(i, SStoreB32)] - assert len(stores) == StreamK._WS_NUM_QUEUES + 1 == 9, ( - "expected 8 per-queue counter resets + 1 completion counter reset" - ) - def test_completion_count_uses_atomic_inc(self): - items = self._emit() - assert any(isinstance(i, SAtomicInc) for i in items), ( - "wave 0 counts completed WGs via an atomic increment" +# =========================================================================== +# 3d. No explicit reset survives anywhere in kernelEnd, and the 0x80 +# completion counter / reset barrier are gone. +# =========================================================================== +class TestNoExplicitReset: + @pytest.mark.parametrize("func", [StreamKDynamic.kernelEnd, StreamKHybrid.kernelEnd]) + def test_kernelend_has_no_ws_calls(self, func): + assert _all_ws_calls(func) == set(), ( + "kernelEnd must not emit any work-stealing reset" ) - def test_only_last_wg_resets(self): - # SCBranchSCC0 guards (a) wave-0-only and (b) last-WG-only. - items = self._emit() - assert sum(isinstance(i, SCBranchSCC0) for i in items) >= 2 + @pytest.mark.parametrize("func", [StreamKDynamic.kernelEnd, StreamKHybrid.kernelEnd]) + def test_kernelend_has_no_completion_counter(self, func): + src = _source_of(func) + assert "0x80" not in src + assert "completion" not in src.lower() or "no explicit" in src.lower() # =========================================================================== -# 3b. Absence-by-toggle: the helpers are only reached behind the +# 5. Per-architecture queue-count lookup (C1): fast masking requires a +# power-of-two queue count; the count is keyed off kernel["ISA"]. +# =========================================================================== +class TestQueueConstants: + @pytest.mark.parametrize("isa", [(9, 4, 0), (9, 5, 0)]) + def test_supported_arches_use_eight_power_of_two_queues(self, isa): + sk = _stream_k_instance(4) + numQueues, mask, log2Queues, cacheLineLog2 = sk._wsQueueConstants({"ISA": isa}) + assert numQueues == 8 + assert numQueues & (numQueues - 1) == 0, "queue count must be power of two" + assert mask == numQueues - 1 == 0x7 + assert (1 << log2Queues) == numQueues + assert (1 << cacheLineLog2) == 256 + + +# =========================================================================== +# 3e. Absence-by-toggle: the helpers are only reached behind the # ``kernel["StreamKWorkStealing"]`` gate at every callsite. Verified # against the real source so "off" provably emits nothing extra. # =========================================================================== @@ -347,26 +434,14 @@ class TestCallsitesAreToggleGated: def test_sk4_grawg_steal_calls_are_all_gated(self): guarded = _ws_guarded_calls(StreamKDynamic.graWorkGroup) allcalls = _all_ws_calls(StreamKDynamic.graWorkGroup) - assert {"streamKWorkStealingHomeNoReset", "streamKWorkStealingSteal"} <= guarded + assert {"streamKWorkStealingHomeBound", "streamKWorkStealingSteal"} <= guarded # Nothing slips through ungated. assert allcalls == guarded - def test_sk4_kernelend_reset_is_gated(self): - guarded = _ws_guarded_calls(StreamKDynamic.kernelEnd) - allcalls = _all_ws_calls(StreamKDynamic.kernelEnd) - assert "streamKWorkStealingKernelEndReset" in guarded - assert allcalls == guarded - def test_sk5_grawg_steal_calls_are_all_gated(self): guarded = _ws_guarded_calls(StreamKHybrid.graWorkGroup) allcalls = _all_ws_calls(StreamKHybrid.graWorkGroup) - assert {"streamKWorkStealingHomeNoReset", "streamKWorkStealingSteal"} <= guarded - assert allcalls == guarded - - def test_sk5_kernelend_reset_is_gated(self): - guarded = _ws_guarded_calls(StreamKHybrid.kernelEnd) - allcalls = _all_ws_calls(StreamKHybrid.kernelEnd) - assert "streamKWorkStealingKernelEndReset" in guarded + assert {"streamKWorkStealingHomeBound", "streamKWorkStealingSteal"} <= guarded assert allcalls == guarded @@ -378,11 +453,12 @@ class TestSolutionValidation: def setup_method(self): self.validate = _extract_ws_validation() - def _run(self, *, streamk, atomic, work_stealing=1): + def _run(self, *, streamk, atomic, work_stealing=1, debug_streamk=0): state = { "StreamKWorkStealing": work_stealing, "StreamK": streamk, "StreamKAtomic": atomic, + "DebugStreamK": debug_streamk, } self.validate(state, False, reject) return state @@ -402,8 +478,15 @@ def test_rejected_with_atomic(self, streamk): state = self._run(streamk=streamk, atomic=1) assert state["Valid"] is False + @pytest.mark.parametrize("debug", [1, 2, 3]) + def test_rejected_with_debug_streamk(self, debug): + # Proof condition #1: DebugStreamK overrides could break the W_q>=1 + # (skGrid>=numQueues) precondition the per-queue auto-reset relies on. + state = self._run(streamk=4, atomic=0, debug_streamk=debug) + assert state["Valid"] is False + def test_off_toggle_is_inert_even_for_bad_combo(self): # With the toggle off the guard must not fire, even for a combination # that would otherwise be rejected. - state = self._run(streamk=3, atomic=1, work_stealing=0) + state = self._run(streamk=3, atomic=1, work_stealing=0, debug_streamk=3) assert "Valid" not in state From 91bc7272d5d6b6e4263116f1ef5984923ebe3b9d Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 13 Jul 2026 21:33:24 +0000 Subject: [PATCH 12/20] feat(tensilelite): derive StreamK work-queue count from per-arch XCD count Add a NumXCD arch capability (populated in rocisa initArchCaps) mirroring origami get_default_num_xcds. StreamK codegen now derives the work-queue count from archCaps["NumXCD"] instead of a hardcoded 8 table; the value is a compile-time constant and asserted to be a power of two. The host derives the same count via origami and rejects at runtime when NUM_XCD is non-power-of-two or does not match the arch's XCD count; the work-stealing workspace is sized from it. Generated assembly is byte-identical for gfx942/gfx950. --- .../tensilelite/Tensile/Components/StreamK.py | 54 +++---- .../Tensile/SolutionStructs/Solution.py | 17 ++- .../Tests/unit/test_streamk_work_stealing.py | 28 ++-- .../rocisa/rocisa/include/hardware_caps.hpp | 7 + .../tensilelite/src/ContractionSolution.cpp | 133 ++++++++++++------ .../tensilelite/tests/CuCount_test.cpp | 66 +++++++-- 6 files changed, 209 insertions(+), 96 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index a8982e03a157..d9a8c89d5db3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -413,37 +413,41 @@ def _skv(self, writer, name): # The dynamic-queue fetch partitions work across one queue per XCD and maps # StreamKIdx -> queueIdx (StreamKIdx & (numQueues-1)) and queueIdx -> cache # line (queueIdx << log2(256)) with shift/AND fast masking. That masking is - # only valid when the queue count is a power of two, so the count is keyed - # by architecture here instead of hardcoding a literal 8. + # only valid when the queue count is a power of two, so the count comes from + # the per-arch capability instead of hardcoding a literal 8. + # + # The count is read at codegen time from ``writer.states.archCaps["NumXCD"]`` + # (see rocisa initArchCaps), which MIRRORS origami's per-arch XCD count + # (origami::hardware_t::get_default_num_xcds): one queue per XCD. Codegen + # cannot import origami (C++), so the cap is the single codegen-side mirror + # of that value; the host (ContractionSolution.cpp streamKBakedQueueCount) + # reads the SAME origami value, keeping the two in lockstep. The host guard + # additionally enforces that the device's RUNTIME NUM_XCD equals this baked + # per-XCD queue count (and is a power of two) before allowing the + # dynamic-queue path. # # NOTE: MI300A and MI300X BOTH report gfx942; only the physical XCD count # differs (MI300A=6, MI300X=8). Codegen cannot tell them apart, so gfx942 is - # generated for 8 queues and the host runtime (ContractionSolution.cpp) - # rejects the dynamic-queue / work-stealing path when the device does not - # expose the assumed power-of-two queue count (e.g. MI300A's 6 XCDs). - _WS_NUM_QUEUES_BY_ISA = { - (9, 4): 8, # gfx942 (MI300X) - (9, 5): 8, # gfx950 - } - _WS_DEFAULT_NUM_QUEUES = 8 + # generated for 8 queues (origami's gfx942 default) and the host runtime + # rejects the dynamic-queue / work-stealing path when the device's runtime + # NUM_XCD does not equal that baked count (e.g. MI300A's 6 XCDs). _WS_CACHE_LINE_LOG2 = 8 # log2(256): per-queue counters are one per 256B line - def _wsQueueConstants(self, kernel): + def _wsQueueConstants(self, writer, kernel): """Return (numQueues, mask, log2Queues, cacheLineLog2) for this arch. Centralizes the dynamic-queue / work-stealing fast-mask constants so the - formerly hardcoded "8" lives in exactly one place. ``numQueues`` is - looked up per architecture from ``kernel["ISA"]`` and must be a power of - two for the shift/AND masking to be valid; the host guards the same - assumption at runtime (rejecting non-power-of-two XCD counts like - MI300A's 6). + queue count lives in exactly one place. ``numQueues`` is read from the + per-arch capability ``writer.states.archCaps["NumXCD"]`` (the codegen + mirror of origami get_default_num_xcds) and must be a power of two for + the shift/AND masking to be valid; the host guards the same assumption + at runtime, rejecting devices whose runtime NUM_XCD is not a power of two + or does not equal this baked count (e.g. MI300A's 6 XCDs). """ - isa = tuple(kernel["ISA"][:2]) - numQueues = self._WS_NUM_QUEUES_BY_ISA.get(isa, self._WS_DEFAULT_NUM_QUEUES) + numQueues = writer.states.archCaps["NumXCD"] assert numQueues > 0 and (numQueues & (numQueues - 1)) == 0, ( - "StreamK dynamic-queue fast masking requires a power-of-two queue " - "count (got %d for ISA %s)" % (numQueues, isa) - ) + "StreamK dynamic-queue fast masking requires a power-of-two queue count " + "(got %d for ISA %s)" % (numQueues, tuple(kernel["ISA"][:2]))) return numQueues, numQueues - 1, log2(numQueues), self._WS_CACHE_LINE_LOG2 def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, comment): @@ -482,7 +486,7 @@ def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, s workgroup per queue (skGrid >= numQueues); the Solution layer rejects debug overrides that could violate this. """ - _, mask, log2Queues, _ = self._wsQueueConstants(kernel) + _, mask, log2Queues, _ = self._wsQueueConstants(writer, kernel) sPred = writer.sgprPool.checkOut(1, "wsPredQueue") sWp = writer.sgprPool.checkOut(1, "wsPredWorkgroups") sTmp = writer.sgprPool.checkOut(1, "wsPredTmp") @@ -520,7 +524,7 @@ def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, stolen queue. This lets the neighbor counter wrap back to 0 on its own, removing the need for an explicit end-of-kernel reset. """ - _, mask, log2Queues, cacheLineLog2 = self._wsQueueConstants(kernel) + _, mask, log2Queues, cacheLineLog2 = self._wsQueueConstants(writer, kernel) skFetchDone = mkLabel("SK_FetchDone") mod.add(SCmpLtU32(src0=sgpr(sWorkItemIdx), src1=sgpr("TotalItems"), comment="Home fetch valid?")) mod.add(SCBranchSCC1(labelName=skFetchDone.getLabelName(), comment="Valid work fetched; no steal")) @@ -3134,7 +3138,7 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for the # StreamKIdx/queue divisions, log2(256) for the cache-line stride). - _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(kernel) + _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") @@ -3792,7 +3796,7 @@ def emitDynamicGRA(mod): # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for # the StreamKIdx/queue divisions, log2(256) for cache-line stride). - _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(kernel) + _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 5889c8d47508..1b2ea4572130 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1665,13 +1665,16 @@ def assignDerivedParameters( if state["StreamKWorkStealing"]: # NOTE: these are codegen-time rejections only; there is no hardware # context here (MI300A and MI300X both compile as gfx942). The kernel - # hardcodes a power-of-two queue count (8), so the check that the device - # actually exposes a power-of-two XCD count -- and the explicit - # rejection of the dynamic-queue / work-stealing path on non-power-of-two - # devices like MI300A's 6 XCDs (a non-work-stealing solution serves the - # GEMM instead, and the user is warned) -- lives host-side in - # ContractionSolution.cpp (streamKDynamicQueueSupported / - # streamKDynamicQueueUnsupported, wired into softwarePredicate). + # bakes in a power-of-two per-XCD queue count (8, mirroring origami's + # per-arch XCD count), so the check that the device actually matches it + # -- the host now requires the device's RUNTIME NUM_XCD to EQUAL the + # baked per-XCD queue count (derived from origami), not merely be a power + # of two, and excludes the dynamic-queue / work-stealing path otherwise + # (e.g. MI300A's 6 XCDs, or a power-of-two-but-mismatched partition); a + # non-work-stealing solution serves the GEMM instead and the user is + # warned -- lives host-side in ContractionSolution.cpp + # (streamKDynamicQueueSupported / streamKDynamicQueueUnsupported / + # streamKBakedQueueCount, wired into softwarePredicate). # Work stealing only exists in the dynamic-queue fetch (auto-mode # SK4 and the SK4 sub-path of SK5). if state["StreamK"] not in (4, 5): diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index 01f215a96798..6c90c64c07c7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -24,6 +24,7 @@ import ast import inspect import textwrap +import types import pytest @@ -83,8 +84,11 @@ def checkIn(self, *args, **kwargs): class _FakeWriter: - def __init__(self): + def __init__(self, numXCD: int = 8): self.sgprPool = _FakeSgprPool() + # gfx942/gfx950 mirror origami get_default_num_xcds == 8; the helpers + # read the per-arch queue count from writer.states.archCaps["NumXCD"]. + self.states = types.SimpleNamespace(archCaps={"NumXCD": numXCD}) def _mk_label(base: str) -> Label: @@ -411,18 +415,26 @@ def test_kernelend_has_no_completion_counter(self, func): # =========================================================================== # 5. Per-architecture queue-count lookup (C1): fast masking requires a -# power-of-two queue count; the count is keyed off kernel["ISA"]. +# power-of-two queue count; the count is read from the per-arch capability +# writer.states.archCaps["NumXCD"] (the codegen mirror of origami +# get_default_num_xcds). # =========================================================================== class TestQueueConstants: @pytest.mark.parametrize("isa", [(9, 4, 0), (9, 5, 0)]) def test_supported_arches_use_eight_power_of_two_queues(self, isa): sk = _stream_k_instance(4) - numQueues, mask, log2Queues, cacheLineLog2 = sk._wsQueueConstants({"ISA": isa}) - assert numQueues == 8 - assert numQueues & (numQueues - 1) == 0, "queue count must be power of two" - assert mask == numQueues - 1 == 0x7 - assert (1 << log2Queues) == numQueues - assert (1 << cacheLineLog2) == 256 + # gfx942/gfx950 mirror origami get_default_num_xcds == 8, so the + # constants tuple is exactly (numQueues=8, mask=7, log2=3, cacheLog2=8). + writer = _FakeWriter(numXCD=8) + assert sk._wsQueueConstants(writer, {"ISA": isa}) == (8, 7, 3, 8) + + def test_non_power_of_two_queue_count_asserts(self): + # The shift/AND fast masking is only valid for a power-of-two queue + # count; a non-power-of-two NumXCD cap must trip the guard assert. + sk = _stream_k_instance(4) + writer = _FakeWriter(numXCD=6) + with pytest.raises(AssertionError): + sk._wsQueueConstants(writer, {"ISA": (9, 9, 0)}) # =========================================================================== diff --git a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp index d8b5badb55b0..d5165c860c34 100644 --- a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp +++ b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp @@ -601,6 +601,13 @@ inline std::map initArchCaps(const IsaVersion& isaVersion) rv["LDSBankCount"] = 64; rv["LDSBankWidth"] = 4; // bytes per bank + // Per-XCD work-queue count for StreamK dynamic-queue kernels. Single + // codegen-side mirror of origami get_default_num_xcds(): gfx942/gfx950 are + // 8-XCD, every other arch is 1. Power-of-two so StreamK queue masking (AND/ + // shift) stays valid; the host guard rejects devices whose runtime NUM_XCD + // != this value. + rv["NumXCD"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 8 : 1; + return rv; } diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 010336e93f8f..7e53618eb2f4 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -60,36 +60,71 @@ namespace TensileLite namespace { // The dynamic-queue StreamK kernels (SK4 and the SK4 sub-path of SK5) - // hardcode a fixed, power-of-two per-XCD queue count for fast index - // masking (StreamKIdx & (Q-1), queueIdx << log2(256)). Codegen emits - // Q = 8 for gfx942/gfx950 (see StreamK.py _WS_NUM_QUEUES_BY_ISA), so the - // per-XCD work-queue region reserved in the Synchronizer workspace is - // always 8 cache lines (256 B each) wide. - constexpr size_t StreamKDynamicQueueCount = 8; + // bake in a fixed, power-of-two per-XCD queue count for fast index + // masking (StreamKIdx & (Q-1), queueIdx << log2(256)). That baked count + // is the per-arch XCD count: codegen keys it off the ISA (see StreamK.py + // _WS_NUM_QUEUES_BY_ISA, which MIRRORS origami get_default_num_xcds) and + // emits Q = 8 for gfx942/gfx950. The host reads the SAME origami value + // here -- via the device's origami architecture -- so the per-XCD + // work-queue region reserved in the Synchronizer workspace and the + // runtime acceptance guard stay in lockstep with codegen instead of + // duplicating a hardcoded literal 8. + // + // Returns the baked per-XCD queue count for the device's architecture, + // or 0 when the architecture cannot be determined (no analytical + // hardware, or an architecture origami has no default XCD count for), + // which the guard below treats as unsupported. + inline size_t streamKBakedQueueCount(Hardware const& hardware) + { + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) + return 0; + try + { + return origami::hardware_t::get_default_num_xcds( + hipAMDGPU->analyticalHardware->arch); + } + catch(std::exception const&) + { + // origami throws for architectures without a hardcoded default + // XCD count; treat that as "cannot determine" (0 == unsupported) + // rather than propagating the exception through solution + // selection. + return 0; + } + } // The dynamic-queue fetch / work stealing is only correct when the - // device exposes a power-of-two number of XCDs (one queue per XCD), - // because the kernel masks with (Q-1). MI300A and MI300X BOTH report - // gfx942, but MI300A has 6 XCDs (not a power of two), so the fast-mask - // assumption breaks and the dynamic-queue path is NOT supported there. - // Returns true when NUM_XCD is known and is NOT a power of two; returns - // false when the XCD count is unknown so historic behavior is preserved. - // The numeric power-of-two classification is intentionally isolated here - // so it stays trivially unit-testable (see CuCount_test.cpp). + // device's runtime XCD count matches the baked per-XCD queue count AND + // is a power of two (the kernel masks with (Q-1)). MI300A and MI300X + // BOTH report gfx942 -- codegen bakes Q = 8 -- but MI300A has 6 XCDs + // (not a power of two), so the fast-mask assumption breaks there. + // + // Returns true (UNSUPPORTED / reject) when the runtime NUM_XCD is 0 + // (defensive), is NOT a power of two, the baked count is unknown, or the + // runtime NUM_XCD does not equal the baked count. The last condition + // additionally closes the partition/CPX gap where NUM_XCD is a power of + // two but != baked (e.g. a 4-XCD partition of an 8-XCD gfx942), which + // would still mis-map the fixed Q=8 queue masking. Returns false when + // there is no analytical hardware so historic behavior is preserved + // (unknown XCD -> allow). The classification is intentionally isolated + // here so it stays trivially unit-testable (see CuCount_test.cpp). inline bool streamKDynamicQueueUnsupported(Hardware const& hardware) { auto const* hipAMDGPU = dynamic_cast(&hardware); if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) return false; + size_t baked = streamKBakedQueueCount(hardware); size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; - return numXCD == 0 || (numXCD & (numXCD - 1)) != 0; + return numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || baked == 0 + || numXCD != baked; } // Emit a single, user-visible warning (not once-per-call spam) when a // StreamK dynamic-queue / work-stealing solution is excluded from - // selection because the device's XCD count is not a power of two. This - // is what surfaces the reject to the user instead of silently degrading - // to tree reduction. + // selection because the device's XCD count does not match the compiled + // per-XCD queue count. This is what surfaces the reject to the user + // instead of silently degrading to tree reduction. void warnStreamKDynamicQueueUnsupportedOnce(Hardware const& hardware) { static std::once_flag warnedFlag; @@ -98,9 +133,11 @@ namespace TensileLite auto const* hipAMDGPU = dynamic_cast(&hardware); if(hipAMDGPU != nullptr && hipAMDGPU->analyticalHardware != nullptr) numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; + size_t baked = streamKBakedQueueCount(hardware); std::cerr << "hipBLASLt Warning: StreamK dynamic-queue (work-stealing) solutions " - "require a power-of-two XCD count; this device reports NUM_XCD=" - << numXCD + "require the device's XCD count to be a power of two and to equal the " + "compiled per-XCD queue count; this device reports NUM_XCD=" + << numXCD << " with a compiled per-XCD queue count of " << baked << ", so those solutions are excluded from selection and a " "non-work-stealing solution will be used instead.\n"; }); @@ -3197,14 +3234,16 @@ namespace TensileLite ? streamK5EffectiveDynamic(problem, hardware) : false; // Defensive: dynamic-queue / work-stealing StreamK solutions are - // excluded from selection on devices whose XCD count is not a power - // of two (see streamKDynamicQueueSupported() wired into - // softwarePredicate). The normal path therefore never reaches solve() - // for such a solution; a different (SK3-static / non-StreamK) - // solution serves the GEMM instead. If we DO get here it means the - // software predicate was bypassed (e.g. an explicit select-by-index), - // so reject EXPLICITLY rather than silently running the fixed-mask - // kernel with a mismatched queue count (which would corrupt results). + // excluded from selection on devices whose runtime XCD count is not + // a power of two or does not equal the baked per-XCD queue count + // (see streamKDynamicQueueSupported() wired into softwarePredicate). + // The normal path therefore never reaches solve() for such a + // solution; a different (SK3-static / non-StreamK) solution serves + // the GEMM instead. If we DO get here it means the software + // predicate was bypassed (e.g. an explicit select-by-index), so + // reject EXPLICITLY rather than silently running the fixed-mask + // kernel with a mismatched queue count (which would corrupt + // results). const bool dynamicQueuePath = (sizeMapping.streamK == 4) || (sizeMapping.streamK == 5 && effectiveDynamic); @@ -3213,7 +3252,8 @@ namespace TensileLite warnStreamKDynamicQueueUnsupportedOnce(hardware); throw std::runtime_error( "hipBLASLt Error: StreamK dynamic-queue (work-stealing) solution selected on a " - "device whose XCD count is not a power of two; this kernel is unsupported here. " + "device whose XCD count is not a power of two or does not equal the compiled " + "per-XCD queue count; this kernel is unsupported here. " "Select a non-work-stealing solution instead."); } if(sizeMapping.streamK == 4) @@ -3239,9 +3279,12 @@ namespace TensileLite { size_t idealWorkspace = partialTileSize(sk.grid); // SK4 and SK5-dynamic need the per-XCD work-queue region; SK5-static - // sizes like standalone SK3. + // sizes like standalone SK3. The region is sized to the kernel's + // baked per-XCD queue count (origami's per-arch XCD count); the + // acceptance guard requires runtime NUM_XCD == baked, so this + // equals 256 * NUM_XCD for every device that reaches here. if(dynamicQueuePath) - idealWorkspace += 256 * StreamKDynamicQueueCount; + idealWorkspace += 256 * streamKBakedQueueCount(hardware); // If given workspace is less than ideal, we can fall back to DP mode // Performance will likely be lower, but the kernel can run if workspace is unavailable. // (The non-power-of-two XCD case is handled earlier by explicit @@ -3672,12 +3715,14 @@ namespace TensileLite { size_t idealWorkspace = partialTileSize(skGrid); // Reserve the per-XCD work-queue region for the dynamic-queue - // path. Sized to the kernel's fixed queue count (8); this may - // slightly over-report on a device that falls back to tree - // reduction (e.g. MI300A), which is safe (never under-sized). + // path. Sized to the kernel's baked per-XCD queue count + // (origami's per-arch XCD count, e.g. 8 for gfx942/gfx950); + // this may slightly over-report on a device that falls back + // to tree reduction (e.g. MI300A), which is safe (never + // under-sized). if(sizeMapping.streamK == 4 || (sizeMapping.streamK == 5 && effectiveDynamic)) - idealWorkspace += 256 * StreamKDynamicQueueCount; + idealWorkspace += 256 * streamKBakedQueueCount(hardware); // If given workspace is less than ideal, we can fall back to DP mode // Performance will likely be lower, but the kernel can run if workspace is unavailable if(idealWorkspace <= problem.workspaceSize()) @@ -3974,17 +4019,19 @@ namespace TensileLite if(sizeMapping.streamK != 4 && sizeMapping.streamK != 5) return true; - // Fast/common path: on hardware whose XCD count IS a power of two the - // fixed power-of-two queue masking is valid, so nothing is excluded. - // This also covers the "unknown XCD" case (predicate returns false), - // preserving historic behavior. Checked before streamK5EffectiveDynamic() - // so the mainline gfx942(MI300X)/gfx950 path stays cheap. + // Fast/common path: on hardware whose runtime XCD count is a power of + // two AND equals the baked per-XCD queue count, the fixed queue masking + // is valid, so nothing is excluded. This also covers the "unknown XCD" + // case (predicate returns false), preserving historic behavior. Checked + // before streamK5EffectiveDynamic() so the mainline gfx942(MI300X)/ + // gfx950 path stays cheap. if(!streamKDynamicQueueUnsupported(hardware)) return true; - // XCD count is not a power of two. Only the dynamic-queue sub-path is - // affected: an SK5 solution that resolves to the static (SK3) sub-path - // for this problem stays valid and selectable. + // Runtime XCD count is not a power of two or does not equal the baked + // per-XCD queue count. Only the dynamic-queue sub-path is affected: an + // SK5 solution that resolves to the static (SK3) sub-path for this + // problem stays valid and selectable. const bool dynamicQueue = (sizeMapping.streamK == 4) || (sizeMapping.streamK == 5 && streamK5EffectiveDynamic(problem, hardware)); diff --git a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp index ce461ad58c9a..f63f3d414f69 100644 --- a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp @@ -742,10 +742,13 @@ TEST(Sk3Sk5OffPartition512Test, NativeSk3MatchesSk5OffHostPack) // =========================================================================== // StreamKDynamicQueueXcdGateTest -- MI300A (NUM_XCD=6) reject-and-continue. // -// The SK4 / SK5-dynamic work-stealing kernels use a fixed, power-of-two per-XCD -// queue count (8) and mask queue/tile indices with (Q-1). That fast masking is -// only valid when the device exposes a power-of-two number of XCDs. MI300A and -// MI300X both report gfx942, but MI300A has 6 XCDs (not a power of two). +// The SK4 / SK5-dynamic work-stealing kernels bake in a fixed, power-of-two +// per-XCD queue count (origami's per-arch XCD count: 8 for gfx942/gfx950) and +// mask queue/tile indices with (Q-1). That fast masking is only valid when the +// device's runtime XCD count equals the baked count AND is a power of two. +// MI300A and MI300X both report gfx942 (baked count 8), but MI300A has 6 XCDs +// (not a power of two). A power-of-two-but-mismatched partition (e.g. a 4-XCD +// slice of an 8-XCD gfx942) is likewise rejected because runtime NUM_XCD != 8. // // Rather than SILENTLY degrading such a solution to tree reduction, the host // now EXCLUDES the dynamic-queue / work-stealing solution from selection and @@ -755,23 +758,45 @@ TEST(Sk3Sk5OffPartition512Test, NativeSk3MatchesSk5OffHostPack) // softwarePredicate() (SolutionLibrary.hpp) so returning false drops the // solution from findBestSolution/findAllSolutions. It builds on the file-local // numeric predicate streamKDynamicQueueUnsupported(hardware): dynamic_cast to -// hip::HipAMDGPU, read analyticalHardware->NUM_XCD, return true when NUM_XCD is -// 0 (defensive) or not a power of two, false otherwise (incl. unknown/no -// analytical hardware -> historic behavior preserved). +// hip::HipAMDGPU, derive the baked per-XCD queue count from the origami arch +// (streamKBakedQueueCount / get_default_num_xcds), read analyticalHardware-> +// NUM_XCD, and return true (reject) when NUM_XCD is 0 (defensive), not a power +// of two, the baked count is unknown, or NUM_XCD != baked; false otherwise +// (incl. unknown/no analytical hardware -> historic behavior preserved). // // Both production predicates live in anonymous namespaces / a .cpp translation // unit and cannot be linked from here, so -- following the same convention as // computeStreamKHostPack above (which mirrors solve()'s internal reduction // decision for host-only testing) -- this test mirrors them and drives them // through a real hip::HipAMDGPU mock. It validates (a) the Hardware& -> -// HipAMDGPU -> analyticalHardware -> NUM_XCD accessor chain, (b) the power-of- -// two classification (6 -> reject, 8 -> allow, unknown -> allow), and (c) the -// selection-predicate composition (dynamic-queue on 6 XCD -> excluded; other -// StreamK modes / non-power-of-two-agnostic solutions -> selectable). It is NOT -// executed on real MI300A silicon. +// HipAMDGPU -> analyticalHardware -> {arch, NUM_XCD} accessor chain, (b) the +// power-of-two AND equals-baked classification (6 -> reject, 4 -> reject +// (pow2 but != baked 8), 8 -> allow, unknown -> allow), and (c) the +// selection-predicate composition (dynamic-queue on a mismatched XCD count -> +// excluded; other StreamK modes / queue-count-agnostic solutions -> +// selectable). It is NOT executed on real MI300A silicon. // =========================================================================== namespace { + // Mirror of the anonymous-namespace helper in ContractionSolution.cpp + // (streamKBakedQueueCount): the baked per-XCD queue count comes from + // origami's per-arch XCD count. Kept in lockstep with the production code. + inline size_t streamKBakedQueueCountRef(Hardware const& hardware) + { + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) + return 0; + try + { + return origami::hardware_t::get_default_num_xcds( + hipAMDGPU->analyticalHardware->arch); + } + catch(std::exception const&) + { + return 0; + } + } + // Byte-for-byte mirror of the anonymous-namespace numeric predicate in // ContractionSolution.cpp (streamKDynamicQueueUnsupported). Kept in lockstep // with the production code; if that predicate changes, update this too. @@ -780,8 +805,10 @@ namespace auto const* hipAMDGPU = dynamic_cast(&hardware); if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) return false; + size_t baked = streamKBakedQueueCountRef(hardware); size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; - return numXCD == 0 || (numXCD & (numXCD - 1)) != 0; + return numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || baked == 0 + || numXCD != baked; } // Mirror of ContractionSolution::streamKDynamicQueueSupported(). Returns @@ -849,6 +876,19 @@ TEST(StreamKDynamicQueueXcdGateTest, AllowsMi300xEightXcd) << "MI300X (NUM_XCD=8, power of two) must keep the dynamic-queue path"; } +TEST(StreamKDynamicQueueXcdGateTest, RejectsGfx942FourXcdPowerOfTwoButMismatched) +{ + // A 4-XCD partition (e.g. a CPX-style slice) of an 8-XCD gfx942: 4 IS a + // power of two, but the kernel bakes Q=8, so runtime NUM_XCD (4) != baked + // (8) and the fixed Q=8 masking would mis-map queues. Must be rejected. + hip::HipAMDGPU gfx942Cpx = makeGfx942DeviceWithXcd(4); + Hardware const& hw = gfx942Cpx; + EXPECT_EQ(streamKBakedQueueCountRef(hw), 8u) + << "gfx942 must bake origami's per-arch XCD count (8)"; + EXPECT_TRUE(streamKDynamicQueueUnsupportedRef(hw)) + << "gfx942 with NUM_XCD=4 (power of two but != baked 8) must be rejected"; +} + TEST(StreamKDynamicQueueXcdGateTest, AllowsGfx950EightXcd) { // gfx950 (local MI355X) analytical hardware advertises 8 XCDs. From 79924af942dd4fbb788fb605a252e46949c157f5 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 13 Jul 2026 22:52:39 +0000 Subject: [PATCH 13/20] refactor(tensilelite): tidy StreamK work-stealing comments and derive cache-line log2 Trim over-verbose and stale StreamK work-stealing comments and docstrings; fix the stale _WS_NUM_QUEUES_BY_ISA reference in ContractionSolution.cpp to point at _wsQueueConstants / archCaps["NumXCD"]; clarify that gfx942 covers both MI300X (8 XCDs) and MI300A (6 XCDs) with MI300A excluded at runtime by the host guard; compute _WS_CACHE_LINE_LOG2 from a named 256B constant. No logic change; emitted asm byte-identical (gfx942 and gfx950, SK4 and SK5, StreamKWorkStealing 0 and 1). --- .../tensilelite/Tensile/Components/StreamK.py | 126 +++++++----------- .../Tensile/SolutionStructs/Solution.py | 28 ++-- .../Tests/unit/test_streamk_work_stealing.py | 20 +-- .../rocisa/rocisa/include/hardware_caps.hpp | 12 +- .../tensilelite/src/ContractionSolution.cpp | 40 ++---- .../tensilelite/tests/CuCount_test.cpp | 44 ++---- 6 files changed, 96 insertions(+), 174 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index d9a8c89d5db3..f60cbad7eb20 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -385,53 +385,30 @@ def _skv(self, writer, name): # ------------------------------------------------------------------ # Single-hop next-neighbor work stealing (codegen-time, off by default) # - # Topology: the dynamic-queue fetch (auto-mode SK4 / SK5-dynamic) uses 8 - # hardcoded queues in a wraparound next-neighbor order. queueIdx = StreamKIdx - # & 0x7; per-queue counters live one per cache line at AddressFlags + - # (queueIdx << 8); the global tile index is (perQueueRaw << 3) + queueIdx. + # Queues are one per XCD: numQueues = writer.states.archCaps["NumXCD"], a + # power of two so the mapping can use shift/AND fast masking: + # queueIdx = StreamKIdx & (numQueues - 1) + # counter addr = AddressFlags + (queueIdx << _WS_CACHE_LINE_LOG2) + # global index = (perQueueRaw << log2(numQueues)) + queueIdx # - # Every queue q steals from its immediate next neighbor s = (q+1) & 0x7 once - # its own home queue drains (there is no remainder/structural-extra guard: an - # empty home fetch always attempts one steal from that single next neighbor). - # A per-WG STICKY-HOME flag (StreamKStickyEmpty) makes each WG touch its home - # counter for its valid dispenses plus EXACTLY ONE empty fetch, then only - # steal thereafter. Because the extra atomics against a queue are now a static - # function of the next-neighbor topology (each queue is stolen from by exactly - # one predecessor p = (q-1) & 0x7), every counter self-resets to 0 via a per-queue auto-reset - # bound that folds in the predecessor's workgroup count. No explicit - # end-of-kernel reset is needed: the Synchronizer, host-zeroed once at handle - # creation, is left zeroed for the next (back-to-back) launch by the bounds - # alone. + # A queue whose home fetch comes up empty makes a single-hop next-neighbor + # steal from s = (q + 1) & (numQueues - 1). Each queue is stolen from by + # exactly one predecessor p = (q - 1) & (numQueues - 1), so its atomic_inc + # auto-reset bound is a static predecessor-inclusive value that self-zeroes + # the counter each launch -- no explicit end-of-kernel reset is needed. # - # AddressFlags buffer layout (per problem), for reference: - # [0x000 .. 0x800) 8 queue counters, one per 256B cache line - # (only the first word of each line is used). - # [0x800 .. ) partials/fixup ready flags (one word per partial tile, - # see storeBranches: offset = (partialIdx << 2) + 256*8). - # Per-architecture work-stealing / dynamic-queue count. + # AddressFlags buffer layout (per problem): + # [0, numQueues*256) per-queue counters, one per 256B cache line + # [numQueues*256, ...) partials/fixup ready flags (one word per tile, + # offset = (partialIdx << 2) + numQueues*256). # - # The dynamic-queue fetch partitions work across one queue per XCD and maps - # StreamKIdx -> queueIdx (StreamKIdx & (numQueues-1)) and queueIdx -> cache - # line (queueIdx << log2(256)) with shift/AND fast masking. That masking is - # only valid when the queue count is a power of two, so the count comes from - # the per-arch capability instead of hardcoding a literal 8. - # - # The count is read at codegen time from ``writer.states.archCaps["NumXCD"]`` - # (see rocisa initArchCaps), which MIRRORS origami's per-arch XCD count - # (origami::hardware_t::get_default_num_xcds): one queue per XCD. Codegen - # cannot import origami (C++), so the cap is the single codegen-side mirror - # of that value; the host (ContractionSolution.cpp streamKBakedQueueCount) - # reads the SAME origami value, keeping the two in lockstep. The host guard - # additionally enforces that the device's RUNTIME NUM_XCD equals this baked - # per-XCD queue count (and is a power of two) before allowing the - # dynamic-queue path. - # - # NOTE: MI300A and MI300X BOTH report gfx942; only the physical XCD count - # differs (MI300A=6, MI300X=8). Codegen cannot tell them apart, so gfx942 is - # generated for 8 queues (origami's gfx942 default) and the host runtime - # rejects the dynamic-queue / work-stealing path when the device's runtime - # NUM_XCD does not equal that baked count (e.g. MI300A's 6 XCDs). - _WS_CACHE_LINE_LOG2 = 8 # log2(256): per-queue counters are one per 256B line + # numQueues is read at codegen time from writer.states.archCaps["NumXCD"] + # (the codegen mirror of origami get_default_num_xcds). The host + # (ContractionSolution.cpp) reads the same origami value and additionally + # rejects the dynamic-queue path when the device's runtime NUM_XCD differs + # from this baked count (e.g. MI300A's 6 XCDs vs the gfx942-baked 8). + _WS_CACHE_LINE_BYTES = 256 # per-queue counter stride: one cache line + _WS_CACHE_LINE_LOG2 = log2(_WS_CACHE_LINE_BYTES) # log2 from ..Common -> int 8 def _wsQueueConstants(self, writer, kernel): """Return (numQueues, mask, log2Queues, cacheLineLog2) for this arch. @@ -467,24 +444,19 @@ def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, sGrid): """Fold the predecessor's workgroup count into the home auto-reset bound. - The non-stealing dynamic path relies on the atomic_inc auto-reset bound - ``tiles_q + W_q - 1`` so that after exactly ``tiles_q + W_q`` fetches (all - tiles dispensed + one empty per WG) the counter wraps back to 0. Under - single-hop next-neighbor stealing every queue q is also stolen from by EXACTLY ONE - predecessor p = (q-1) & (numQueues-1): once each of p's W_p workgroups - goes sticky-empty it makes one atomic against q per persistent iteration, - contributing W_p additional increments to q's counter. The self-resetting - bound therefore becomes ``tiles_q + W_q + W_p - 1``. This adds the W_p - term (W_p = (skGrid >> log2) + [p < (skGrid & mask)]) to the already - computed ``tiles_q + W_q - 1`` in ``sBound``. Caller must gate on - kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR - name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). ``sQueueIdx`` - is preserved. - - Precondition (see Solution validation): the fold is exact only when - W_q >= 1 whenever tiles_q >= 1, i.e. the grid assigns at least one - workgroup per queue (skGrid >= numQueues); the Solution layer rejects - debug overrides that could violate this. + The non-stealing bound ``tiles_q + W_q - 1`` wraps queue q's counter back + to 0 after ``tiles_q + W_q`` fetches (all tiles + one empty per WG). With + stealing q also absorbs W_p increments from its single predecessor + p = (q-1) & (numQueues-1), so the bound becomes ``tiles_q + W_q + W_p - 1``. + This adds the W_p term (W_p = (skGrid >> log2) + [p < (skGrid & mask)]) to + the ``tiles_q + W_q - 1`` already in ``sBound``. Caller must gate on + kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR name + (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). ``sQueueIdx`` is + preserved. + + Precondition (see Solution validation): exact only when W_q >= 1 whenever + tiles_q >= 1, i.e. skGrid >= numQueues; the Solution layer rejects debug + overrides that could violate this. """ _, mask, log2Queues, _ = self._wsQueueConstants(writer, kernel) sPred = writer.sgprPool.checkOut(1, "wsPredQueue") @@ -505,24 +477,18 @@ def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, s def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, sGrid, mkLabel): """Single-hop next-neighbor steal on the per-XCD queue topology. - On entry sQueueIdx holds the home queue index q and sWorkItemIdx holds - the home fetch result; both must be live. If the home fetch was valid - (index < TotalItems) this is a no-op. Otherwise the WG ALWAYS steals one - tile from its immediate next neighbor s = (q+1) & (numQueues-1): a single - s_atomic_inc against the neighbor's counter, then the global tile index - is recomputed from s. There is NO remainder / structural-extra guard -- - an empty home always attempts exactly one steal. A lost race leaves - sWorkItemIdx >= TotalItems so the downstream valid-index check turns this - WG into a no-op. sQueueIdx is clobbered (advanced to s). Caller must gate - on kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR - name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). - - The steal atomic runs with a STATIC auto-reset bound - ``tiles_s + W_s + W_q - 1`` (rather than a disabled 0xFFFFFFFF bound): - s is stolen from by exactly its predecessor q, so this is the same - predecessor-inclusive self-reset used on the home path, evaluated for the - stolen queue. This lets the neighbor counter wrap back to 0 on its own, - removing the need for an explicit end-of-kernel reset. + Register contract: on entry sQueueIdx holds the home queue index q and + sWorkItemIdx holds the home fetch result; both must be live. If the home + fetch was valid (index < TotalItems) this is a no-op. Otherwise the WG + steals one tile from its next neighbor s = (q+1) & (numQueues-1) via a + single s_atomic_inc, then recomputes the global tile index from s. A lost + race leaves sWorkItemIdx >= TotalItems so the downstream valid-index check + turns this WG into a no-op. sQueueIdx is clobbered (advanced to s). Caller + must gate on kernel["StreamKWorkStealing"] and pass the mode-appropriate + grid SGPR name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). + + The steal atomic uses the same predecessor-inclusive static bound as the + home path, evaluated for the stolen queue: ``tiles_s + W_s + W_q - 1``. """ _, mask, log2Queues, cacheLineLog2 = self._wsQueueConstants(writer, kernel) skFetchDone = mkLabel("SK_FetchDone") diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 1b2ea4572130..1a4d3fb982e2 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1663,31 +1663,23 @@ def assignDerivedParameters( "DebugPersistentKernelLoopForever requires StreamK=3 (got %d)" % state["StreamK"]) if state["StreamKWorkStealing"]: - # NOTE: these are codegen-time rejections only; there is no hardware - # context here (MI300A and MI300X both compile as gfx942). The kernel - # bakes in a power-of-two per-XCD queue count (8, mirroring origami's - # per-arch XCD count), so the check that the device actually matches it - # -- the host now requires the device's RUNTIME NUM_XCD to EQUAL the - # baked per-XCD queue count (derived from origami), not merely be a power - # of two, and excludes the dynamic-queue / work-stealing path otherwise - # (e.g. MI300A's 6 XCDs, or a power-of-two-but-mismatched partition); a - # non-work-stealing solution serves the GEMM instead and the user is - # warned -- lives host-side in ContractionSolution.cpp - # (streamKDynamicQueueSupported / streamKDynamicQueueUnsupported / - # streamKBakedQueueCount, wired into softwarePredicate). - # Work stealing only exists in the dynamic-queue fetch (auto-mode - # SK4 and the SK4 sub-path of SK5). + # Codegen-time rejections only; there is no hardware context here + # (MI300A/MI300X both compile as gfx942). The kernel bakes a power-of-two + # per-XCD queue count from origami; the host (ContractionSolution.cpp) + # enforces the device's runtime NUM_XCD against that baked count and + # otherwise serves a non-work-stealing solution. Work stealing only exists + # in the dynamic-queue fetch (auto-mode SK4 and the SK4 sub-path of SK5). if state["StreamK"] not in (4, 5): reject(state, printRejectionReason, "StreamKWorkStealing requires StreamK in {4,5} (got %d)" % state["StreamK"]) - # Stealing was designed/validated against the non-atomic - # partials+fixup path. Atomic SK4/SK5 is already rejected above; keep - # this explicit guard so the combination can never slip through. + # Stealing is only defined for the non-atomic partials+fixup path. + # Atomic SK4/SK5 is already rejected above; keep this explicit guard so + # the combination can never slip through. if state["StreamKAtomic"]: reject(state, printRejectionReason, "StreamKWorkStealing is not supported with StreamKAtomic") - # Auto-reset precondition (proof condition #1): the per-queue atomic_inc + # Auto-reset precondition: the per-queue atomic_inc # auto-reset bound is now predecessor-inclusive (home bound = # tiles_q + W_q + W_(q-1) - 1, steal bound = tiles_s + W_s + W_q - 1). # It self-resets each launch ONLY while every queue that owns tiles also diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index 6c90c64c07c7..cc6ed920ab91 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -2,23 +2,23 @@ # SPDX-License-Identifier: MIT """Unit tests for the single-hop next-neighbor StreamK work-stealing codegen. -These tests assert that the new work-stealing assembly is emitted by the -helper methods on the ``StreamK`` base class, and -- crucially -- that those -helpers are only ever reached behind the codegen-time ``StreamKWorkStealing`` -toggle. Following the StreamK=5 hybrid tests, they import rocisa instructions -and inspect emitted modules rather than matching source text; the toggle gating +These tests assert that the work-stealing assembly is emitted by the helper +methods on the ``StreamK`` base class, and -- crucially -- that those helpers +are only ever reached behind the codegen-time ``StreamKWorkStealing`` toggle. +Following the StreamK=5 hybrid tests, they import rocisa instructions and +inspect emitted modules rather than matching source text; the toggle gating and the Solution-level validation are verified by executing the *real* source (via the AST) so the assertions track the actual code, not a copy of it. -New emission contract (single-hop next-neighbor + sticky-home + static auto-reset): - * The steal always fires on an empty home fetch -- there is NO remainder / +Emission contract (single-hop next-neighbor + sticky-home + static auto-reset): + * The steal always fires on an empty home fetch -- there is no remainder / structural-extra guard (no ``s_cmp_ge_u32`` neighbor guard, no ``remainder == 0`` skip). * The steal & home atomic bounds are the predecessor-inclusive self-reset - value, NOT the old 0xFFFFFFFF "disable auto-reset" sentinel. + value. * A per-WG sticky-empty SGPR gates the home fetch. - * There is NO ``streamKWorkStealingKernelEndReset`` / completion counter / - reset barrier anymore. + * There is no ``streamKWorkStealingKernelEndReset`` / completion counter / + reset barrier. """ import ast diff --git a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp index d5165c860c34..9dbc83c43b09 100644 --- a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp +++ b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp @@ -601,11 +601,13 @@ inline std::map initArchCaps(const IsaVersion& isaVersion) rv["LDSBankCount"] = 64; rv["LDSBankWidth"] = 4; // bytes per bank - // Per-XCD work-queue count for StreamK dynamic-queue kernels. Single - // codegen-side mirror of origami get_default_num_xcds(): gfx942/gfx950 are - // 8-XCD, every other arch is 1. Power-of-two so StreamK queue masking (AND/ - // shift) stays valid; the host guard rejects devices whose runtime NUM_XCD - // != this value. + // Per-XCD work-queue count baked into StreamK dynamic-queue kernels. Single + // codegen-side mirror of origami get_default_num_xcds(): gfx942/gfx950 bake + // 8 (the MI300X value), every other arch 1. gfx942 covers BOTH MI300X (8 + // XCDs) and MI300A (6 XCDs), which codegen cannot tell apart, so it always + // bakes 8; the host guard rejects a device whose runtime NUM_XCD != this + // baked value (so MI300A's 6 is excluded at runtime). Power-of-two keeps the + // StreamK queue masking (AND/shift) valid. rv["NumXCD"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 8 : 1; return rv; diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 7e53618eb2f4..1414468e28fc 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -60,20 +60,12 @@ namespace TensileLite namespace { // The dynamic-queue StreamK kernels (SK4 and the SK4 sub-path of SK5) - // bake in a fixed, power-of-two per-XCD queue count for fast index - // masking (StreamKIdx & (Q-1), queueIdx << log2(256)). That baked count - // is the per-arch XCD count: codegen keys it off the ISA (see StreamK.py - // _WS_NUM_QUEUES_BY_ISA, which MIRRORS origami get_default_num_xcds) and - // emits Q = 8 for gfx942/gfx950. The host reads the SAME origami value - // here -- via the device's origami architecture -- so the per-XCD - // work-queue region reserved in the Synchronizer workspace and the - // runtime acceptance guard stay in lockstep with codegen instead of - // duplicating a hardcoded literal 8. - // - // Returns the baked per-XCD queue count for the device's architecture, - // or 0 when the architecture cannot be determined (no analytical - // hardware, or an architecture origami has no default XCD count for), - // which the guard below treats as unsupported. + // bake a fixed power-of-two per-XCD queue count for fast index masking. + // Codegen derives it from the arch's XCD count (StreamK.py + // _wsQueueConstants / archCaps["NumXCD"], mirroring origami + // get_default_num_xcds); the host reads the SAME origami value here so + // codegen and the runtime guard stay in lockstep. Returns 0 when the + // architecture cannot be determined (guard treats that as unsupported). inline size_t streamKBakedQueueCount(Hardware const& hardware) { auto const* hipAMDGPU = dynamic_cast(&hardware); @@ -95,20 +87,12 @@ namespace TensileLite } // The dynamic-queue fetch / work stealing is only correct when the - // device's runtime XCD count matches the baked per-XCD queue count AND - // is a power of two (the kernel masks with (Q-1)). MI300A and MI300X - // BOTH report gfx942 -- codegen bakes Q = 8 -- but MI300A has 6 XCDs - // (not a power of two), so the fast-mask assumption breaks there. - // - // Returns true (UNSUPPORTED / reject) when the runtime NUM_XCD is 0 - // (defensive), is NOT a power of two, the baked count is unknown, or the - // runtime NUM_XCD does not equal the baked count. The last condition - // additionally closes the partition/CPX gap where NUM_XCD is a power of - // two but != baked (e.g. a 4-XCD partition of an 8-XCD gfx942), which - // would still mis-map the fixed Q=8 queue masking. Returns false when - // there is no analytical hardware so historic behavior is preserved - // (unknown XCD -> allow). The classification is intentionally isolated - // here so it stays trivially unit-testable (see CuCount_test.cpp). + // device's runtime NUM_XCD is a power of two AND equals the baked + // per-XCD queue count. Returns true (UNSUPPORTED) when NUM_XCD is 0, not + // a power of two, the baked count is unknown, or NUM_XCD != baked (e.g. + // MI300A's 6 XCDs, or a 4-XCD partition of an 8-XCD gfx942). Returns + // false when there is no analytical hardware (unknown XCD -> allow). Kept + // isolated here so it stays trivially unit-testable (see CuCount_test.cpp). inline bool streamKDynamicQueueUnsupported(Hardware const& hardware) { auto const* hipAMDGPU = dynamic_cast(&hardware); diff --git a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp index f63f3d414f69..213f5e60b000 100644 --- a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp @@ -742,39 +742,17 @@ TEST(Sk3Sk5OffPartition512Test, NativeSk3MatchesSk5OffHostPack) // =========================================================================== // StreamKDynamicQueueXcdGateTest -- MI300A (NUM_XCD=6) reject-and-continue. // -// The SK4 / SK5-dynamic work-stealing kernels bake in a fixed, power-of-two -// per-XCD queue count (origami's per-arch XCD count: 8 for gfx942/gfx950) and -// mask queue/tile indices with (Q-1). That fast masking is only valid when the -// device's runtime XCD count equals the baked count AND is a power of two. -// MI300A and MI300X both report gfx942 (baked count 8), but MI300A has 6 XCDs -// (not a power of two). A power-of-two-but-mismatched partition (e.g. a 4-XCD -// slice of an 8-XCD gfx942) is likewise rejected because runtime NUM_XCD != 8. -// -// Rather than SILENTLY degrading such a solution to tree reduction, the host -// now EXCLUDES the dynamic-queue / work-stealing solution from selection and -// warns the user once, so a different (SK3-static / non-StreamK) solution -// serves the GEMM. This is enforced by ContractionSolution:: -// streamKDynamicQueueSupported(problem, hardware), which is wired into -// softwarePredicate() (SolutionLibrary.hpp) so returning false drops the -// solution from findBestSolution/findAllSolutions. It builds on the file-local -// numeric predicate streamKDynamicQueueUnsupported(hardware): dynamic_cast to -// hip::HipAMDGPU, derive the baked per-XCD queue count from the origami arch -// (streamKBakedQueueCount / get_default_num_xcds), read analyticalHardware-> -// NUM_XCD, and return true (reject) when NUM_XCD is 0 (defensive), not a power -// of two, the baked count is unknown, or NUM_XCD != baked; false otherwise -// (incl. unknown/no analytical hardware -> historic behavior preserved). -// -// Both production predicates live in anonymous namespaces / a .cpp translation -// unit and cannot be linked from here, so -- following the same convention as -// computeStreamKHostPack above (which mirrors solve()'s internal reduction -// decision for host-only testing) -- this test mirrors them and drives them -// through a real hip::HipAMDGPU mock. It validates (a) the Hardware& -> -// HipAMDGPU -> analyticalHardware -> {arch, NUM_XCD} accessor chain, (b) the -// power-of-two AND equals-baked classification (6 -> reject, 4 -> reject -// (pow2 but != baked 8), 8 -> allow, unknown -> allow), and (c) the -// selection-predicate composition (dynamic-queue on a mismatched XCD count -> -// excluded; other StreamK modes / queue-count-agnostic solutions -> -// selectable). It is NOT executed on real MI300A silicon. +// SK4 / SK5-dynamic work-stealing kernels bake a fixed power-of-two per-XCD +// queue count (8 for gfx942/gfx950) and mask indices with (Q-1), so they are +// valid only when the device's runtime NUM_XCD equals that baked count. MI300A +// reports gfx942 (baked 8) but has 6 XCDs; a mismatched partition (e.g. a 4-XCD +// slice of an 8-XCD gfx942) is likewise rejected. The host excludes such a +// solution from selection (streamKDynamicQueueSupported wired into +// softwarePredicate) and warns once instead of silently degrading. The +// production predicates live in a .cpp anonymous namespace, so -- like +// computeStreamKHostPack above -- this test mirrors them over a hip::HipAMDGPU +// mock (6 -> reject, 4 -> reject, 8 -> allow, unknown -> allow). Not run on +// real MI300A silicon. // =========================================================================== namespace { From f64b3fe28cf606ed512712a486e28f516f4042cf Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 13 Jul 2026 23:33:08 +0000 Subject: [PATCH 14/20] refactor(tensilelite): set StreamK queue stride to the origami cache-line size Derive the per-queue counter stride from the hardware cache-line size (128 B for gfx942/gfx950) via a new origami accessor (hardware_t::get_default_cache_line_bytes), mirrored into a rocisa CacheLineBytes arch-cap. Stride == cache-line size so each per-XCD atomic counter occupies its own line (no false sharing), replacing the hardcoded 256 B over-alignment. Halves the per-queue stride/workspace; queue-index math (count/mask/log2 from NumXCD) is unchanged. --- .../tensilelite/Tensile/Components/StreamK.py | 67 +++++++++++++------ .../Tests/unit/test_streamk_work_stealing.py | 20 +++--- .../rocisa/rocisa/include/hardware_caps.hpp | 7 ++ .../tensilelite/src/ContractionSolution.cpp | 48 ++++++++++--- .../tensilelite/tests/CuCount_test.cpp | 7 +- shared/origami/include/origami/hardware.hpp | 18 +++++ shared/origami/src/origami/hardware.cpp | 11 +++ 7 files changed, 137 insertions(+), 41 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index f60cbad7eb20..19c43722dfeb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -388,7 +388,7 @@ def _skv(self, writer, name): # Queues are one per XCD: numQueues = writer.states.archCaps["NumXCD"], a # power of two so the mapping can use shift/AND fast masking: # queueIdx = StreamKIdx & (numQueues - 1) - # counter addr = AddressFlags + (queueIdx << _WS_CACHE_LINE_LOG2) + # counter addr = AddressFlags + (queueIdx << cacheLineLog2) # global index = (perQueueRaw << log2(numQueues)) + queueIdx # # A queue whose home fetch comes up empty makes a single-hop next-neighbor @@ -397,35 +397,62 @@ def _skv(self, writer, name): # auto-reset bound is a static predecessor-inclusive value that self-zeroes # the counter each launch -- no explicit end-of-kernel reset is needed. # + # The per-queue counter stride is the hardware L2 cache-line size (read from + # writer.states.archCaps["CacheLineBytes"], the codegen mirror of origami + # hardware_t::get_default_cache_line_bytes; 128B for gfx942/gfx950). Stride + # == cache-line size, so each per-XCD atomic counter occupies its own line + # -> no false sharing. + # # AddressFlags buffer layout (per problem): - # [0, numQueues*256) per-queue counters, one per 256B cache line - # [numQueues*256, ...) partials/fixup ready flags (one word per tile, - # offset = (partialIdx << 2) + numQueues*256). + # [0, numQueues*stride) per-queue counters, one per cache line + # [numQueues*stride, ...) partials/fixup ready flags (one word per tile, + # offset = (partialIdx << 2) + numQueues*stride). # # numQueues is read at codegen time from writer.states.archCaps["NumXCD"] # (the codegen mirror of origami get_default_num_xcds). The host # (ContractionSolution.cpp) reads the same origami value and additionally # rejects the dynamic-queue path when the device's runtime NUM_XCD differs # from this baked count (e.g. MI300A's 6 XCDs vs the gfx942-baked 8). - _WS_CACHE_LINE_BYTES = 256 # per-queue counter stride: one cache line - _WS_CACHE_LINE_LOG2 = log2(_WS_CACHE_LINE_BYTES) # log2 from ..Common -> int 8 def _wsQueueConstants(self, writer, kernel): """Return (numQueues, mask, log2Queues, cacheLineLog2) for this arch. Centralizes the dynamic-queue / work-stealing fast-mask constants so the - queue count lives in exactly one place. ``numQueues`` is read from the - per-arch capability ``writer.states.archCaps["NumXCD"]`` (the codegen - mirror of origami get_default_num_xcds) and must be a power of two for - the shift/AND masking to be valid; the host guards the same assumption - at runtime, rejecting devices whose runtime NUM_XCD is not a power of two - or does not equal this baked count (e.g. MI300A's 6 XCDs). + queue count and per-queue counter stride each live in exactly one place. + ``numQueues`` is read from ``writer.states.archCaps["NumXCD"]`` (the + codegen mirror of origami get_default_num_xcds) and must be a power of + two for the shift/AND masking to be valid; the host guards the same + assumption at runtime, rejecting devices whose runtime NUM_XCD is not a + power of two or does not equal this baked count (e.g. MI300A's 6 XCDs). + + ``cacheLineLog2`` is log2 of the per-queue counter stride, which equals + the hardware L2 cache-line size ``writer.states.archCaps["CacheLineBytes"]`` + (the codegen mirror of origami get_default_cache_line_bytes; 128B for + gfx942/gfx950). Stride == cache-line size so each per-XCD counter sits on + its own line (no false sharing); it must be a power of two for the + queue-address shift to be valid. """ numQueues = writer.states.archCaps["NumXCD"] assert numQueues > 0 and (numQueues & (numQueues - 1)) == 0, ( "StreamK dynamic-queue fast masking requires a power-of-two queue count " "(got %d for ISA %s)" % (numQueues, tuple(kernel["ISA"][:2]))) - return numQueues, numQueues - 1, log2(numQueues), self._WS_CACHE_LINE_LOG2 + strideBytes = writer.states.archCaps["CacheLineBytes"] + assert strideBytes > 0 and (strideBytes & (strideBytes - 1)) == 0, ( + "StreamK per-queue counter stride must be a power-of-two cache-line " + "size (got %d for ISA %s)" % (strideBytes, tuple(kernel["ISA"][:2]))) + return numQueues, numQueues - 1, log2(numQueues), log2(strideBytes) + + def _wsFlagsBaseOffset(self, writer, kernel): + """Byte offset where the partials/fixup ready flags begin. + + The per-queue counter region occupies ``numQueues`` cache lines + (``numQueues * strideBytes`` bytes, with stride == the cache-line size), + so the flags region starts right after it. Both terms come from + ``_wsQueueConstants`` (numQueues and log2(stride)); for gfx942/gfx950 + this is 8 * 128 = 1024. + """ + numQueues, _, _, cacheLineLog2 = self._wsQueueConstants(writer, kernel) + return numQueues << cacheLineLog2 def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, comment): """Emit sDst = (sTotal >> log2Queues) + [sQueue < (sTotal & mask)]. @@ -1505,7 +1532,7 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, # TODO modularize this section into abstract function module.add(self.calculatePartialIdx(tmpSgpr)) module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(tmpSgpr), shiftHex=log2(4), comment="flag offset based on partial index")) - module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=(256*8), comment="Offset flags to come after the work queues")) + module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=self._wsFlagsBaseOffset(writer, kernel), comment="Offset flags to come after the work queues")) elif kernel["StreamK"] == 5: # SK5 hybrid: dispatch on StreamKHybridMode bit # (0 = static SK3 -> use StreamKIdx, 1 = dynamic SK4 -> use calculatePartialIdx). @@ -1519,7 +1546,7 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(self.calculatePartialIdx(tmpSgpr)) module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(tmpSgpr), shiftHex=log2(4), comment="SK5/SK4: flag offset based on partial index")) - module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=(256*8), + module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=self._wsFlagsBaseOffset(writer, kernel), comment="SK5/SK4: offset flags to come after the work queues")) module.add(SBranch(labelName=sk5FlagDone.getLabelName(), comment="SK5: skip static flag offset")) @@ -3103,7 +3130,8 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): writer.sgprPool.checkIn(sWave) # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for the - # StreamKIdx/queue divisions, log2(256) for the cache-line stride). + # StreamKIdx/queue divisions, log2(cache-line size) for the per-queue + # counter stride == one cache line, no false sharing). _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index @@ -3374,7 +3402,7 @@ def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, # Check flag module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sPartialIdx), shiftHex=log2(4), comment="flag offset based on partial index")) - module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=(256*8), comment="Offset flags to come after the work queues")) + module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=self._wsFlagsBaseOffset(writer, kernel), comment="Offset flags to come after the work queues")) module.add(SLoadB32(dst=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=True, dlc=True, scope=CacheScope.SCOPE_DEV), comment="get flag")) module.add(SWaitCnt(kmcnt=0, comment="wait for flag load")) @@ -3761,7 +3789,8 @@ def emitDynamicGRA(mod): writer.sgprPool.checkIn(sWave) # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for - # the StreamKIdx/queue divisions, log2(256) for cache-line stride). + # the StreamKIdx/queue divisions, log2(cache-line size) for the + # per-queue counter stride == one cache line, no false sharing). _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index @@ -4198,7 +4227,7 @@ def emitDynamicStore(mod): mod.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sPartialIdx), shiftHex=log2(4), comment="flag offset based on partial index")) - mod.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=(256*8), + mod.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=self._wsFlagsBaseOffset(writer, kernel), comment="Offset flags to come after the work queues")) mod.add(SLoadB32(dst=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index cc6ed920ab91..6fc44098b264 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -84,11 +84,14 @@ def checkIn(self, *args, **kwargs): class _FakeWriter: - def __init__(self, numXCD: int = 8): + def __init__(self, numXCD: int = 8, cacheLineBytes: int = 128): self.sgprPool = _FakeSgprPool() - # gfx942/gfx950 mirror origami get_default_num_xcds == 8; the helpers - # read the per-arch queue count from writer.states.archCaps["NumXCD"]. - self.states = types.SimpleNamespace(archCaps={"NumXCD": numXCD}) + # gfx942/gfx950 mirror origami get_default_num_xcds == 8 and origami + # get_default_cache_line_bytes == 128; the helpers read the per-arch + # queue count from writer.states.archCaps["NumXCD"] and the per-queue + # counter stride from writer.states.archCaps["CacheLineBytes"]. + self.states = types.SimpleNamespace( + archCaps={"NumXCD": numXCD, "CacheLineBytes": cacheLineBytes}) def _mk_label(base: str) -> Label: @@ -423,10 +426,11 @@ class TestQueueConstants: @pytest.mark.parametrize("isa", [(9, 4, 0), (9, 5, 0)]) def test_supported_arches_use_eight_power_of_two_queues(self, isa): sk = _stream_k_instance(4) - # gfx942/gfx950 mirror origami get_default_num_xcds == 8, so the - # constants tuple is exactly (numQueues=8, mask=7, log2=3, cacheLog2=8). - writer = _FakeWriter(numXCD=8) - assert sk._wsQueueConstants(writer, {"ISA": isa}) == (8, 7, 3, 8) + # gfx942/gfx950 mirror origami get_default_num_xcds == 8 and + # get_default_cache_line_bytes == 128, so the constants tuple is exactly + # (numQueues=8, mask=7, log2=3, cacheLineLog2=log2(128)=7). + writer = _FakeWriter(numXCD=8, cacheLineBytes=128) + assert sk._wsQueueConstants(writer, {"ISA": isa}) == (8, 7, 3, 7) def test_non_power_of_two_queue_count_asserts(self): # The shift/AND fast masking is only valid for a power-of-two queue diff --git a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp index 9dbc83c43b09..3f281bc19c6e 100644 --- a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp +++ b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp @@ -610,6 +610,13 @@ inline std::map initArchCaps(const IsaVersion& isaVersion) // StreamK queue masking (AND/shift) valid. rv["NumXCD"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 8 : 1; + // codegen-side mirror of origami's L2 cache-line size + // (hardware_t::get_default_cache_line_bytes): 128 B for the dynamic-queue + // arches gfx942/gfx950. Power-of-two so the StreamK queue-address shift + // stays valid; the StreamK per-XCD counter stride is set equal to this + // cache-line size so each counter occupies its own line (no false sharing). + rv["CacheLineBytes"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 128 : 128; + return rv; } diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 1414468e28fc..9ac35f1874c7 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -86,6 +86,24 @@ namespace TensileLite } } + // Per-XCD counter stride (bytes) for the dynamic-queue work-queue + // region. Set equal to the hardware L2 cache-line size so each per-XCD + // atomic counter occupies its own line (no false sharing). Sourced from + // origami (hardware_t::get_default_cache_line_bytes) -- the SAME value + // the codegen mirrors via rocisa archCaps["CacheLineBytes"] + // (StreamK.py _wsQueueConstants). Host (origami) and codegen (archCaps) + // strides are two mirrors of the one origami cache-line size, so the + // workspace the host reserves matches the layout the kernel addresses. + // Returns 0 when the architecture cannot be determined. + inline size_t streamKPerQueueStrideBytes(Hardware const& hardware) + { + auto const* hipAMDGPU = dynamic_cast(&hardware); + if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) + return 0; + return origami::hardware_t::get_default_cache_line_bytes( + hipAMDGPU->analyticalHardware->arch); + } + // The dynamic-queue fetch / work stealing is only correct when the // device's runtime NUM_XCD is a power of two AND equals the baked // per-XCD queue count. Returns true (UNSUPPORTED) when NUM_XCD is 0, not @@ -3263,12 +3281,16 @@ namespace TensileLite { size_t idealWorkspace = partialTileSize(sk.grid); // SK4 and SK5-dynamic need the per-XCD work-queue region; SK5-static - // sizes like standalone SK3. The region is sized to the kernel's - // baked per-XCD queue count (origami's per-arch XCD count); the - // acceptance guard requires runtime NUM_XCD == baked, so this - // equals 256 * NUM_XCD for every device that reaches here. + // sizes like standalone SK3. The region is sized as (per-queue + // stride) * (baked per-XCD queue count), both sourced from origami: + // the stride is the L2 cache-line size (get_default_cache_line_bytes, + // 128 B for gfx942/gfx950) so each counter owns its line (no false + // sharing), and the queue count is the per-arch XCD count. The + // acceptance guard requires runtime NUM_XCD == baked, so this equals + // cacheLineBytes * NUM_XCD for every device that reaches here. if(dynamicQueuePath) - idealWorkspace += 256 * streamKBakedQueueCount(hardware); + idealWorkspace + += streamKPerQueueStrideBytes(hardware) * streamKBakedQueueCount(hardware); // If given workspace is less than ideal, we can fall back to DP mode // Performance will likely be lower, but the kernel can run if workspace is unavailable. // (The non-power-of-two XCD case is handled earlier by explicit @@ -3699,14 +3721,18 @@ namespace TensileLite { size_t idealWorkspace = partialTileSize(skGrid); // Reserve the per-XCD work-queue region for the dynamic-queue - // path. Sized to the kernel's baked per-XCD queue count - // (origami's per-arch XCD count, e.g. 8 for gfx942/gfx950); - // this may slightly over-report on a device that falls back - // to tree reduction (e.g. MI300A), which is safe (never - // under-sized). + // path. Sized as (per-queue stride) * (baked per-XCD queue + // count), both from origami: the stride is the L2 cache-line + // size (get_default_cache_line_bytes, 128 B for gfx942/gfx950) + // so each counter owns its line (no false sharing), and the + // queue count is the per-arch XCD count (e.g. 8 for + // gfx942/gfx950). This may slightly over-report on a device + // that falls back to tree reduction (e.g. MI300A), which is + // safe (never under-sized). if(sizeMapping.streamK == 4 || (sizeMapping.streamK == 5 && effectiveDynamic)) - idealWorkspace += 256 * streamKBakedQueueCount(hardware); + idealWorkspace + += streamKPerQueueStrideBytes(hardware) * streamKBakedQueueCount(hardware); // If given workspace is less than ideal, we can fall back to DP mode // Performance will likely be lower, but the kernel can run if workspace is unavailable if(idealWorkspace <= problem.workspaceSize()) diff --git a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp index 213f5e60b000..b68541a315d3 100644 --- a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp @@ -631,8 +631,9 @@ TEST(StreamK5WorkspaceRegressionTest, QueryAndLaunchAgreeForDynamicMode) size_t ws = env.solution.requiredWorkspaceSize(problem, env.device); EXPECT_GT(ws, 0u) << "Dynamic mode with partial tiles must request workspace"; - // The workspace must be at least partialTileSize; the +2048 queue - // region is included by both query and launch so they agree. + // The workspace must be at least partialTileSize; the per-XCD work-queue + // region (cacheLineBytes * numXCD = 128 * 8 = 1024 B on gfx942/gfx950) is + // included by both query and launch so they agree. EXPECT_GE(ws, env.solution.partialTileSize(grid)) << "Workspace must cover at least partialTileSize(grid)"; } @@ -657,7 +658,7 @@ TEST(StreamK5WorkspaceRegressionTest, StaticModeOmitsQueueRegion) size_t ws = env.solution.requiredWorkspaceSize(problem, env.device); // Static (SK3) path does not use the work-queue, so workspace - // should be exactly partialTileSize — no +2048. + // should be exactly partialTileSize — no per-XCD work-queue region. EXPECT_EQ(ws, env.solution.partialTileSize(grid)) << "OFF workspace must equal partialTileSize(staticGrid)"; } diff --git a/shared/origami/include/origami/hardware.hpp b/shared/origami/include/origami/hardware.hpp index b02f257477f6..7056a28d13ec 100644 --- a/shared/origami/include/origami/hardware.hpp +++ b/shared/origami/include/origami/hardware.hpp @@ -756,6 +756,24 @@ class ORIGAMI_EXPORT hardware_t { */ static size_t get_default_num_xcds(architecture_t arch); + /** + * @brief Get the default L2 cache-line size (in bytes) for an architecture. + * + * Single source of truth for the hardware L2 cache-line size, consolidating + * the scattered 128-byte literals used across origami (streamk.cpp + * CACHE_LINE_BYTES, origami.cpp L2_CACHE_LINE_BYTES, + * heuristics.hpp EPILOGUE_CACHE_LINE_BYTES, formocast.cpp L2CacheLineSize). + * CDNA3 (gfx942) and CDNA4 (gfx950) both use a 128-byte L2 line (confirmed by + * AMD whitepapers); 128 is a safe default for the other supported + * architectures. Consumers that place one datum per cache line (e.g. the + * StreamK per-XCD atomic counters) use this so each occupies its own line and + * avoids false sharing. + * + * @param arch Architecture enum value + * @return L2 cache-line size in bytes + */ + static size_t get_default_cache_line_bytes(architecture_t arch); + /** * @brief Check if the hardware described by properties is supported. * diff --git a/shared/origami/src/origami/hardware.cpp b/shared/origami/src/origami/hardware.cpp index 31bdbd576a35..7fdc3be1313f 100644 --- a/shared/origami/src/origami/hardware.cpp +++ b/shared/origami/src/origami/hardware.cpp @@ -197,6 +197,17 @@ size_t hardware_t::get_default_num_xcds(architecture_t arch) { } } +size_t hardware_t::get_default_cache_line_bytes(architecture_t arch) { + // L2 cache-line size in bytes. Consolidates the 128-byte literals that were + // duplicated across origami. gfx942 (CDNA3) and gfx950 (CDNA4) use a 128-byte + // L2 line (AMD whitepapers); 128 is a safe default for the rest. + switch (arch) { + case architecture_t::gfx942: return 128; + case architecture_t::gfx950: return 128; + default: return 128; + } +} + void hardware_t::print() const { std::cout << "================== Hardware Configuration ==================\n"; std::cout << "Number of CUs (N_CU) : " << N_CU << "\n"; From e5934aa9bba211e01e973d25005304bfb469f523 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 14 Jul 2026 15:30:54 +0000 Subject: [PATCH 15/20] refactor(tensilelite): trim StreamK work-stealing comments, docs, and dead code Behavior-neutral cleanup from a multi-model code review; no host logic changes and emitted kernel assembly is byte-identical for gfx942/gfx950. - collapse dead CacheLineBytes ternary (rocisa hardware_caps.hpp) and the all-128 origami get_default_cache_line_bytes switch to a plain return 128 - correct the origami hardware.hpp and ValidParameters StreamKWorkStealing docstrings to match the shipped single-hop next-neighbor implementation - trim the over-verbose StreamK banner/helper docstrings, the kernelEnd reset comments, the Solution DebugStreamK reject comment, and DECISIONS D17 - remove session/history spill from StreamK comments and the streamk work-stealing YAML headers (no test params/problem sizes changed) - delete two ghost-guarding unit tests that asserted absence of symbols from an abandoned in-PR design; de-session-spill remaining test docstrings --- .../Tensile/Common/ValidParameters.py | 10 +- .../tensilelite/Tensile/Components/StreamK.py | 122 ++++++------------ .../Tensile/SolutionStructs/Solution.py | 14 +- .../streamk/sk_dynamic_work_stealing.yaml | 29 +---- .../streamk/sk_hybrid_work_stealing.yaml | 36 ++---- .../Tests/unit/characterization/DECISIONS.md | 32 +---- .../Tests/unit/test_streamk_work_stealing.py | 44 ++----- .../rocisa/rocisa/include/hardware_caps.hpp | 8 +- shared/origami/include/origami/hardware.hpp | 11 +- shared/origami/src/origami/hardware.cpp | 12 +- 10 files changed, 83 insertions(+), 235 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index e4655365af19..48ca693df14b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -817,11 +817,11 @@ def makeValidMatrixInstructions(): # 1: uses atomics to accumulate partial tiles "StreamKAtomic": [0, 1], # Codegen-time toggle for single-hop next-neighbor work stealing in the - # dynamic-queue StreamK fetch (auto-mode SK4 / SK5-dynamic, 8 hardcoded - # queues). When a workgroup's home queue is empty it makes ONE atomic - # attempt to claim the structural extra tile of the next queue. Valid only - # for StreamK in (4, 5). - # 0: off (emitted assembly is byte-for-byte the baseline) + # dynamic-queue StreamK fetch (SK4 / SK5-dynamic). Queue count = + # archCaps['NumXCD'] (8 on gfx942/gfx950). When a workgroup's home queue + # empties, it makes one atomic attempt on its next-neighbor per-XCD queue. + # Valid only for StreamK in (4, 5). + # 0: off # 1: on "StreamKWorkStealing": [0, 1], # Enables XCC-based remapping of workgroups, set the value to the number of XCCs diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 19c43722dfeb..ed1c40461b03 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -385,52 +385,25 @@ def _skv(self, writer, name): # ------------------------------------------------------------------ # Single-hop next-neighbor work stealing (codegen-time, off by default) # - # Queues are one per XCD: numQueues = writer.states.archCaps["NumXCD"], a - # power of two so the mapping can use shift/AND fast masking: - # queueIdx = StreamKIdx & (numQueues - 1) - # counter addr = AddressFlags + (queueIdx << cacheLineLog2) - # global index = (perQueueRaw << log2(numQueues)) + queueIdx - # - # A queue whose home fetch comes up empty makes a single-hop next-neighbor - # steal from s = (q + 1) & (numQueues - 1). Each queue is stolen from by - # exactly one predecessor p = (q - 1) & (numQueues - 1), so its atomic_inc - # auto-reset bound is a static predecessor-inclusive value that self-zeroes - # the counter each launch -- no explicit end-of-kernel reset is needed. - # - # The per-queue counter stride is the hardware L2 cache-line size (read from - # writer.states.archCaps["CacheLineBytes"], the codegen mirror of origami - # hardware_t::get_default_cache_line_bytes; 128B for gfx942/gfx950). Stride - # == cache-line size, so each per-XCD atomic counter occupies its own line - # -> no false sharing. + # Queues are one per XCD: numQueues = archCaps["NumXCD"], a power of two so + # queue mapping uses shift/AND fast masking (queueIdx = StreamKIdx & mask). + # A queue whose home fetch is empty steals once from its next neighbor + # s = (q+1) & mask; each queue has exactly one predecessor p = (q-1) & mask. # # AddressFlags buffer layout (per problem): # [0, numQueues*stride) per-queue counters, one per cache line - # [numQueues*stride, ...) partials/fixup ready flags (one word per tile, - # offset = (partialIdx << 2) + numQueues*stride). - # - # numQueues is read at codegen time from writer.states.archCaps["NumXCD"] - # (the codegen mirror of origami get_default_num_xcds). The host - # (ContractionSolution.cpp) reads the same origami value and additionally - # rejects the dynamic-queue path when the device's runtime NUM_XCD differs - # from this baked count (e.g. MI300A's 6 XCDs vs the gfx942-baked 8). + # [numQueues*stride, ...) partials/fixup ready flags (one word per tile) + # Counter stride == archCaps["CacheLineBytes"] (128B on gfx942/gfx950), so + # each per-XCD counter sits on its own line. Each counter's atomic_inc uses a + # static predecessor-inclusive auto-reset bound, so it self-zeroes every + # launch -- there is no explicit end-of-kernel reset. def _wsQueueConstants(self, writer, kernel): """Return (numQueues, mask, log2Queues, cacheLineLog2) for this arch. - Centralizes the dynamic-queue / work-stealing fast-mask constants so the - queue count and per-queue counter stride each live in exactly one place. - ``numQueues`` is read from ``writer.states.archCaps["NumXCD"]`` (the - codegen mirror of origami get_default_num_xcds) and must be a power of - two for the shift/AND masking to be valid; the host guards the same - assumption at runtime, rejecting devices whose runtime NUM_XCD is not a - power of two or does not equal this baked count (e.g. MI300A's 6 XCDs). - - ``cacheLineLog2`` is log2 of the per-queue counter stride, which equals - the hardware L2 cache-line size ``writer.states.archCaps["CacheLineBytes"]`` - (the codegen mirror of origami get_default_cache_line_bytes; 128B for - gfx942/gfx950). Stride == cache-line size so each per-XCD counter sits on - its own line (no false sharing); it must be a power of two for the - queue-address shift to be valid. + ``numQueues`` = archCaps["NumXCD"] and the counter stride = + archCaps["CacheLineBytes"]; both must be powers of two for the shift/AND + queue masking and queue-address shift to be valid (asserted below). """ numQueues = writer.states.archCaps["NumXCD"] assert numQueues > 0 and (numQueues & (numQueues - 1)) == 0, ( @@ -445,11 +418,8 @@ def _wsQueueConstants(self, writer, kernel): def _wsFlagsBaseOffset(self, writer, kernel): """Byte offset where the partials/fixup ready flags begin. - The per-queue counter region occupies ``numQueues`` cache lines - (``numQueues * strideBytes`` bytes, with stride == the cache-line size), - so the flags region starts right after it. Both terms come from - ``_wsQueueConstants`` (numQueues and log2(stride)); for gfx942/gfx950 - this is 8 * 128 = 1024. + The flags region starts right after the per-queue counters, i.e. after + ``numQueues * strideBytes`` bytes (8 * 128 = 1024 on gfx942/gfx950). """ numQueues, _, _, cacheLineLog2 = self._wsQueueConstants(writer, kernel) return numQueues << cacheLineLog2 @@ -471,19 +441,13 @@ def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, sGrid): """Fold the predecessor's workgroup count into the home auto-reset bound. - The non-stealing bound ``tiles_q + W_q - 1`` wraps queue q's counter back - to 0 after ``tiles_q + W_q`` fetches (all tiles + one empty per WG). With - stealing q also absorbs W_p increments from its single predecessor - p = (q-1) & (numQueues-1), so the bound becomes ``tiles_q + W_q + W_p - 1``. - This adds the W_p term (W_p = (skGrid >> log2) + [p < (skGrid & mask)]) to - the ``tiles_q + W_q - 1`` already in ``sBound``. Caller must gate on - kernel["StreamKWorkStealing"] and pass the mode-appropriate grid SGPR name - (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). ``sQueueIdx`` is - preserved. - - Precondition (see Solution validation): exact only when W_q >= 1 whenever - tiles_q >= 1, i.e. skGrid >= numQueues; the Solution layer rejects debug - overrides that could violate this. + Adds the predecessor term W_p to the ``tiles_q + W_q - 1`` already in + ``sBound``, giving the stealing bound ``tiles_q + W_q + W_p - 1`` (queue q + also absorbs W_p increments from its one predecessor p = (q-1) & mask). + Caller gates on kernel["StreamKWorkStealing"] and passes the grid SGPR + name ("skGrid" for SK4, "SKGrid" for SK5-dynamic); ``sQueueIdx`` is + preserved. Exact only when W_q >= 1 whenever tiles_q >= 1 (skGrid >= + numQueues); the Solution layer rejects debug overrides that break this. """ _, mask, log2Queues, _ = self._wsQueueConstants(writer, kernel) sPred = writer.sgprPool.checkOut(1, "wsPredQueue") @@ -504,18 +468,15 @@ def streamKWorkStealingHomeBound(self, writer, mod, kernel, sBound, sQueueIdx, s def streamKWorkStealingSteal(self, writer, mod, kernel, sQueueIdx, sWorkItemIdx, sGrid, mkLabel): """Single-hop next-neighbor steal on the per-XCD queue topology. - Register contract: on entry sQueueIdx holds the home queue index q and - sWorkItemIdx holds the home fetch result; both must be live. If the home - fetch was valid (index < TotalItems) this is a no-op. Otherwise the WG - steals one tile from its next neighbor s = (q+1) & (numQueues-1) via a - single s_atomic_inc, then recomputes the global tile index from s. A lost - race leaves sWorkItemIdx >= TotalItems so the downstream valid-index check - turns this WG into a no-op. sQueueIdx is clobbered (advanced to s). Caller - must gate on kernel["StreamKWorkStealing"] and pass the mode-appropriate - grid SGPR name (``"skGrid"`` for SK4, ``"SKGrid"`` for SK5-dynamic). - - The steal atomic uses the same predecessor-inclusive static bound as the - home path, evaluated for the stolen queue: ``tiles_s + W_s + W_q - 1``. + On entry sQueueIdx holds home queue q and sWorkItemIdx holds the home + fetch result (both live). If the home fetch was valid (index < + TotalItems) this is a no-op; otherwise one s_atomic_inc steals from the + next neighbor s = (q+1) & mask and the global tile index is recomputed + from s. A lost race leaves sWorkItemIdx >= TotalItems, so the downstream + valid-index check turns this WG into a no-op. sQueueIdx is clobbered + (advanced to s). Caller gates on kernel["StreamKWorkStealing"] and passes + the grid SGPR name ("skGrid" for SK4, "SKGrid" for SK5-dynamic). The + steal atomic uses the stolen queue's bound ``tiles_s + W_s + W_q - 1``. """ _, mask, log2Queues, cacheLineLog2 = self._wsQueueConstants(writer, kernel) skFetchDone = mkLabel("SK_FetchDone") @@ -3129,9 +3090,8 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): module.add(SCBranchSCC0(labelName=skSkipWorkItem.getLabelName(), comment="Skip work item")) writer.sgprPool.checkIn(sWave) - # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for the - # StreamKIdx/queue divisions, log2(cache-line size) for the per-queue - # counter stride == one cache line, no false sharing). + # Per-arch dynamic-queue fast-mask constants: log2(numQueues) for the + # StreamKIdx/queue divisions, log2(cache-line size) for the counter stride. _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index @@ -3486,13 +3446,7 @@ def routeToGeneralBatchedOrStridedBatched(self, stridedBatchedGemmLoad, generalB def kernelEnd(self, writer, kernel): module = Module("StreamK Dynamic kernelEnd") - # We don't need to track completed kernels if we know total tiles and grid size. - # The reset is baked into the atomic_inc auto-reset bound at the top of the - # loop: under single-hop next-neighbor work stealing the per-queue bound folds in the - # predecessor's workgroup count (see streamKWorkStealingHomeBound / - # streamKWorkStealingSteal), so every counter wraps back to 0 on its own and - # leaves the (host-zeroed-once) Synchronizer clean for the next launch. No - # explicit end-of-kernel reset is required. + # Per-queue atomic_inc auto-resets; no kernelEnd reset needed. return module @@ -3788,9 +3742,8 @@ def emitDynamicGRA(mod): comment="Skip work item")) writer.sgprPool.checkIn(sWave) - # Per-arch dynamic-queue fast-mask constants (log2(numQueues) for - # the StreamKIdx/queue divisions, log2(cache-line size) for the - # per-queue counter stride == one cache line, no false sharing). + # Per-arch dynamic-queue fast-mask constants: log2(numQueues) for the + # StreamKIdx/queue divisions, log2(cache-line size) for the counter stride. _, _, wsLog2Queues, wsCacheLineLog2 = self._wsQueueConstants(writer, kernel) # Default queue index @@ -4337,10 +4290,7 @@ def routeToGeneralBatchedOrStridedBatched(self, stridedBatchedGemmLoad, generalB def kernelEnd(self, writer, kernel): module = Module("StreamK Hybrid kernelEnd") - # Single-hop next-neighbor work stealing self-resets every per-queue counter via the - # predecessor-inclusive atomic_inc auto-reset bound at the top of the loop - # (see streamKWorkStealingHomeBound / streamKWorkStealingSteal), so no - # explicit end-of-kernel reset is required on either SK5 sub-path. + # Per-queue atomic_inc auto-resets; no kernelEnd reset needed. return module diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index eec0ce842289..8bfb18e91dcc 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1679,18 +1679,8 @@ def assignDerivedParameters( if state["StreamKAtomic"]: reject(state, printRejectionReason, "StreamKWorkStealing is not supported with StreamKAtomic") - # Auto-reset precondition: the per-queue atomic_inc - # auto-reset bound is now predecessor-inclusive (home bound = - # tiles_q + W_q + W_(q-1) - 1, steal bound = tiles_s + W_s + W_q - 1). - # It self-resets each launch ONLY while every queue that owns tiles also - # owns at least one workgroup, i.e. W_q >= 1 whenever tiles_q >= 1 - # (equivalently skGrid >= numQueues, guaranteed by the CU-count-derived - # grid at runtime). DebugStreamK overrides the standard partials+fixup - # dispatch (1=no fixup, 2=no partials, 3=both) and can leave a queue - # owning tiles that are never dispensed by a home workgroup -- two - # consecutive zero-home queues owning tiles -- which breaks the wrap and - # would leave the Synchronizer non-zero for the next launch. Reject the - # combination so the invariant cannot be violated by a debug override. + # Reject DebugStreamK with work stealing: it can leave a tile-owning + # queue with no home workgroup, breaking the W_q>=1 auto-reset precondition. if state["DebugStreamK"]: reject(state, printRejectionReason, "StreamKWorkStealing requires DebugStreamK=0 (the per-queue " diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml index 8beba20b7e05..107677cc23fd 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_dynamic_work_stealing.yaml @@ -14,22 +14,15 @@ GlobalParameters: DataInitTypeB: 13 DataInitTypeC: 12 MaxWorkspaceSize: 134217728 - # Repeated back-to-back enqueues per sync reuse the SAME (host-zeroed-once) - # Synchronizer workspace without re-zeroing between launches. This is exactly - # what the per-queue static auto-reset must guarantee: every counter wraps - # back to 0 on its own so the next launch starts clean (there is no explicit - # end-of-kernel reset anymore). + # Back-to-back enqueues reuse the same Synchronizer workspace, exercising the + # per-queue auto-reset across launches. NumWarmups: 2 EnqueuesPerSync: 4 - # SleepPercent injects a variable per-WG delay so tiles drain unevenly across - # queues, forcing the single-hop next-neighbor steal + sticky-home path to fire under - # contention rather than in lockstep. + # Variable per-WG delay drains queues unevenly so the next-neighbor steal fires under contention. SleepPercent: 50 -# Shared SK4-dynamic fork entries (single-value) plus common benchmark params. -# StreamKWorkStealing is forked over [0, 1] so reference-vs-kernel validation -# runs for BOTH the stealing-off (must match baseline asm) and stealing-on -# paths in a single invocation. +# Shared SK4-dynamic fork entries; StreamKWorkStealing forked over [0, 1] to +# validate both the stealing-off and stealing-on paths. .SkDynamicCommonFork: &sk_dynamic_common_fork - KernelLanguage: ["Assembly"] - PrefetchLocalRead: [1] @@ -53,16 +46,8 @@ GlobalParameters: JoinParameters: BenchmarkJoinParameters: BenchmarkFinalParameters: - # Multiple sizes + repeated back-to-back launches exercise the per-queue - # static auto-reset across launches that reuse the same (host-zeroed-once) - # Synchronizer -- there is no longer an explicit kernelEnd reset. - # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0); - # with the next-neighbor steal this still exercises the neighbor steal - # + auto-reset (the old remainder==0 skip is gone). - # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (steal fires) - # 896x896 -> 49 (MT128) / 196 (MT64) tiles : unaligned, larger grid -> - # contention size (uneven drain crossed with SleepPercent) - # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) + # Aligned and unaligned sizes plus a batched case exercise the neighbor + # steal and per-queue auto-reset across repeated launches. - ProblemSizes: - Exact: [512, 512, 1, 512] - Exact: [640, 640, 1, 512] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml index 50d89739bcd9..ad69afca1fe1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/sk_hybrid_work_stealing.yaml @@ -14,28 +14,18 @@ GlobalParameters: DataInitTypeB: 13 DataInitTypeC: 12 MaxWorkspaceSize: 134217728 - # Repeated back-to-back enqueues per sync reuse the SAME (host-zeroed-once) - # Synchronizer workspace without re-zeroing between launches. This is exactly - # what the per-queue static auto-reset must guarantee: every counter wraps - # back to 0 on its own so the next launch starts clean (there is no explicit - # end-of-kernel reset anymore). + # Back-to-back enqueues reuse the same Synchronizer workspace, exercising the + # per-queue auto-reset across launches. NumWarmups: 2 EnqueuesPerSync: 4 - # SleepPercent injects a variable per-WG delay so tiles drain unevenly across - # queues, forcing the single-hop next-neighbor steal + sticky-home path to fire under - # contention rather than in lockstep. + # Variable per-WG delay drains queues unevenly so the next-neighbor steal fires under contention. SleepPercent: 50 - # Exercise the two deterministic SK5 hybrid-mode paths in one run: - # 0 -> static work assignment (SK3 sub-path) - # 1 -> dynamic per-queue work-queue path (SK4 sub-path; work stealing lives here) - # Crossed with StreamKWorkStealing [0,1] below this validates: - # static + steal-off, static + steal-on (steal is a no-op / harmless reset), - # dynamic + steal-off, dynamic + steal-on (steal actually fires). + # SK5 hybrid-mode paths: 0 = static (SK3 sub-path), 1 = dynamic per-queue + # (SK4 sub-path, where work stealing applies). Crossed with StreamKWorkStealing below. StreamKHybridMode: [0, 1] -# Shared SK5-hybrid fork entries (single-value) plus common benchmark params. -# StreamKWorkStealing is forked over [0, 1] so reference-vs-kernel validation -# runs for both stealing-off and stealing-on across both hybrid sub-paths. +# Shared SK5-hybrid fork entries; StreamKWorkStealing forked over [0, 1] to +# validate both stealing-off and stealing-on across both hybrid sub-paths. .SkHybridCommonFork: &sk_hybrid_common_fork - KernelLanguage: ["Assembly"] - PrefetchLocalRead: [1] @@ -59,16 +49,8 @@ GlobalParameters: JoinParameters: BenchmarkJoinParameters: BenchmarkFinalParameters: - # Multiple sizes + repeated back-to-back launches exercise the per-queue - # static auto-reset across launches that reuse the same (host-zeroed-once) - # Synchronizer -- there is no longer an explicit kernelEnd reset. - # 512x512 -> 16 (MT128) / 64 (MT64) tiles : aligned (remainder 0); - # with the next-neighbor steal this still exercises the neighbor steal - # + auto-reset (the old remainder==0 skip is gone). - # 640x640 -> 25 (MT128) / 100 (MT64) tiles : unaligned (steal fires) - # 896x896 -> 49 (MT128) / 196 (MT64) tiles : unaligned, larger grid -> - # contention size (uneven drain crossed with SleepPercent) - # 384x384x3-> 27 (MT128) / 108 (MT64) tiles : unaligned + batched (steal fires) + # Aligned and unaligned sizes plus a batched case exercise the neighbor + # steal and per-queue auto-reset across repeated launches. - ProblemSizes: - Exact: [512, 512, 1, 512] - Exact: [640, 640, 1, 512] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md index b5c5b8d187c3..f863dca06254 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/DECISIONS.md @@ -349,28 +349,10 @@ accepted equivalents/pragmas here, each with its one-line reason. **Context** kernel basename hash changes across all archs; assembly verified unchanged/correct; no err or kernel-count changes." ## D17 — StreamKWorkStealing added to the required (min-naming) parameter set -**Decision:** Accept the regenerated `_codegen` / SolutionClass / ValidParameters goldens after -adding the new codegen parameter `StreamKWorkStealing` to `Common/RequiredParameters.py` -(`getRequiredParametersMin`). Regenerate surgically via the characterization suite; do not -hand-edit goldens. -**Classification:** intended behavior change (overlay class (a)), not a bug and not test fragility. -Adding the parameter to the min-naming set inserts the `SKWS0` token into the verbose solution -name (next to its StreamK siblings `SKA`/`SKFTR`/`SKFDPO`/`SKXCCM`) and perturbs the kernel -identity-hash `basename` on **every** kernel across **all** archs — `num_keys` 334→335. This is -the same required-parameter-promotion footprint as D16 (BufferLoad/BufferStore). -**Why the diff spans all arch nodes (not blanket masking):** the change is a single new required -parameter, so by construction it touches every kernel's identity hash; the wide golden diff is the -faithful, minimal consequence of that one source change, not an unrelated multi-node re-record. -Verified: only `basename` hashes + the `SKWS0` name token + the added roster/valid-values entry -change — **no `err`, instruction-count, or emitted-assembly changes**. `minNaming` still drops the -constant param from the *short* visible kernel name, so no `_WS0` token pollutes production kernel -names. -**Stable-arch signal (gfx908/gfx90a/gfx942):** the digest changes on the stable archs are this -intended naming/identity change, **not** a compiler/codegen regression — root cause is the -required-param addition above; assembly is byte-for-byte unchanged with `StreamKWorkStealing=0`. -**Re-run:** goldens are byte-identical on two further `--snapshot-update`-free runs and the full -`-m unit` suite stays green. -**Alternatives rejected:** (a) keep the param out of `RequiredParameters` — rejected: it would -drop the param from the kernel identity, so two solutions differing only in `StreamKWorkStealing` -would collide on the same name/hash; (b) hand-edit the goldens — rejected: derived state + name -hashes are computed, hand-editing desynchronizes the serialized state from real codegen output. \ No newline at end of file +**Decision:** Promote `StreamKWorkStealing` to the required (min-naming) parameter set in +`Common/RequiredParameters.py` and accept the regenerated `_codegen` / SolutionClass / +ValidParameters goldens. +**Why:** without it, two solutions differing only in `StreamKWorkStealing` would collide on the +same kernel identity name/hash. +**Verification:** only `basename` hashes + the `SKWS0` name token + the roster/valid-values entry +change (`num_keys` 334→335); no `err`, instruction-count, or emitted-assembly changes. \ No newline at end of file diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index 6fc44098b264..08d9201b159a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -5,20 +5,15 @@ These tests assert that the work-stealing assembly is emitted by the helper methods on the ``StreamK`` base class, and -- crucially -- that those helpers are only ever reached behind the codegen-time ``StreamKWorkStealing`` toggle. -Following the StreamK=5 hybrid tests, they import rocisa instructions and -inspect emitted modules rather than matching source text; the toggle gating -and the Solution-level validation are verified by executing the *real* source -(via the AST) so the assertions track the actual code, not a copy of it. +They import rocisa instructions and inspect emitted modules rather than matching +source text; the toggle gating and the Solution-level validation are verified by +executing the *real* source (via the AST) so the assertions track the actual code. Emission contract (single-hop next-neighbor + sticky-home + static auto-reset): - * The steal always fires on an empty home fetch -- there is no remainder / - structural-extra guard (no ``s_cmp_ge_u32`` neighbor guard, no - ``remainder == 0`` skip). - * The steal & home atomic bounds are the predecessor-inclusive self-reset - value. + * The steal fires on an empty home fetch with no neighbor structural-extra guard. + * The steal & home atomic bounds are the predecessor-inclusive self-reset value. * A per-WG sticky-empty SGPR gates the home fetch. - * There is no ``streamKWorkStealingKernelEndReset`` / completion counter / - reset barrier. + * kernelEnd emits no explicit reset; per-queue counters auto-reset. """ import ast @@ -230,8 +225,7 @@ def test_work_stealing_param_is_zero_one(self): # =========================================================================== -# 2. The work-stealing helper methods: the new steal/bound helpers exist and -# the removed single-hop-reset helpers are gone. +# 2. The work-stealing helper methods exist on the StreamK base class. # =========================================================================== class TestHelperMethodsExist: @pytest.mark.parametrize( @@ -244,23 +238,6 @@ class TestHelperMethodsExist: def test_method_is_defined_on_base(self, name): assert callable(getattr(StreamK, name)) - @pytest.mark.parametrize( - "name", - [ - "streamKWorkStealingHomeNoReset", - "streamKWorkStealingKernelEndReset", - ], - ) - def test_removed_methods_are_gone(self, name): - assert not hasattr(StreamK, name), ( - "explicit-reset / no-reset helpers must be removed" - ) - - def test_completion_counter_constant_is_gone(self): - assert not hasattr(StreamK, "_WS_COMPLETION_COUNTER_OFFSET"), ( - "the 0x80 completion-counter offset must be removed" - ) - # =========================================================================== # 3a. Home auto-reset bound: the predecessor's workgroup count is folded into @@ -399,8 +376,7 @@ def test_steal_passes_grid_sgpr(self, func): # =========================================================================== -# 3d. No explicit reset survives anywhere in kernelEnd, and the 0x80 -# completion counter / reset barrier are gone. +# 3d. kernelEnd emits no work-stealing reset and no completion counter. # =========================================================================== class TestNoExplicitReset: @pytest.mark.parametrize("func", [StreamKDynamic.kernelEnd, StreamKHybrid.kernelEnd]) @@ -496,8 +472,8 @@ def test_rejected_with_atomic(self, streamk): @pytest.mark.parametrize("debug", [1, 2, 3]) def test_rejected_with_debug_streamk(self, debug): - # Proof condition #1: DebugStreamK overrides could break the W_q>=1 - # (skGrid>=numQueues) precondition the per-queue auto-reset relies on. + # DebugStreamK overrides can break the W_q>=1 precondition the per-queue + # auto-reset relies on, so the combination is rejected. state = self._run(streamk=4, atomic=0, debug_streamk=debug) assert state["Valid"] is False diff --git a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp index 3f281bc19c6e..0e36262af408 100644 --- a/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp +++ b/projects/hipblaslt/tensilelite/rocisa/rocisa/include/hardware_caps.hpp @@ -610,12 +610,8 @@ inline std::map initArchCaps(const IsaVersion& isaVersion) // StreamK queue masking (AND/shift) valid. rv["NumXCD"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 8 : 1; - // codegen-side mirror of origami's L2 cache-line size - // (hardware_t::get_default_cache_line_bytes): 128 B for the dynamic-queue - // arches gfx942/gfx950. Power-of-two so the StreamK queue-address shift - // stays valid; the StreamK per-XCD counter stride is set equal to this - // cache-line size so each counter occupies its own line (no false sharing). - rv["CacheLineBytes"] = checkInList(isaVersion, {{9, 4, 2}, {9, 5, 0}}) ? 128 : 128; + // Per-queue counter stride = L2 cache-line size (uniform 128B on supported archs). + rv["CacheLineBytes"] = 128; return rv; } diff --git a/shared/origami/include/origami/hardware.hpp b/shared/origami/include/origami/hardware.hpp index 7056a28d13ec..5c782c4b3ab0 100644 --- a/shared/origami/include/origami/hardware.hpp +++ b/shared/origami/include/origami/hardware.hpp @@ -759,15 +759,8 @@ class ORIGAMI_EXPORT hardware_t { /** * @brief Get the default L2 cache-line size (in bytes) for an architecture. * - * Single source of truth for the hardware L2 cache-line size, consolidating - * the scattered 128-byte literals used across origami (streamk.cpp - * CACHE_LINE_BYTES, origami.cpp L2_CACHE_LINE_BYTES, - * heuristics.hpp EPILOGUE_CACHE_LINE_BYTES, formocast.cpp L2CacheLineSize). - * CDNA3 (gfx942) and CDNA4 (gfx950) both use a 128-byte L2 line (confirmed by - * AMD whitepapers); 128 is a safe default for the other supported - * architectures. Consumers that place one datum per cache line (e.g. the - * StreamK per-XCD atomic counters) use this so each occupies its own line and - * avoids false sharing. + * Returns the per-arch L2 cache-line size used for the StreamK per-queue + * counter stride (currently uniform 128 B across supported archs). * * @param arch Architecture enum value * @return L2 cache-line size in bytes diff --git a/shared/origami/src/origami/hardware.cpp b/shared/origami/src/origami/hardware.cpp index 7fdc3be1313f..0221017c1c16 100644 --- a/shared/origami/src/origami/hardware.cpp +++ b/shared/origami/src/origami/hardware.cpp @@ -197,15 +197,9 @@ size_t hardware_t::get_default_num_xcds(architecture_t arch) { } } -size_t hardware_t::get_default_cache_line_bytes(architecture_t arch) { - // L2 cache-line size in bytes. Consolidates the 128-byte literals that were - // duplicated across origami. gfx942 (CDNA3) and gfx950 (CDNA4) use a 128-byte - // L2 line (AMD whitepapers); 128 is a safe default for the rest. - switch (arch) { - case architecture_t::gfx942: return 128; - case architecture_t::gfx950: return 128; - default: return 128; - } +size_t hardware_t::get_default_cache_line_bytes(architecture_t /*arch*/) { + // Per-arch L2 cache-line size, currently uniform 128 B across supported archs. + return 128; } void hardware_t::print() const { From f9569cadb8b5c86068ffe84404bf239f77d39444 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 14 Jul 2026 15:44:42 +0000 Subject: [PATCH 16/20] fix(tensilelite): reject StreamK dynamic-queue/work-stealing on unknown XCD count and gfx1250 (1) Treat unknown/null NUM_XCD (missing analyticalHardware) as unsupported for the dynamic-queue path and fail early rather than under-allocating the counter workspace. streamKDynamicQueueUnsupported now returns true for unknown hardware so the softwarePredicate excludes the dynamic-queue solution and a non-dynamic-queue solution serves the GEMM; solve() throws an explicit error early if the baked queue count / NUM_XCD is unknown (0) before workspace sizing. (2) Reject StreamKWorkStealing on gfx1250, which lacks scalar atomics (HasSAtomic=false) for the steal path (streamKWorkStealingSteal emits s_atomic_inc with no vector fallback). gtest (CuCount_test) and unit test (test_streamk_work_stealing) updated. No codegen change for gfx942/gfx950 (asm byte-identical). --- .../Tensile/SolutionStructs/Solution.py | 11 ++++ .../Tests/unit/test_streamk_work_stealing.py | 51 +++++++++++++++++-- .../tensilelite/src/ContractionSolution.cpp | 39 ++++++++++---- .../tensilelite/tests/CuCount_test.cpp | 29 +++++++---- 4 files changed, 104 insertions(+), 26 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 8bfb18e91dcc..71499c612b7c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1686,6 +1686,17 @@ def assignDerivedParameters( "StreamKWorkStealing requires DebugStreamK=0 (the per-queue " "auto-reset relies on W_q>=1 whenever tiles_q>=1); got %d" % state["DebugStreamK"]) + # The steal path (streamKWorkStealingSteal) emits s_atomic_inc + # unconditionally, but the home fetch (_fetchNextWorkItem) only falls + # back to a returning vector atomic when scalar atomics are absent. On + # arches without scalar atomics (HasSAtomic=false, e.g. gfx1250) the + # steal would emit an unsupported s_atomic_inc with no vector fallback, + # so reject work stealing there. + if not isaInfoMap[isa].asmCaps["HasSAtomic"]: + reject(state, printRejectionReason, + "StreamKWorkStealing requires scalar atomics (HasSAtomic); the " + "work-stealing steal path emits s_atomic_inc with no vector " + "fallback (e.g. gfx1250 has HasSAtomic=false)") if not state["Valid"]: print2("in assignDerivedParameters, state['Valid'] = False") return diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py index 08d9201b159a..445ed1fde4ec 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_work_stealing.py @@ -177,7 +177,12 @@ def _source_of(func) -> str: def _extract_ws_validation(): """Compile the *real* ``if state["StreamKWorkStealing"]:`` block out of ``Solution.assignDerivedParameters`` into a standalone callable so the - actual rejection logic can be exercised without a full Solution state.""" + actual rejection logic can be exercised without a full Solution state. + + The block reads ``isaInfoMap[isa].asmCaps`` (for the HasSAtomic / gfx1250 + gate), so ``isa`` and ``isaInfoMap`` are threaded in as parameters -- the + real function derives ``isa = tuple(state["ISA"])`` and receives + ``isaInfoMap`` as an argument.""" tree = ast.parse( textwrap.dedent(inspect.getsource(Solution.assignDerivedParameters)) ) @@ -194,7 +199,13 @@ def _extract_ws_validation(): name="_validate", args=ast.arguments( posonlyargs=[], - args=[ast.arg("state"), ast.arg("printRejectionReason"), ast.arg("reject")], + args=[ + ast.arg("state"), + ast.arg("printRejectionReason"), + ast.arg("reject"), + ast.arg("isa"), + ast.arg("isaInfoMap"), + ], vararg=None, kwonlyargs=[], kw_defaults=[], @@ -213,6 +224,17 @@ def _extract_ws_validation(): return ns["_validate"] +# gfx1250 (12,5,0) has no scalar atomics; gfx942/gfx950 do. Mirror just the +# HasSAtomic capability the work-stealing validation reads. +_NO_SATOMIC_ISAS = {(12, 5, 0)} + + +def _fake_isa_info_map(isa): + isa = tuple(isa) + asm_caps = {"HasSAtomic": isa not in _NO_SATOMIC_ISAS} + return {isa: types.SimpleNamespace(asmCaps=asm_caps)} + + # =========================================================================== # 1. ValidParameters: the codegen-time toggle exists and is boolean. # =========================================================================== @@ -445,14 +467,16 @@ class TestSolutionValidation: def setup_method(self): self.validate = _extract_ws_validation() - def _run(self, *, streamk, atomic, work_stealing=1, debug_streamk=0): + def _run(self, *, streamk, atomic, work_stealing=1, debug_streamk=0, isa=(9, 4, 2)): + isa = tuple(isa) state = { "StreamKWorkStealing": work_stealing, "StreamK": streamk, "StreamKAtomic": atomic, "DebugStreamK": debug_streamk, + "ISA": isa, } - self.validate(state, False, reject) + self.validate(state, False, reject, isa, _fake_isa_info_map(isa)) return state @pytest.mark.parametrize("streamk", [0, 1, 2, 3]) @@ -482,3 +506,22 @@ def test_off_toggle_is_inert_even_for_bad_combo(self): # that would otherwise be rejected. state = self._run(streamk=3, atomic=1, work_stealing=0, debug_streamk=3) assert "Valid" not in state + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_rejected_on_gfx1250_no_scalar_atomic(self, streamk): + # gfx1250 lacks scalar atomics (HasSAtomic=false); the steal path emits + # s_atomic_inc with no vector fallback, so work stealing is rejected. + state = self._run(streamk=streamk, atomic=0, isa=(12, 5, 0)) + assert state["Valid"] is False + + @pytest.mark.parametrize("isa", [(9, 4, 2), (9, 5, 0)]) + @pytest.mark.parametrize("streamk", [4, 5]) + def test_accepted_on_scalar_atomic_arches(self, isa, streamk): + # gfx942/gfx950 have scalar atomics, so work stealing stays accepted. + state = self._run(streamk=streamk, atomic=0, isa=isa) + assert state.get("Valid", True) is True + + def test_off_toggle_is_inert_on_gfx1250(self): + # StreamKWorkStealing=0 on gfx1250 must not fire the reject. + state = self._run(streamk=4, atomic=0, work_stealing=0, isa=(12, 5, 0)) + assert "Valid" not in state diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 9ac35f1874c7..c05035b0d8a8 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -106,19 +106,24 @@ namespace TensileLite // The dynamic-queue fetch / work stealing is only correct when the // device's runtime NUM_XCD is a power of two AND equals the baked - // per-XCD queue count. Returns true (UNSUPPORTED) when NUM_XCD is 0, not - // a power of two, the baked count is unknown, or NUM_XCD != baked (e.g. - // MI300A's 6 XCDs, or a 4-XCD partition of an 8-XCD gfx942). Returns - // false when there is no analytical hardware (unknown XCD -> allow). Kept - // isolated here so it stays trivially unit-testable (see CuCount_test.cpp). + // per-XCD queue count. Returns true (UNSUPPORTED) when the hardware is + // unknown (not a HipAMDGPU, missing analytical hardware, or no baked + // per-XCD queue count), when NUM_XCD is 0, not a power of two, or + // NUM_XCD != baked (e.g. MI300A's 6 XCDs, or a 4-XCD partition of an + // 8-XCD gfx942). Unknown hardware is treated as UNSUPPORTED: the + // dynamic-queue solution is then excluded from selection and a + // non-dynamic-queue solution serves the GEMM, rather than staying + // selectable while the per-XCD counter workspace is sized with an + // unknown (0) queue count (which would under-allocate). Kept isolated + // here so it stays trivially unit-testable (see CuCount_test.cpp). inline bool streamKDynamicQueueUnsupported(Hardware const& hardware) { auto const* hipAMDGPU = dynamic_cast(&hardware); if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) - return false; + return true; size_t baked = streamKBakedQueueCount(hardware); size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; - return numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || baked == 0 + return baked == 0 || numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || numXCD != baked; } @@ -3252,6 +3257,17 @@ namespace TensileLite if(dynamicQueuePath && streamKDynamicQueueUnsupported(hardware)) { warnStreamKDynamicQueueUnsupportedOnce(hardware); + // Fail EARLY -- before workspace sizing / kernel-arg packing -- + // when NUM_XCD is unknown (baked queue count == 0, e.g. missing + // analyticalHardware). Sizing the per-XCD counter region with a + // 0 queue count would under-allocate the workspace the kernel + // writes; reject with an actionable message instead. + if(streamKBakedQueueCount(hardware) == 0) + throw std::runtime_error( + "hipBLASLt Error: StreamK dynamic-queue (work-stealing) requires a known " + "NUM_XCD (analyticalHardware unavailable); refusing to size the per-XCD " + "counter workspace with an unknown queue count. " + "Select a non-work-stealing solution instead."); throw std::runtime_error( "hipBLASLt Error: StreamK dynamic-queue (work-stealing) solution selected on a " "device whose XCD count is not a power of two or does not equal the compiled " @@ -4031,10 +4047,11 @@ namespace TensileLite // Fast/common path: on hardware whose runtime XCD count is a power of // two AND equals the baked per-XCD queue count, the fixed queue masking - // is valid, so nothing is excluded. This also covers the "unknown XCD" - // case (predicate returns false), preserving historic behavior. Checked - // before streamK5EffectiveDynamic() so the mainline gfx942(MI300X)/ - // gfx950 path stays cheap. + // is valid, so nothing is excluded. Unknown hardware (missing analytical + // info / no baked count) is treated as UNSUPPORTED by the predicate, so + // it falls through to the reject-and-continue path below rather than + // being kept. Checked before streamK5EffectiveDynamic() so the mainline + // gfx942(MI300X)/gfx950 path stays cheap. if(!streamKDynamicQueueUnsupported(hardware)) return true; diff --git a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp index b68541a315d3..e06e8124dbf1 100644 --- a/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/CuCount_test.cpp @@ -779,14 +779,16 @@ namespace // Byte-for-byte mirror of the anonymous-namespace numeric predicate in // ContractionSolution.cpp (streamKDynamicQueueUnsupported). Kept in lockstep // with the production code; if that predicate changes, update this too. + // Unknown hardware (not a HipAMDGPU, no analytical hardware, or no baked + // per-XCD queue count) is treated as UNSUPPORTED (returns true). inline bool streamKDynamicQueueUnsupportedRef(Hardware const& hardware) { auto const* hipAMDGPU = dynamic_cast(&hardware); if(hipAMDGPU == nullptr || hipAMDGPU->analyticalHardware == nullptr) - return false; + return true; size_t baked = streamKBakedQueueCountRef(hardware); size_t numXCD = hipAMDGPU->analyticalHardware->NUM_XCD; - return numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || baked == 0 + return baked == 0 || numXCD == 0 || (numXCD & (numXCD - 1)) != 0 || numXCD != baked; } @@ -877,21 +879,26 @@ TEST(StreamKDynamicQueueXcdGateTest, AllowsGfx950EightXcd) << "gfx950 (NUM_XCD=8) must keep the dynamic-queue work-stealing path"; } -TEST(StreamKDynamicQueueXcdGateTest, MissingAnalyticalHardwarePreservesHistoricBehavior) +TEST(StreamKDynamicQueueXcdGateTest, MissingAnalyticalHardwareIsUnsupported) { - // The real "unknown -> preserve historic behavior (allow)" path in the - // production predicate is a null analyticalHardware: it returns false so a - // device without analytical info keeps the pre-gate dynamic-queue path. - // (The predicate's separate NUM_XCD==0 guard returns true as a defensive - // fallback, but that state is not constructible here: origami::hardware_t - // divides by NUM_XCD, so a real analytical hardware can never carry 0 XCDs.) + // Unknown hardware (null analyticalHardware -> unknown NUM_XCD / baked + // queue count == 0) must be treated as UNSUPPORTED so the dynamic-queue + // solution is excluded from selection and a non-dynamic-queue solution + // serves the GEMM, rather than staying selectable while the per-XCD counter + // workspace is sized with an unknown (0) queue count (under-allocation). hip::HipAMDGPU noAnalytical; noAnalytical.processor = AMDGPU::Processor::gfx942; noAnalytical.deviceName = "test-gfx942-no-analytical"; Hardware const& hwNoAnalyt = noAnalytical; ASSERT_EQ(noAnalytical.analyticalHardware, nullptr); - EXPECT_FALSE(streamKDynamicQueueUnsupportedRef(hwNoAnalyt)) - << "Missing analyticalHardware must fall through to historic behavior (allow)"; + EXPECT_TRUE(streamKDynamicQueueUnsupportedRef(hwNoAnalyt)) + << "Missing analyticalHardware (unknown NUM_XCD) must be treated as unsupported"; + // And the selection predicate must therefore EXCLUDE the dynamic-queue + // solution (SK4) while keeping non-dynamic-queue solutions selectable. + EXPECT_FALSE(streamKDynamicQueueSupportedRef(4, /*effectiveDynamic=*/false, hwNoAnalyt)) + << "SK4 work-stealing solution must be excluded when NUM_XCD is unknown"; + EXPECT_TRUE(streamKDynamicQueueSupportedRef(3, /*effectiveDynamic=*/false, hwNoAnalyt)) + << "SK3-static solution must remain selectable when NUM_XCD is unknown"; } // Selection-predicate contract: on MI300A (6 XCD) the dynamic-queue solution is From 4cf435dc20aa0268bef6f7478ce4275650172e99 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Fri, 17 Jul 2026 19:36:41 +0000 Subject: [PATCH 17/20] Regenerate characterization .ambr snapshots after merging develop --- .../test_emit_bigfiles_char.ambr | 110 +++++++++--------- .../__snapshots__/test_emit_gfx1250_char.ambr | 8 +- .../__snapshots__/test_r3_datamover_char.ambr | 4 +- .../test_r3_streamk_gfx1250_char.ambr | 2 +- 4 files changed, 62 insertions(+), 62 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr index c22038cb4017..0c90fb5cf336 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_bigfiles_char.ambr @@ -2,27 +2,27 @@ # name: test_bigfile_capped_emit[aqua_FreeSize_GSU9] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x128_MI16x16xS-Gljw7Tndlcz4DyWBcnsgvq3a2XnFHzrLYjk74Pzdc=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x128_MI16x16x0PEaqgf_3RNkFiTAYvEWBPz1xQTaRvKD9CCT0wpWzO8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x32_MI16x16x1qoMYuCs_HiKfJoyDVjJF73iEbywxwpKqyYZG-pAZZxM=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x32_MI16x16x1YtOY4yg6GqEedQwTv-n9heqwEGq3xaaRM1XPu__B898=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x64_MI16x16x1QuEx_LLnYKhBzZS3anZFPLWuXBLYopd4QyxslRfomFg=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x16x64_MI16x16x1hyUZ5k8XjqFqTa9F4lvmH3KVeG7MolFIfZBX4e-vAQg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x128_MI16x16x7Xuu72GxZdrwjLqko6M-grE0eUmaxIq67pliAXtFLsA=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x128_MI16x16xgqZBDoCQlSnUDsyhEHDS_IywPtOFXyVyS9MrmTd2V_o=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x32_MI16x16x1TB88pJVwEkDVdlgbF-lGwdUOsLEmhdOy147cN_NNt9k=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x32_MI16x16x1XemHpkLDsG2jRjriI-QsPNhqOkIi1ChlH5RUqLCJOo0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x64_MI16x16x10DhfuxvbEVPlwpdPMyKHw58eOGsXDWtplJfAOlIe-Pc=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_MT128x32x64_MI16x16x1CVv_Fn6a_cErQuelTtW-gz9OaFSRPryMGtZjoKyZJCA=', 'err': 0, }), ]) @@ -30,27 +30,27 @@ # name: test_bigfile_capped_emit[equality_gfx950_HSS_big] list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT11XTh2yIemLgZzvT8Y55Aa0IvVgrAKL6JPGW9AK8iv_c=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1AXp0F5JGbnx4dhbPfcfdiKLiZpzsdi2Qh-QdYKqJVds=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2-GpJOvig6JRtx42eDC7p2ObeS9MhqXSKDjaLJiBHbdQ=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2BG_Xr0IlyABFCKVD885PVZTijzjpldjXKvwYqpS2u5Y=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT24_bNGgNsfgaVnmsBLerK44Nb1f2KXpxWKkiJcXUA5_o=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2Ixc5wNAiu0KxZpoJd5m1Njv5sCb21hdhWqFoUx3oFJw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2C_oVG0KdOGf-0Rg_3ZUYmJFVbUFw1dZ7yuUfJJ1HTBY=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2LutL_WMujk_KDSlqnd6HHg28p89ARRyKdIMKnlqRsiw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2FFwlilLGG3fT5kv36Q8b2zsAmwmdK38pv44gtbsQurY=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2_jfECdyYw388q4Z5NVWRhFa3Lzm4ezEjc4kBolTGXkE=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2RZkVObdYiuO0nuXeONKEMF8qzIG4_hcwXc8cjFW2f3A=', + 'basename': 'Cijk_Ailk_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2aMtCgdbT7HrTb6g8kVjovM_zsuZwC23HONnAy8K-vUI=', 'err': 0, }), ]) @@ -58,27 +58,27 @@ # name: test_bigfile_capped_emit[freesize_gfx942_F8NH_GSU] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT133qRnATgZ6_SATxqR8eys5SQD0q41vstAoLMtxv1QWY=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1E5aZ9_fehYw6fSt_tScq6myj_-d6eNVdGx2aC6pfeso=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT13Vej6vrkkNYN7Lr0t5xwlfy-q5SHfIaHsPXPEeK5JmE=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1NO2HVcfYsfG84BvFWBHzQWBnOmF1w9dhZMfX5eVL4dw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT17oFB9E_hglLQMLAOQeW-IP6AKk4eiEGeGPAnOP6P6Gc=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1_JW277HsdkoUBIwc-fs5Rlh_ZvhPmDPhrJ_XiPaNLjk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1JpAcoj4hLJ615psm1yxpaeutLmA65Ez8u4DjMp6CXrk=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1lVzYoGr91c8XWvfeEIZbdpQ5jvMhUIy5l1pPl_UmgQ8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1u1FRQ9WG5lIu1XxfDDKmSwL1EGW5eIYgO4JHv2A5kow=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1mzGzNC--azxkizUIzkjlJR8msadgWFQ10afBUvVxuBM=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1xaablpTT8n2AOqnnJG24vsPuioA6wNxLZqme7JYYEWQ=', + 'basename': 'Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_HA_S_SAB_SAV_MT1rRExUpAg67uP8HjM0uHpxdoHGw5Y-g6VV_PAj0d7MF4=', 'err': 0, }), ]) @@ -86,27 +86,27 @@ # name: test_bigfile_capped_emit[gfx1201_I8II] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT128x1t-w3mKQN4gPbAHvx7dEmtx5k2vVFzdhJvVVoZMNZlkU=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT128x1VWV9dbHQKcIAhtN1me2X8z1f9JHeVFF-CYoSQrD1dTg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64UqI7LKTAo8E27Z3PNZybELRT8OQMQHIuWDkWQPcC8As=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64Owh9Npi9kG0OFr53vXnmChtzzU6vTV1Pz3uil-bfsck=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64r-vXE83Tgqxz1GCSrKmwqRNS7NJ29DsjhklzuhU0JUE=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT16x64eA9elviyPpe4aDf2sJGUfdSv5bIL1WC9c5GakgJjTx0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x11hOwt_IvHXQI_4oxZXmGBo0VrgdMZdx29D_13160H0Y=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x11FGzDdqaXJHCGnOB7WIDkYA-p7NxZmdzO9DIWaXE730=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x13VFHx09b6USykTyUU9-dvtP3ZFIrk7z424bSSKla7zw=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x120Quts8nquDZEpPMPSV9CvZA35ufBLTCuIVjO61pZYw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x13va2QNOinbNB_T7QxVhpsn0KhKXiJB-BgEEGRgku7hU=', + 'basename': 'Cijk_Ailk_Bljk_I8II_BH_HA_I_SAV_UserArgs_MT256x15ESidfh4mCdT4xoEf0fiFaOuH5sQAdlaQ40J1I9DwbA=', 'err': 0, }), ]) @@ -114,15 +114,15 @@ # name: test_bigfile_capped_emit[gfx1250_GG] list([ dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2I8Z4MnNhn0H6OkrgkCJB2REmBpHKVBnmyvU3B0pEk5o=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT27R-Me2IAXS91gvesLlSE9It6N5WAoCUnDCgktX1SbT0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2On8jZq2r35NyCPrHV_DUmxEc59zqvQbrqb6HnMIdEFA=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT2ox_DJro5lCW_-A7qhSVaKKLmiWJZRtkDZsjP1TDj0Bk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3qS1-XgGxY_ZW8LCkQ2h9qcTZZNFDrv93bs9HZOExcPM=', + 'basename': 'Cijk_Alik_Bljk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3iQRATBChzKmj1EIiwQhdonRchLYdnqrTnOjyxl1v-Nw=', 'err': 0, }), ]) @@ -130,27 +130,27 @@ # name: test_bigfile_capped_emit[gfx90a_HSS_big] list([ dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32URRlBbZ_0D3f_0tbTrOj16A5kCb4eY_Qj7e89uwrRU8=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32DLt8rifdTTwEiorPWkIA0f1r3Yn0EgSh2PCav5RnSWc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32wJQwzk7gLrdBXa4gk8QLXH-e6H-VV9NkcshCy7j3U1Q=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32VeX7cJBv7G9NceBAXrpx-mKacNH0sm-NHzjdSpTD5rA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32xz67w3tBRw2b2OoNKoaOeNFnXAPua-muhWZCehLg7XY=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x128x32_MI32tgx4OkjbymJApkgI7_Sj2zmeA7YJSQDXivmS-G7Lby8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x192x64_MI32X-_c4pYsuuvRP7JlvpuuONFQfNpg4b2YkHLzwDz01wA=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x192x64_MI32M9aFe5VVPZ_teiAeUhk_8QoWuEDm1s03_qMcDZ0h0sQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32x7chYdE4VjFBaxocWCppI_6yoqGomi78hnzjCr0B8Nzo=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32x1mDz2Fydou1101lZBUD6Ar8ijHp2c0lvRebadryheYg=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32xgVjp1hBfMr_RfHk8OSzMRzghqb_eoxLgxDBaQL8mGLI=', + 'basename': 'Cijk_Ailk_Bljk_HSS_BH_UserArgs_MT128x32x64_MI32x5b5fakZXtv75dqRUjtgqlptKIH32NWU16Osh5Sx8hTc=', 'err': 0, }), ]) @@ -158,27 +158,27 @@ # name: test_bigfile_capped_emit[gfx950_origami_MX] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT15ELi53QSQPpRNXPhhP4pGjQ-VQOHGAaSQyjynFkCceg=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT19C35JeeY-qDrrHdhgd7Va0C2swFtPIZpMjtAp3E6lAk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1FaIXyeyDH-rnatBNFsi1bGRpE9j9Y0bzfkxTgNoem-U=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT19X7kPJSy59Wef9Z3WMgTsvLose5z_pkGDcDJa1OlHF0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1HSd8OaIgyIc0kO9VvvWo-rv8QC4ILmbLWUc9n3yNCDI=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1Ca23TPEModSClyjnn4g1E5qDzzSFI181SJ-eOQyZelo=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1HgH3mKb2OiSx5w7r-7ekQRdnfTDS2-ju_YvgY0f7qnI=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1D3mWEpWvmyQnH2p33U0xAMtDpFY461JOXxtqEXYMn2A=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1I5XdJ_aanho06clTD_WHmzIMINCtQGDIhEeyXsBYJqI=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1FzpkkGJIpIBVxsEPi161T7OS-SYgMalTiUeTAgdDqog=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1NRoU4nLErdpJ9zNVlNGliHdrcWGeTEh3_2zCK0ga1CM=', + 'basename': 'Cijk_Alik_Bjlk_S_MX_B_Bias_HA_S_SAV_UserArgs_MT1H3_nM0eqLE3jQSKKUJawz3C42KEGQmZccpPWRI6nglM=', 'err': 0, }), ]) @@ -186,27 +186,27 @@ # name: test_bigfile_capped_emit[navi31_HSS] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1B1L59GQFcbjjcFBhkHOu89GRxNBmkZ2MrQikGCLSt1c=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1EStc3x092hvUP7fmro5DgeLTNZ54Wui8d_jbUSHalWk=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1lk047TNGyOQQsMfnkLNs9meWp0eyoFemwd9F4DW8JqQ=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT1I4vZ6WDpwHg31N1-egbRfLG3F8vVFQGlJpbwiK2PU2U=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3pSpXxwGvq311WN_Et9H1eYMTHWiETfIlWDUQqXB0k6U=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3kFPCvpHiPqMSGQfs66PnL_lZiJxAy9qVi9pw_1CdHvc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3uff6pT_n6SUbHos7kXoAyeaMGTt-vxtwniT31ekPN1o=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT3sPwilggvpVIZG3DYqhMqnwY4OVJSl0E8_RavcQOI2-c=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4-leGXNR2Cg8vmPrQnZu9cvWZFbOpzKwD0XjNwcNIQ4E=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4iHinGmpo6U9XZLki6M7rkVqcxQ6ZaeP3otCR8URNjNA=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT44hPvPwd2wfINL2uDgCL9jVjKRZWck1F-dPudxi7w1Ws=', + 'basename': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT4rzujJqyV9D4EA5Bal0CwBhJxQ9gEe-w10uIKXsTkNCs=', 'err': 0, }), ]) @@ -214,19 +214,19 @@ # name: test_bigfile_capped_emit[streamk_gfx942_Ailk] list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x16_MI16x164488HlTT3XRdQ1iMiUWtbU-7BGJW-ZPExv7bEtaz3v4=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16iamdKh77SiXB_tsBpkoYVJkqXz7yJXFcFX7zLAN-4e0=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16nmlCwPFNaaWTdIJARwvCUATraAv01Bp_49325nQjMLw=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16RtgaOEkRV8gTmNr339CCNhv41S6CH7M0IP134rKubs8=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16F0GTsPMl_srsNgviNuetd6ghGCvbN5BAkQVNHgnroUE=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16pQbK8lxueBbBe5oFnbPGmVGfvw-J71kHkCknwFIc_Lc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16ttjQ6-75OzKbfYcLAtGh0VSFgrSFnC7ceMtxxR66_ng=', + 'basename': 'Cijk_Ailk_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16HzSAmeO3iAdidAF6iz7WibCBqHVFeKoTR9uYOAPPEx8=', 'err': 0, }), ]) @@ -234,27 +234,27 @@ # name: test_bigfile_capped_emit[streamk_gfx942_S] list([ dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16LIPzO5RGOH-JAgrnAmNmDNstuT7ffdgJnNQwA_C5wEw=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x16_MI16x16sUXwBn3XkfnISoWST8-w6dcxuX4c9gb1b-ACCXCmITQ=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16shsJpGZCbngCiXB-1j-vOIKjWsvtroIwqzvnzhcpHsI=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT112x256x32_MI16x16mICYU5z8Zeu9GVGXH6Q23-00wl4qQS3dpU1OEZeLH-c=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16GxunDHjBJr9zSOOvIHwDzhlWONbdgnmhRbwhX7SpETA=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x16_MI16x16iuBlVFr2UmLymPSjN3Tg2wGxkWAakfqmGQbhFQiSz3U=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x32_MI16x16bKJ1I-I9myKTFgC8TCcJ4cKXinqQexXtxx8Oeq6pIxw=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x128x32_MI16x1626fyQeIWhR0VKyMZB16tLOIV6cOLZSsLIXN5XLBPg-k=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x16_MI16x16wh8AYgwInIbd9m_6RLeSGkJeByUk_8-ZVK25Uo7ijoc=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x16_MI16x16j82k1qF_CMrsFJQJIwRAEjMkjf2SllvK-9iw23wVwaE=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x32_MI16x164lvqXbTOKTPbMjr0Nfo9cEi_2H-CV8-aI2Ed1z_dyJU=', + 'basename': 'Cijk_Alik_Bjlk_S_B_UserArgs_MT128x160x32_MI16x16jvqTs3KuIGVtneIVcxMdIsn2Lyf6RfbrWJU5P3-v_9Q=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr index 87ca0975fe12..2e19eee1e285 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_emit_gfx1250_char.ambr @@ -5,7 +5,7 @@ 'file': 'BBS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_BBS_BH_Bias_A_UserArgs_MT16x16x322B4FnC0zasYp4cXH3fT-GKmdbjeLh6UCDBMSbnvr9H8=', + 'basename': 'Cijk_Ailk_Bjlk_BBS_BH_Bias_A_UserArgs_MT16x16x32fskvF-LdB9pCDb5Lk-YWNWT7yA1ZpgoZCFaV4S_LkaU=', 'err': 0, }), ]), @@ -14,7 +14,7 @@ 'file': 'F4_MX.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_MXAE8B32_MXBE8B32_BH_Bias_HAz_J2wAOUGT1m_1qie4t6SOBIJJoMHRuEepJqyROT6IM=', + 'basename': 'Cijk_Alik_Bljk_F4BS_MXAE8B32_MXBE8B32_BH_Bias_HAuVSexCu7NbiS_hMe_6fkxQ6QFaiFgTbZA-MnUSCg0NM=', 'err': 0, }), ]), @@ -23,7 +23,7 @@ 'file': 'HHS.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_A_UserArgs_MT16x16x322B4FnC0zasYp4cXH3fT-GKmdbjeLh6UCDBMSbnvr9H8=', + 'basename': 'Cijk_Ailk_Bjlk_HHS_BH_Bias_A_UserArgs_MT16x16x32fskvF-LdB9pCDb5Lk-YWNWT7yA1ZpgoZCFaV4S_LkaU=', 'err': 0, }), ]), @@ -32,7 +32,7 @@ 'file': 'SB.yaml', 'kernels': list([ dict({ - 'basename': 'Cijk_Ailk_Bjlk_S_MX_B_Bias_SB_HA_S_SAB_SAV_UserA8iQvvl7Y4IR1E4EWOYJTgkvkJTtcL1ly4QaNKprHPhM=', + 'basename': 'Cijk_Ailk_Bjlk_S_MX_B_Bias_SB_HA_S_SAB_SAV_UserAkNup8pM1Q7eqjciJzU8wtg4ohn_FHaMlYaCGFoZKsWE=', 'err': 0, }), ]), diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr index 6b00a991fb38..cf10d198d9a5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_datamover_char.ambr @@ -2,11 +2,11 @@ # name: test_r3_datamover_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT16x16x128_MI16_QobKpeqHxanvL7qwfLO2v0ata2Bsw5nva5aJLS5RhY=', + 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT16x16x128_MI16MVzLZOqm-oDhwdblMUu9axemtO1vW31Ug1---EVAxT4=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT32x32x128_MI167MEFMPkl0yanICd8aYGkQ-qZqG20dGGqi59GoZn4JvU=', + 'basename': 'Cijk_Alik_Bljk_F4BS_BH_UserArgs_MT32x32x128_MI16dmV4LHVjZgzfC3xHWPtz7ckoJEtYPaKw8vtI2DbVaZQ=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr index 61799447ef6e..6d2cae7bbeb7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_gfx1250_char.ambr @@ -2,7 +2,7 @@ # name: test_r3_streamk_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgNg-IV7t7EnYLL6mzebH3P5CbTswCA1Faj4hopzB-L_A=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgWuFPSmyxgZZS9lNLqQt2ml5kLoTKRLL3JkGmxXB1SBo=', 'err': 0, }), ]) From 61c460babf804a0668b2cd9af88e9ad8fa07efa8 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Fri, 17 Jul 2026 19:44:26 +0000 Subject: [PATCH 18/20] Regenerate test_r3_streamk_tdmsplit_gfx1250 golden after merging develop This _codegen golden did not textually conflict during the merge (only develop touched it, via #9227/#9217), so it auto-merged to develop's content. But the branch's StreamKWorkStealing -> required-parameter promotion (D17) also churns this kernel's basename hash, leaving the golden stale. Diff is basename-hash-only; err:0 unchanged, no instruction-count or emitted-assembly changes. --- .../__snapshots__/test_r3_streamk_tdmsplit_gfx1250_char.ambr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_tdmsplit_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_tdmsplit_gfx1250_char.ambr index 36d436eadb57..8464cbc6e995 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_tdmsplit_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_r3_streamk_tdmsplit_gfx1250_char.ambr @@ -2,7 +2,7 @@ # name: test_r3_streamk_tdmsplit_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgZMQJ-hHyLz5yAyZk2P0zHciaExSDGnjyuEM24FkFLYA=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgRTb8_28SDU2S5aKZHEQM10z4b86hnOEYR4I_Cfu-3L8=', 'err': 0, }), ]) From d23e85c7032ee0d8cf090b06a908a213e7ebbb73 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 16:02:06 +0000 Subject: [PATCH 19/20] Regenerate characterization .ambr snapshots after merging develop --- .../__snapshots__/test_solution_class_char.ambr | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 71229dcd381a..70a11307db3a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -3,7 +3,7 @@ dict({ 'count': 1, 'names': list([ - 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', ]), }) # --- @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 337, + 'len': 338, }) # --- # name: test_solution_construction @@ -295,6 +295,7 @@ 'StreamKAtomic', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', + 'StreamKWorkStealing', 'StreamKXCCMapping', 'SubGroup0', 'SubGroup1', @@ -399,8 +400,8 @@ 'tailLoopOptA', 'tailLoopOptB', ]), - 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 337, + 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'num_keys': 338, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32, From 684ef650f7f9d5a1422292598c48d8295f8431b5 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Fri, 24 Jul 2026 15:26:55 +0000 Subject: [PATCH 20/20] Regenerate characterization .ambr snapshots after merging develop --- .../__snapshots__/test_solution_class_char.ambr | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 1cd33e82b59c..0e124a014f1b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -3,7 +3,7 @@ dict({ 'count': 1, 'names': list([ - 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', ]), }) # --- @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 338, + 'len': 339, }) # --- # name: test_solution_construction @@ -296,6 +296,7 @@ 'StreamKAtomic', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', + 'StreamKWorkStealing', 'StreamKXCCMapping', 'SubGroup0', 'SubGroup1', @@ -400,8 +401,8 @@ 'tailLoopOptA', 'tailLoopOptB', ]), - 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 338, + 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'num_keys': 339, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32,