#5569 [Engine] IQ3_S grouped GEMM: half2-SIMD dequantization + a decode tile, -24%/-36% on sm_70 (bit-exact)
open · @Lennier · 0 comentarios · En GitHub
BenchmarksServer & APINVIDIA / CUDAModels & quantsSecurity
Descripción
### Area
Engine
### Problem
The IQ3_S grouped expert GEMM (`gemm_iq_f16_grouped_kernel<256>`) is latency/instruction-bound on sm_70 at decode shapes, far below what the card can read:
- gate/up (K=2560, n_out=640, 10 experts, rows/expert 1..12) on V100-SXM2-32GB: **195 µs/call = 36 GB/s** effective weight read — 25x below HBM2 (~900 GB/s).
- Causes: `dq_iq3_s` dequantizes byte-by-byte (scattered LDG.8 + `iq3s_grid` lookups through L1, scalar cvt/mul/sign per value); the 128x64 tile needs 84992 B of shared memory -> 1 block/SM; at rows<=12 only 1 of 4 m-tiles is live (48 of 64 accumulator fragments per warp hold zeros).
### Result (patch below, bit-exact)
A standalone harness (`decode_gemm_bench`, random IQ3_S blocks, FNV-1a checksum of Y, reps=50, engine 0.1.58 from v0.2.12, V100-SXM2-32GB):
| shape | baseline | + SIMD dequant | + SIMD & decode tile |
|---|---|---|---|
| e10 rows/e=1..12 | 194.5-197.1 us | 148-149 us (-24%) | 124-125 us (-36%) |
| e20 rows/e=1..12 | 380-390 us | ~290 us | 217-220 us (-43%) |
All Y checksums are **bit-identical** to the baseline in every shape (e.g. `e6b89acee638abc8`, `29ea6e45296fc7ee`, `51f418098d2ed185`).
End-to-end on this box the patch is neutral (agent decode 88.0 vs 87.8 tok/s): decode here is PCIe-bound (`fetch_blobs` saturates the SXM2-adapter link at ~12 GB/s), so the GEMM win is hidden. The upside is real where experts are resident (prefill grouped calls take the same -24%) and on cards whose host link is not the wall.
### Approach
1. **`dq_iq3_s_simd`** - the `w2_dq_iq2_s` pattern moved into the grouped/decode tiles: grid bytes become floats through the magic `0x4B000000`, the FP32 multiply by `dl` is exact (11 + 5 + 8 <= 24 bits), one `__floats2half2_rn` rounds, and the sign XORs into the rounded halves (`round(-x) = -round(x)`). The 512-entry grid and a 16-entry sign-mask nibble table are staged into shared memory (2176 B). Bit-for-bit `cvt<__half>(dl * grid * +-1)` by construction.
2. **`gemm_iq_f16_decode_kernel`** (opt-in `STRATA_DG_DECODE=1`): a 32x64 tile (36480 B smem -> 2 blocks/SM), the 8 warps split one m-tile as 2 column x 4 row fragments; K walks exactly as in the grouped kernel (k0 by WK, xh by WG_XK, kk by 16, same wmma 16x16x16), so every output element accumulates the same k-slices in the same order - bit-identical to the grouped kernel.
Both are gated: SIMD on by default for ty==21 (`STRATA_DG_SIMD=0` falls back), the decode tile only for `max_rows <= 64 && n_out % 32 == 0 && K % 256 == 0`.
### Limits
- It does not remove the PCIe wall: end-to-end decode on a PCIe-heavy box stays where it was; the effective weight read is still ~25x below HBM2 after the patch (latency-bound, not bandwidth-bound).
- The decode tile covers the IQ3_S gate/up shape only (down projections of this pack run through `gemm_iq_f16_w2_kernel<20>`).
### Validation
Harness source is small; happy to add it as `bench/decode_gemm_bench.cpp` if a PR form is preferred. Measured on V100-SXM2-32GB, CUDA 12.8, engine 0.1.58 (v0.2.12) + the patch below; the existing `pf_wmma_parity` suite is untouched (w2 kernels unchanged).
Related: #5523 (same Volta dequantization line - the K/V union finding).
```
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index c70ca9e1..c5d2c4c3 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -6700,11 +6700,40 @@ int hash_slots(const LayerVram& L, const unsigned long long* slot_ptrs, int64_t
namespace strata::kernels {
namespace {
constexpr int WG_BM = 64, WG_BN = 128, WG_THREADS = 256, WG_XK = 128, WG_MT = 4;
+
+// w2's byte-to-float magic and half2 packing (up here so the grouped/decode weight tiles use them too)
+__device__ __forceinline__ uint32_t w2_u32(half2 h) { uint32_t v; memcpy(&v, &h, 4); return v; }
+__device__ __forceinline__ float w2_b2f(uint32_t w, int j) { return __int_as_float(0x4B000000 | ((w >> 8 * j) & 0xff)) - 8388608.0f; }
+
+// IQ3_S dequantization for the grouped/decode tiles, the w2_dq_iq2_s pattern: the grid's bytes become floats through
+// the magic 0x4B000000, the FP32 multiply by dl is EXACT (11 + 5 + 8 <= 24 bits), one RN to half2, and the sign
+// XORs into the rounded halves (round(-x) = -round(x)). Bit for bit dq_iq3_s's cvt<__half>(dl * grid * +-1), with
+// the grid read from shared memory (tab, iq3s_grid as is) and the sign masks from a 16-entry nibble table (sgn).
+__device__ __forceinline__ void dq_iq3_s_simd(const uint8_t* __restrict__ w, int ibs, __half* o, int tid,
+ const uint32_t* __restrict__ tab, const uint2* __restrict__ sgn) {
+ const block_iq3_s* x = (const block_iq3_s*) w + ibs;
+ const int il = tid / 8, ib = tid % 8;
+ const uint8_t* qs = x->qs + 8 * ib;
+ const int i1 = qs[2 * il + 0] | ((x->qh[ib] << (8 - 2 * il)) & 256);
+ const int i2 = qs[2 * il + 1] | ((x->qh[ib] << (7 - 2 * il)) & 256);
+ const float dl = (float) x->d * (1 + 2 * ((x->scales[ib / 2] >> 4 * (ib % 2)) & 0xf));
+ const uint32_t g1 = tab[i1], g2 = tab[i2];
+ const uint8_t sg = x->signs[4 * ib + il];
+ const uint2 m0 = sgn[sg & 15], m1 = sgn[sg >> 4];
+ uint4 out;
+ out.x = w2_u32(__floats2half2_rn(dl * w2_b2f(g1, 0), dl * w2_b2f(g1, 1))) ^ m0.x;
+ out.y = w2_u32(__floats2half2_rn(dl * w2_b2f(g1, 2), dl * w2_b2f(g1, 3))) ^ m0.y;
+ out.z = w2_u32(__floats2half2_rn(dl * w2_b2f(g2, 0), dl * w2_b2f(g2, 1))) ^ m1.x;
+ out.w = w2_u32(__floats2half2_rn(dl * w2_b2f(g2, 2), dl * w2_b2f(g2, 3))) ^ m1.y;
+ *reinterpret_cast<uint4*>(o + 32 * ib + 8 * il) = out;
+}
// A block: BN = 128 weight rows of one expert x up to WG_MT * BM = 256 of its token rows (blockIdx.y walks an expert's
// rows in such chunks), so every weight superblock is dequantized ONCE per chunk into shared memory (WK 256: an IQ
// superblock through dq_dispatch; WK 128: two Q2_0 blocks, or four IQ4_NL blocks); the activations stream through a
// BM x 128 tile. 8 warps of 32 x 32 per m-tile (2 x 2 fragments), FP32 accumulators for the chunk's m-tiles in registers.
-template<int WK, bool Q20, bool IQNL = false>
+// BN/BM/MT are template parameters: the decode instantiation (64, 16, 1) halves the weight tile and drops the dead
+// m-tiles, so two blocks fit one SM and their dequantization overlaps each other's tensor-core work.
+template<int WK, bool Q20, bool IQNL = false, int BN = WG_BN, int BM = WG_BM, int MT = WG_MT, bool SIMD = false>
__global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty, const uint8_t* __restrict__ W,
size_t expert_bytes, size_t row_bytes, int K,
const __half* __restrict__ X, int ldx,
@@ -6715,39 +6744,52 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
constexpr int LDW = WK + 8, LDX = WG_XK + 8;
extern __shared__ __align__(16) unsigned char wg_smem[];
__half* sw = reinterpret_cast<__half*>(wg_smem); // [BN][LDW]
- __half* sx = sw + WG_BN * LDW; // [BM][LDX]
+ __half* sx = sw + BN * LDW; // [BM][LDX]
const int e = blockIdx.y / ch_per_e, ch = blockIdx.y % ch_per_e;
- const int c0 = bounds[e] + ch * WG_MT * WG_BM, r1 = bounds[e + 1];
+ const int c0 = bounds[e] + ch * MT * BM, r1 = bounds[e + 1];
if (c0 >= r1) return;
- const int crow = min(WG_MT * WG_BM, r1 - c0);
- const int nmt = (crow + WG_BM - 1) / WG_BM;
- const int n0 = blockIdx.x * WG_BN;
+ const int crow = min(MT * BM, r1 - c0);
+ const int nmt = (crow + BM - 1) / BM;
+ const int n0 = blockIdx.x * BN;
const uint8_t* we = W + (size_t) e * expert_bytes;
const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31;
const int wm = warp >> 2, wn = warp & 3; // rows wm*32, cols wn*32 of an m-tile
- wmma::fragment<wmma::accumulator, 16, 16, 16, float> acc[WG_MT][2][2];
+ wmma::fragment<wmma::accumulator, 16, 16, 16, float> acc[MT][2][2];
#pragma unroll
- for (int t = 0; t < WG_MT; ++t)
+ for (int t = 0; t < MT; ++t)
#pragma unroll
for (int a = 0; a < 2; ++a)
#pragma unroll
for (int b = 0; b < 2; ++b) wmma::fill_fragment(acc[t][a][b], 0.0f);
// the activation tiles in order (k0, m-tile, half): the next one's 4 uint4 per thread are loaded into registers
// while the current one is multiplied (the first before the weights' dequantization)
- constexpr int XQ = WG_BM * (WG_XK / 8) / WG_THREADS; // 4
+ constexpr int XQ = BM * (WG_XK / 8) / WG_THREADS; // 4 at BM 64, 1 at BM 16
const int nx = WK / WG_XK, per_k = nmt * nx, total = (K / WK) * per_k;
uint4 xr[XQ];
auto load_x = [&](int it) {
const int kb = it / per_k, rem = it % per_k, t = rem / nx, xh = rem % nx;
- const int rows = min(WG_BM, crow - t * WG_BM);
+ const int rows = min(BM, crow - t * BM);
#pragma unroll
for (int q = 0; q < XQ; ++q) {
const int i = tid + q * WG_THREADS, r = i / (WG_XK / 8), c = (i % (WG_XK / 8)) * 8;
- xr[q] = r < rows ? *reinterpret_cast<const uint4*>(X + (size_t) (c0 + t * WG_BM + r) * ldx + kb * WK + xh * WG_XK + c)
+ xr[q] = r < rows ? *reinterpret_cast<const uint4*>(X + (size_t) (c0 + t * BM + r) * ldx + kb * WK + xh * WG_XK + c)
: make_uint4(0, 0, 0, 0);
}
};
load_x(0);
+ // the SIMD path's tables: the IQ3_S grid as is (the magic is applied at use) and the 16 sign-mask nibbles
+ uint32_t* s_tab = SIMD ? reinterpret_cast<uint32_t*>(sx + BM * LDX) : nullptr;
+ uint2* s_sgn = SIMD ? reinterpret_cast<uint2*>(s_tab + 512) : nullptr;
+ if (SIMD) {
+ for (int i = tid; i < 512; i += WG_THREADS) s_tab[i] = iq3s_grid[i];
+ if (tid < 16) {
+ uint2 m;
+ m.x = (tid & 1 ? 0x8000u : 0u) | (tid & 2 ? 0x80000000u : 0u);
+ m.y = (tid & 4 ? 0x8000u : 0u) | (tid & 8 ? 0x80000000u : 0u);
+ s_sgn[tid] = m;
+ }
+ __syncthreads();
+ }
int it = 0;
for (int k0 = 0; k0 < K; k0 += WK) {
if constexpr (IQNL) {
@@ -6767,8 +6809,9 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
}
}
} else if constexpr (!Q20) {
- for (int r = warp; r < WG_BN; r += WG_THREADS / 32)
- dq_dispatch<__half>(ty, we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane);
+ for (int r = warp; r < BN; r += WG_THREADS / 32)
+ if constexpr (SIMD) dq_iq3_s_simd(we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane, s_tab, s_sgn);
+ else dq_dispatch<__half>(ty, we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane);
} else {
const int r = tid >> 1, blk = tid & 1; // one thread per (row, 64-value block)
const block_q2_0* b = reinterpret_cast<const block_q2_0*>(we + (size_t) (n0 + r) * row_bytes) + k0 / 64 + blk;
@@ -6836,6 +6879,101 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
#endif
}
+// ---- STRATA_DG_DECODE=1 (opt-in): the grouped kernel's decode shape - at most 16 token rows per expert, so the
+// grouped tile's WG_MT = 4 m-tiles are dead weight (48 of its 64 accumulator fragments per warp hold zeros, and its
+// 84992-byte tile keeps one block per SM). A decode block: DG_BN = 32 weight rows x DG_BM = 64 token rows of one
+// expert, 34304 bytes of shared memory -> two blocks per SM, so one block's dq_dispatch overlaps the other's
+// tensor-core work. The 8 warps split the single m-tile as 2 column fragments x 4 row fragments (wn = waEn el sitio
Enlaces a install, modelos, releases.