#5569 [Engine] IQ3_S grouped GEMM: half2-SIMD dequantization + a decode tile, -24%/-36% on sm_70 (bit-exact)

open · @Lennier · 0 commentaires · Sur GitHub

BenchmarksServer & APINVIDIA / CUDAModels & quantsSecurity

Description

### Area

Engine

### Problem

The IQ3_S grouped expert GEMM (`gemm_iq_f16_grouped_kernel<256>`) is latency/instruction-bound on sm_70 at decode shapes, far below what the card can read:

- gate/up (K=2560, n_out=640, 10 experts, rows/expert 1..12) on V100-SXM2-32GB: **195 µs/call = 36 GB/s** effective weight read — 25x below HBM2 (~900 GB/s).
- Causes: `dq_iq3_s` dequantizes byte-by-byte (scattered LDG.8 + `iq3s_grid` lookups through L1, scalar cvt/mul/sign per value); the 128x64 tile needs 84992 B of shared memory -> 1 block/SM; at rows<=12 only 1 of 4 m-tiles is live (48 of 64 accumulator fragments per warp hold zeros).

### Result (patch below, bit-exact)

A standalone harness (`decode_gemm_bench`, random IQ3_S blocks, FNV-1a checksum of Y, reps=50, engine 0.1.58 from v0.2.12, V100-SXM2-32GB):

| shape | baseline | + SIMD dequant | + SIMD & decode tile |
|---|---|---|---|
| e10 rows/e=1..12 | 194.5-197.1 us | 148-149 us (-24%) | 124-125 us (-36%) |
| e20 rows/e=1..12 | 380-390 us | ~290 us | 217-220 us (-43%) |

All Y checksums are **bit-identical** to the baseline in every shape (e.g. `e6b89acee638abc8`, `29ea6e45296fc7ee`, `51f418098d2ed185`).

End-to-end on this box the patch is neutral (agent decode 88.0 vs 87.8 tok/s): decode here is PCIe-bound (`fetch_blobs` saturates the SXM2-adapter link at ~12 GB/s), so the GEMM win is hidden. The upside is real where experts are resident (prefill grouped calls take the same -24%) and on cards whose host link is not the wall.

### Approach

1. **`dq_iq3_s_simd`** - the `w2_dq_iq2_s` pattern moved into the grouped/decode tiles: grid bytes become floats through the magic `0x4B000000`, the FP32 multiply by `dl` is exact (11 + 5 + 8 <= 24 bits), one `__floats2half2_rn` rounds, and the sign XORs into the rounded halves (`round(-x) = -round(x)`). The 512-entry grid and a 16-entry sign-mask nibble table are staged into shared memory (2176 B). Bit-for-bit `cvt<__half>(dl * grid * +-1)` by construction.
2. **`gemm_iq_f16_decode_kernel`** (opt-in `STRATA_DG_DECODE=1`): a 32x64 tile (36480 B smem -> 2 blocks/SM), the 8 warps split one m-tile as 2 column x 4 row fragments; K walks exactly as in the grouped kernel (k0 by WK, xh by WG_XK, kk by 16, same wmma 16x16x16), so every output element accumulates the same k-slices in the same order - bit-identical to the grouped kernel.

Both are gated: SIMD on by default for ty==21 (`STRATA_DG_SIMD=0` falls back), the decode tile only for `max_rows <= 64 && n_out % 32 == 0 && K % 256 == 0`.

### Limits

- It does not remove the PCIe wall: end-to-end decode on a PCIe-heavy box stays where it was; the effective weight read is still ~25x below HBM2 after the patch (latency-bound, not bandwidth-bound).
- The decode tile covers the IQ3_S gate/up shape only (down projections of this pack run through `gemm_iq_f16_w2_kernel<20>`).

### Validation

Harness source is small; happy to add it as `bench/decode_gemm_bench.cpp` if a PR form is preferred. Measured on V100-SXM2-32GB, CUDA 12.8, engine 0.1.58 (v0.2.12) + the patch below; the existing `pf_wmma_parity` suite is untouched (w2 kernels unchanged).

Related: #5523 (same Volta dequantization line - the K/V union finding).

```
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index c70ca9e1..c5d2c4c3 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -6700,11 +6700,40 @@ int hash_slots(const LayerVram& L, const unsigned long long* slot_ptrs, int64_t
 namespace strata::kernels {
 namespace {
 constexpr int WG_BM = 64, WG_BN = 128, WG_THREADS = 256, WG_XK = 128, WG_MT = 4;
+
+// w2's byte-to-float magic and half2 packing (up here so the grouped/decode weight tiles use them too)
+__device__ __forceinline__ uint32_t w2_u32(half2 h) { uint32_t v; memcpy(&v, &h, 4); return v; }
+__device__ __forceinline__ float w2_b2f(uint32_t w, int j) { return __int_as_float(0x4B000000 | ((w >> 8 * j) & 0xff)) - 8388608.0f; }
+
+// IQ3_S dequantization for the grouped/decode tiles, the w2_dq_iq2_s pattern: the grid's bytes become floats through
+// the magic 0x4B000000, the FP32 multiply by dl is EXACT (11 + 5 + 8 <= 24 bits), one RN to half2, and the sign
+// XORs into the rounded halves (round(-x) = -round(x)). Bit for bit dq_iq3_s's cvt<__half>(dl * grid * +-1), with
+// the grid read from shared memory (tab, iq3s_grid as is) and the sign masks from a 16-entry nibble table (sgn).
+__device__ __forceinline__ void dq_iq3_s_simd(const uint8_t* __restrict__ w, int ibs, __half* o, int tid,
+                                              const uint32_t* __restrict__ tab, const uint2* __restrict__ sgn) {
+    const block_iq3_s* x = (const block_iq3_s*) w + ibs;
+    const int il = tid / 8, ib = tid % 8;
+    const uint8_t* qs = x->qs + 8 * ib;
+    const int i1 = qs[2 * il + 0] | ((x->qh[ib] << (8 - 2 * il)) & 256);
+    const int i2 = qs[2 * il + 1] | ((x->qh[ib] << (7 - 2 * il)) & 256);
+    const float dl = (float) x->d * (1 + 2 * ((x->scales[ib / 2] >> 4 * (ib % 2)) & 0xf));
+    const uint32_t g1 = tab[i1], g2 = tab[i2];
+    const uint8_t sg = x->signs[4 * ib + il];
+    const uint2 m0 = sgn[sg & 15], m1 = sgn[sg >> 4];
+    uint4 out;
+    out.x = w2_u32(__floats2half2_rn(dl * w2_b2f(g1, 0), dl * w2_b2f(g1, 1))) ^ m0.x;
+    out.y = w2_u32(__floats2half2_rn(dl * w2_b2f(g1, 2), dl * w2_b2f(g1, 3))) ^ m0.y;
+    out.z = w2_u32(__floats2half2_rn(dl * w2_b2f(g2, 0), dl * w2_b2f(g2, 1))) ^ m1.x;
+    out.w = w2_u32(__floats2half2_rn(dl * w2_b2f(g2, 2), dl * w2_b2f(g2, 3))) ^ m1.y;
+    *reinterpret_cast<uint4*>(o + 32 * ib + 8 * il) = out;
+}
 // A block: BN = 128 weight rows of one expert x up to WG_MT * BM = 256 of its token rows (blockIdx.y walks an expert's
 // rows in such chunks), so every weight superblock is dequantized ONCE per chunk into shared memory (WK 256: an IQ
 // superblock through dq_dispatch; WK 128: two Q2_0 blocks, or four IQ4_NL blocks); the activations stream through a
 // BM x 128 tile.  8 warps of 32 x 32 per m-tile (2 x 2 fragments), FP32 accumulators for the chunk's m-tiles in registers.
-template<int WK, bool Q20, bool IQNL = false>
+// BN/BM/MT are template parameters: the decode instantiation (64, 16, 1) halves the weight tile and drops the dead
+// m-tiles, so two blocks fit one SM and their dequantization overlaps each other's tensor-core work.
+template<int WK, bool Q20, bool IQNL = false, int BN = WG_BN, int BM = WG_BM, int MT = WG_MT, bool SIMD = false>
 __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty, const uint8_t* __restrict__ W,
                                                                          size_t expert_bytes, size_t row_bytes, int K,
                                                                          const __half* __restrict__ X, int ldx,
@@ -6715,39 +6744,52 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
     constexpr int LDW = WK + 8, LDX = WG_XK + 8;
     extern __shared__ __align__(16) unsigned char wg_smem[];
     __half* sw = reinterpret_cast<__half*>(wg_smem);            // [BN][LDW]
-    __half* sx = sw + WG_BN * LDW;                               // [BM][LDX]
+    __half* sx = sw + BN * LDW;                                  // [BM][LDX]
     const int e = blockIdx.y / ch_per_e, ch = blockIdx.y % ch_per_e;
-    const int c0 = bounds[e] + ch * WG_MT * WG_BM, r1 = bounds[e + 1];
+    const int c0 = bounds[e] + ch * MT * BM, r1 = bounds[e + 1];
     if (c0 >= r1) return;
-    const int crow = min(WG_MT * WG_BM, r1 - c0);
-    const int nmt = (crow + WG_BM - 1) / WG_BM;
-    const int n0 = blockIdx.x * WG_BN;
+    const int crow = min(MT * BM, r1 - c0);
+    const int nmt = (crow + BM - 1) / BM;
+    const int n0 = blockIdx.x * BN;
     const uint8_t* we = W + (size_t) e * expert_bytes;
     const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31;
     const int wm = warp >> 2, wn = warp & 3;                     // rows wm*32, cols wn*32 of an m-tile
-    wmma::fragment<wmma::accumulator, 16, 16, 16, float> acc[WG_MT][2][2];
+    wmma::fragment<wmma::accumulator, 16, 16, 16, float> acc[MT][2][2];
 #pragma unroll
-    for (int t = 0; t < WG_MT; ++t)
+    for (int t = 0; t < MT; ++t)
 #pragma unroll
         for (int a = 0; a < 2; ++a)
 #pragma unroll
             for (int b = 0; b < 2; ++b) wmma::fill_fragment(acc[t][a][b], 0.0f);
     // the activation tiles in order (k0, m-tile, half): the next one's 4 uint4 per thread are loaded into registers
     // while the current one is multiplied (the first before the weights' dequantization)
-    constexpr int XQ = WG_BM * (WG_XK / 8) / WG_THREADS;        // 4
+    constexpr int XQ = BM * (WG_XK / 8) / WG_THREADS;           // 4 at BM 64, 1 at BM 16
     const int nx = WK / WG_XK, per_k = nmt * nx, total = (K / WK) * per_k;
     uint4 xr[XQ];
     auto load_x = [&](int it) {
         const int kb = it / per_k, rem = it % per_k, t = rem / nx, xh = rem % nx;
-        const int rows = min(WG_BM, crow - t * WG_BM);
+        const int rows = min(BM, crow - t * BM);
 #pragma unroll
         for (int q = 0; q < XQ; ++q) {
             const int i = tid + q * WG_THREADS, r = i / (WG_XK / 8), c = (i % (WG_XK / 8)) * 8;
-            xr[q] = r < rows ? *reinterpret_cast<const uint4*>(X + (size_t) (c0 + t * WG_BM + r) * ldx + kb * WK + xh * WG_XK + c)
+            xr[q] = r < rows ? *reinterpret_cast<const uint4*>(X + (size_t) (c0 + t * BM + r) * ldx + kb * WK + xh * WG_XK + c)
                              : make_uint4(0, 0, 0, 0);
         }
     };
     load_x(0);
+    // the SIMD path's tables: the IQ3_S grid as is (the magic is applied at use) and the 16 sign-mask nibbles
+    uint32_t* s_tab = SIMD ? reinterpret_cast<uint32_t*>(sx + BM * LDX) : nullptr;
+    uint2* s_sgn = SIMD ? reinterpret_cast<uint2*>(s_tab + 512) : nullptr;
+    if (SIMD) {
+        for (int i = tid; i < 512; i += WG_THREADS) s_tab[i] = iq3s_grid[i];
+        if (tid < 16) {
+            uint2 m;
+            m.x = (tid & 1 ? 0x8000u : 0u) | (tid & 2 ? 0x80000000u : 0u);
+            m.y = (tid & 4 ? 0x8000u : 0u) | (tid & 8 ? 0x80000000u : 0u);
+            s_sgn[tid] = m;
+        }
+        __syncthreads();
+    }
     int it = 0;
     for (int k0 = 0; k0 < K; k0 += WK) {
         if constexpr (IQNL) {
@@ -6767,8 +6809,9 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
                 }
             }
         } else if constexpr (!Q20) {
-            for (int r = warp; r < WG_BN; r += WG_THREADS / 32)
-                dq_dispatch<__half>(ty, we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane);
+            for (int r = warp; r < BN; r += WG_THREADS / 32)
+                if constexpr (SIMD) dq_iq3_s_simd(we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane, s_tab, s_sgn);
+                else dq_dispatch<__half>(ty, we + (size_t) (n0 + r) * row_bytes, k0 / 256, sw + r * LDW, lane);
         } else {
             const int r = tid >> 1, blk = tid & 1;   // one thread per (row, 64-value block)
             const block_q2_0* b = reinterpret_cast<const block_q2_0*>(we + (size_t) (n0 + r) * row_bytes) + k0 / 64 + blk;
@@ -6836,6 +6879,101 @@ __global__ void __launch_bounds__(WG_THREADS) gemm_iq_f16_grouped_kernel(int ty,
 #endif
 }
 
+// ---- STRATA_DG_DECODE=1 (opt-in): the grouped kernel's decode shape - at most 16 token rows per expert, so the
+// grouped tile's WG_MT = 4 m-tiles are dead weight (48 of its 64 accumulator fragments per warp hold zeros, and its
+// 84992-byte tile keeps one block per SM). A decode block: DG_BN = 32 weight rows x DG_BM = 64 token rows of one
+// expert, 34304 bytes of shared memory -> two blocks per SM, so one block's dq_dispatch overlaps the other's
+// tensor-core work. The 8 warps split the single m-tile as 2 column fragments x 4 row fragments (wn = wa

Sur le site

Liens install, modèles, releases.