fix(qwen35) span the whole KV pool for slot-mapped verify attention

server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -473,14 +473,16 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
         case GGML_OP_PAGED_ATTN:
             return false;
         case GGML_OP_SSM_CONV:
-            // The Specla layout (op param 0 == 1) needs the packed HLD state and
-            // is only supported by the CUDA kernel; the generic CPU kernel would
-            // silently compute garbage.
-            return ggml_get_op_params_i32(op, 0) != 1;
+            // Every nonzero mode is a dflash CUDA/HIP extension (SpecLA,
+            // fused step, or dynamic conv). The generic CPU kernel only
+            // implements the original mode and asserts if one reaches it.
+            return ggml_get_op_params_i32(op, 0) == 0;
         case GGML_OP_GATED_DELTA_NET:
             // The Specla GDN variant (op param 2 == 1) is stateful via HLD and is
-            // only supported by the CUDA kernel.
-            return ggml_get_op_params_i32(op, 2) != 1;
+            // only supported by CUDA/HIP. Raw-gate mode is also CUDA/HIP-only:
+            // the CPU kernel expects beta/g to have already been transformed.
+            return ggml_get_op_params_i32(op, 2) != 1 &&
+                   ggml_get_op_params_i32(op, 10) == 0 && op->src[9] == nullptr;
         case GGML_OP_OUT_PROD:
             return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
                 src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;

server/scripts/convert_dflash_to_gguf.py

@@ -29,6 +29,7 @@
 
 import argparse
 import json
+import math
 import struct
 import sys
 from pathlib import Path
@@ -122,6 +123,16 @@ def pick(*keys):
                 a["yarn_orig_ctx"]  = int(rp.get("original_max_position_embeddings")
                                           or c.get("original_max_position_embeddings")
                                           or 0)
+                attn_factor = rp.get("attention_factor")
+                # GGML applies YaRN's standard 1 + 0.1*log(factor) magnitude
+                # internally. HF attention_factor is the final magnitude, so
+                # normalize only an explicit override; 1.0 retains GGML's
+                # standard default when the config omits it.
+                ggml_mscale = (1.0 + 0.1 * math.log(a["yarn_factor"])
+                               if a["yarn_factor"] > 1.0 else 1.0)
+                a["yarn_attn_factor"] = (
+                    float(attn_factor) / ggml_mscale
+                    if attn_factor is not None else 1.0)
                 a["yarn_beta_fast"] = float(rp.get("beta_fast", 32.0))
                 a["yarn_beta_slow"] = float(rp.get("beta_slow", 1.0))
         if dfc.get("mask_token_id") is not None:
@@ -528,6 +539,7 @@ def main():
         writer.add_string(f"{ARCH}.rope.scaling.type", "yarn")
         writer.add_float32(f"{ARCH}.rope.scaling.factor", a["yarn_factor"])
         writer.add_uint32(f"{ARCH}.rope.scaling.original_context_length", a["yarn_orig_ctx"])
+        writer.add_float32(f"{ARCH}.rope.scaling.attn_factor", a["yarn_attn_factor"])
         writer.add_float32(f"{ARCH}.rope.scaling.beta_fast", a["yarn_beta_fast"])
         writer.add_float32(f"{ARCH}.rope.scaling.beta_slow", a["yarn_beta_slow"])
 

server/src/common/dflash2_head.cpp

@@ -115,8 +115,17 @@ bool dflash2_score_candidates(const DraftWeights & dw,
         ggml_set_input(g.inp_succ);
         ggml_set_input(g.inp_pred);
         g.hproj = ggml_mul_mat(g.ctx, sel.hproj, g.inp_hidden);      // [rank, n_cand]
-        g.succ  = ggml_get_rows(g.ctx, sel.succ_cb, g.inp_succ);      // [rank, n_cand*K] f32
-        g.pred  = ggml_get_rows(g.ctx, sel.pred_cb, g.inp_pred);      // [rank, 1+n_cand*K] f32
+        // Current ggml_get_rows dequantizes floating-point/quantized sources
+        // to F32. Keep the explicit fallback so float readback remains safe
+        // if that API later starts preserving F16/BF16 source types.
+        auto get_rows_f32 = [&](ggml_tensor * codebook, ggml_tensor * ids) {
+            ggml_tensor * rows = ggml_get_rows(g.ctx, codebook, ids);
+            return rows->type == GGML_TYPE_F32
+                ? rows : ggml_cast(g.ctx, rows, GGML_TYPE_F32);
+        };
+        g.succ  = get_rows_f32(sel.succ_cb, g.inp_succ);      // [rank, n_cand*K] f32
+        g.pred  = get_rows_f32(sel.pred_cb, g.inp_pred);      // [rank, 1+n_cand*K] f32
+        GGML_ASSERT(g.succ->type == GGML_TYPE_F32 && g.pred->type == GGML_TYPE_F32);
         ggml_set_output(g.hproj);
         ggml_set_output(g.succ);
         ggml_set_output(g.pred);

server/src/common/dspark_head.cpp

@@ -361,22 +361,21 @@ bool dspark_markov_correct_greedy_chain_fused(const DraftWeights & dw,
     draft_tok.assign((size_t)q_len, 0);
     draft_tok[0] = last_tok;
     // One synchronize instead of n_cand blocking readbacks.
-    int32_t t_out[16];
-    float c_out[16] = {};
-    const int n_get = n_cand < 16 ? n_cand : 16;
-    for (int i = 0; i < n_get; ++i) {
+    std::vector<int32_t> t_out((size_t)n_cand);
+    std::vector<float> c_out(want_confidence ? (size_t)n_cand : 0);
+    for (int i = 0; i < n_cand; ++i) {
         ggml_backend_tensor_get_async(backend, g.toks[(size_t)i], &t_out[i], 0, sizeof(int32_t));
         if (want_confidence && g.confidence[(size_t)i]) {
             ggml_backend_tensor_get_async(
                 backend, g.confidence[(size_t)i], &c_out[i], 0, sizeof(float));
         }
     }
     ggml_backend_synchronize(backend);
-    for (int i = 0; i < n_get; ++i) {
+    for (int i = 0; i < n_cand; ++i) {
         draft_tok[(size_t)i + 1] = t_out[i];
     }
     if (want_confidence && !g.confidence.empty() && g.confidence[0]) {
-        confidence_out->assign(c_out, c_out + n_get);
+        *confidence_out = std::move(c_out);
     }
     ggml_free(g.ctx);
     return true;

server/src/draft/draft_gguf_loader.cpp

@@ -522,11 +522,39 @@ bool load_draft_gguf(const std::string & path,
             }
             out.conv_kernel_size = conv_k;
             out.conv_group_size  = (int)read_u32("dflash.dflash2.conv_group_size", 16);
-            const DraftLayer & L0 = out.layers[0];
+            if (out.conv_group_size <= 0 || out.n_embd % out.conv_group_size != 0) {
+                char b[192];
+                std::snprintf(b, sizeof(b),
+                              "draft GGUF: dflash.dflash2.conv_group_size=%d "
+                              "must be positive and divide embedding_length=%d",
+                              out.conv_group_size, out.n_embd);
+                set_last_error(b);
+                ggml_free(meta_ctx); out.ctx = nullptr; gguf_free(gctx);
+                return false;
+            }
             const int64_t groups = out.n_embd / out.conv_group_size;
             char shape_err[192];
-            if (!check_shape_3d(L0.attn_conv.base, out.n_embd, conv_k, 2, "attn_conv.base", shape_err, sizeof(shape_err)) ||
-                !check_shape_2d(L0.attn_conv.proj, out.n_embd, 2 * conv_k * groups, "attn_conv.proj", shape_err, sizeof(shape_err))) {
+            bool shapes_ok = true;
+            for (int il = 0; il < out.n_layer && shapes_ok; ++il) {
+                const DraftLayer & L = out.layers[(size_t)il];
+                const DraftConvWeights * convs[] = {&L.attn_conv, &L.mlp_conv};
+                const char * kinds[] = {"attn_conv", "ffn_conv"};
+                for (int ci = 0; ci < 2 && shapes_ok; ++ci) {
+                    char base_name[64];
+                    char proj_name[64];
+                    std::snprintf(base_name, sizeof(base_name),
+                                  "blk.%d.%s.base", il, kinds[ci]);
+                    std::snprintf(proj_name, sizeof(proj_name),
+                                  "blk.%d.%s.proj", il, kinds[ci]);
+                    shapes_ok =
+                        check_shape_3d(convs[ci]->base, out.n_embd, conv_k, 2,
+                                       base_name, shape_err, sizeof(shape_err)) &&
+                        check_shape_2d(convs[ci]->proj, out.n_embd,
+                                       2 * conv_k * groups, proj_name,
+                                       shape_err, sizeof(shape_err));
+                }
+            }
+            if (!shapes_ok) {
                 set_last_error(shape_err);
                 ggml_free(meta_ctx); out.ctx = nullptr; gguf_free(gctx);
                 return false;

server/src/qwen35/qwen35_target_graph.cpp

@@ -41,6 +41,8 @@
 #include "common/specla_mode.h"
 
 #include "ggml-alloc.h"
+#include "ggml-backend-impl.h"
+#include "ggml-cuda.h"
 
 #include <cmath>
 #include <cstdio>
@@ -75,6 +77,24 @@ constexpr float EPS         = 1e-6f;
 constexpr float ROPE_THETA  = 10000000.0f;
 }  // namespace q35
 
+// CUDA and ROCm share ggml's CUDA backend interface. Tensor-parallel caches
+// use a meta backend, so inspect every rank-local backend before enabling ops
+// that have no CPU/Metal/Vulkan implementation.
+static bool supports_qwen35_fused_kernels(ggml_backend_t backend) {
+    if (ggml_backend_is_cuda(backend)) return true;
+    if (!ggml_backend_is_meta(backend)) return false;
+
+    const size_t n_backends = ggml_backend_meta_n_backends(backend);
+    if (n_backends == 0) return false;
+    for (size_t i = 0; i < n_backends; ++i) {
+        if (!ggml_backend_is_cuda(
+                ggml_backend_meta_simple_backend(backend, i))) {
+            return false;
+        }
+    }
+    return true;
+}
+
 // ─── TargetCache allocation ─────────────────────────────────────────
 
 bool create_target_cache(const TargetWeights & w,
@@ -1341,6 +1361,18 @@ static ggml_tensor * build_full_attn_block(
             // Never view past the read tensor (its rows may not be 256-aligned).
             win_len_padded = std::min(win_len_padded, (int)cache_k->ne[1]);
         }
+        // kvflash: KV lives at pool SLOTS, and the caller's mask is built in
+        // slot space over the whole pool. Slot indices are not bounded by the
+        // logical context length, so a view sized from kv_start can end below
+        // slots the mask still marks visible: those rows fall outside the
+        // view and the softmax row degenerates, which surfaces as an argmax
+        // of -1 for every verify row past the first. Span the whole pool
+        // instead; the mask, sized from that same pool, is what decides which
+        // slots are readable. Detect the mode by the pair only slot-mapped
+        // verify sets: a set_rows KV write together with an explicit mask.
+        if (kv_write_rows != nullptr && attn_mask != nullptr) {
+            win_len_padded = (int)cache_k->ne[1];
+        }
 
         // K and V from cache: a windowed view starting at win_start.
         ggml_tensor * Kfa = ggml_view_3d(ctx, cache_k,
@@ -1398,6 +1430,7 @@ static ggml_tensor * build_delta_net_block(
     DeltaNetCapture * cap,        // optional: populated on capture_delta_intermediate
     ggml_tensor * parent_ids,     // optional [n_tokens] i32; tree mode when non-null
     bool skip_gdn_intermediate,
+    bool fused_kernel_backend,    // CUDA/HIP backend implements fused conv/raw gates
     // Supported shapes are one sequence with any number of timesteps
     // (prefill/verify), or compact decode with one timestep per mapped row.
     int n_seqs = 1,
@@ -1525,7 +1558,8 @@ static ggml_tensor * build_delta_net_block(
     }();
     const bool chunked_call = chunked_env_on && can_skip_gdn_intermediate && !ragged &&
         !active_slot_ids && !use_specla_factorized && !use_specla_hld && n_tokens > 1;
-    const bool fused_plain = fused_kernels_env && !parent_ids && !ragged && !active_slot_ids &&
+    const bool fused_plain = fused_kernels_env && fused_kernel_backend &&
+                             !parent_ids && !ragged && !active_slot_ids &&
                              !use_specla_factorized && !use_specla_hld;
     const bool fused_conv  = fused_plain;
     const bool raw_gates   = fused_plain && !chunked_call && L.ssm_gate_ba != nullptr;
@@ -2076,7 +2110,8 @@ static ggml_tensor * build_single_layer(
         cur = build_delta_net_block(ctx, gf, w, L, cur,
                                     cache.conv_state[dn_idx], cache.ssm_state[dn_idx],
                                     n_tokens, cap_ptr, parent_ids,
-                                    /*skip_gdn_intermediate=*/true);
+                                    /*skip_gdn_intermediate=*/true,
+                                    supports_qwen35_fused_kernels(cache.backend));
     }
 
     cur = ggml_add(ctx, cur, inpSA);
@@ -2270,6 +2305,7 @@ QwenGraphOutputs build_qwen35_graph(
                                         conv_st, ssm_st,
                                         n_tokens, cap_ptr, in.parent_ids,
                                         /*skip_gdn_intermediate=*/true,
+                                        supports_qwen35_fused_kernels(cache.backend),
                                         in.n_seqs,
                                         in.prefill_segments,
                                         in.n_prefill_segments,
@@ -2477,7 +2513,8 @@ QwenLayerPrefnOutputs build_qwen35_layer_prefn(
         cur = build_delta_net_block(ctx, gf, w, L, cur,
                                     cache.conv_state[dn_idx], cache.ssm_state[dn_idx],
                                     n_tokens, nullptr, nullptr,
-                                    skip_gdn_intermediate);
+                                    skip_gdn_intermediate,
+                                    supports_qwen35_fused_kernels(cache.backend));
     }
 
     cur = ggml_add(ctx, cur, inpSA);

server/test/test_batched_gdn.cpp

@@ -212,6 +212,73 @@ bool run_conv(ggml_backend_t backend, int n_seqs,
     return ok;
 }
 
+bool test_cpu_rejects_gpu_only_extensions(ggml_backend_t backend) {
+    ggml_init_params params{};
+    params.mem_size = 2 * 1024 * 1024;
+    params.no_alloc = true;
+    ggml_context * ctx = ggml_init(params);
+    if (!ctx) return false;
+
+    ggml_tensor * sx = ggml_new_tensor_3d(
+        ctx, GGML_TYPE_F32, D_CONV, CONV_CHANNELS, 1);
+    ggml_tensor * conv_w = ggml_new_tensor_2d(
+        ctx, GGML_TYPE_F32, D_CONV, CONV_CHANNELS);
+    ggml_tensor * plain_conv = ggml_ssm_conv(ctx, sx, conv_w);
+
+    ggml_tensor * step_x = ggml_new_tensor_3d(
+        ctx, GGML_TYPE_F32, CONV_CHANNELS, 1, 1);
+    ggml_tensor * step_state = ggml_new_tensor_3d(
+        ctx, GGML_TYPE_F32, D_CONV - 1, CONV_CHANNELS, 1);
+    ggml_tensor * fused_step = ggml_ssm_conv_step(
+        ctx, step_x, conv_w, step_state, nullptr);
+
+    constexpr int DYN_HIDDEN = 32;
+    constexpr int DYN_GROUP = 16;
+    constexpr int DYN_TOKENS = 2;
+    ggml_tensor * dyn_x = ggml_new_tensor_2d(
+        ctx, GGML_TYPE_F32, DYN_HIDDEN, DYN_TOKENS);
+    ggml_tensor * dyn_base = ggml_new_tensor_3d(
+        ctx, GGML_TYPE_F32, DYN_HIDDEN, D_CONV, 2);
+    ggml_tensor * dyn_weights = ggml_new_tensor_2d(
+        ctx, GGML_TYPE_F32,
+        2 * D_CONV * (DYN_HIDDEN / DYN_GROUP), DYN_TOKENS);
+    ggml_tensor * fused_dyn = ggml_dflash_dyn_conv(
+        ctx, dyn_x, dyn_base, dyn_weights, 0, D_CONV, DYN_GROUP);
+
+    ggml_tensor * q = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1);
+    ggml_tensor * k = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1);
+    ggml_tensor * v = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1);
+    ggml_tensor * g = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, 1, N_HEAD, 1, 1);
+    ggml_tensor * beta = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, 1, N_HEAD, 1, 1);
+    ggml_tensor * state = ggml_new_tensor_4d(
+        ctx, GGML_TYPE_F32, S_V, S_V, N_HEAD, 1);
+    ggml_tensor * plain_gdn = ggml_gated_delta_net(
+        ctx, q, k, v, g, beta, state);
+    const bool plain_gdn_supported =
+        ggml_backend_supports_op(backend, plain_gdn);
+    ggml_tensor * raw_gdn = ggml_gated_delta_net(
+        ctx, q, k, v, g, beta, state);
+    ggml_tensor * gate_ba = ggml_new_tensor_1d(
+        ctx, GGML_TYPE_F32, 2 * N_HEAD);
+    ggml_gated_delta_net_set_raw_gates(raw_gdn, gate_ba);
+
+    const bool ok =
+        ggml_backend_supports_op(backend, plain_conv) &&
+        !ggml_backend_supports_op(backend, fused_step) &&
+        !ggml_backend_supports_op(backend, fused_dyn) &&
+        plain_gdn_supported &&
+        !ggml_backend_supports_op(backend, raw_gdn);
+    std::printf("batched gdn CPU capability guards       %s\n",
+                ok ? "PASS" : "FAIL");
+    ggml_free(ctx);
+    return ok;
+}
+
 bool test_masked_set_rows(ggml_backend_t backend) {
     constexpr int ROW_WIDTH = 4;
     constexpr int DEST_ROWS = 4;
@@ -530,6 +597,9 @@ int main(int argc, char ** argv) {
     ok = test_gdn_active_slots(backend, rng) && ok;
     ok = test_conv(backend, rng) && ok;
     ok = test_masked_set_rows(backend) && ok;
+    if (cpu) {
+        ok = test_cpu_rejects_gpu_only_extensions(backend) && ok;
+    }
 
     ggml_backend_free(backend);
     return ok ? 0 : 1;