llama.cpp CUDA runner for Qwen3.6-35B-A3B at 128k context

Hybrid SSM+attention MoE (qwen35moe): only 10 of 40 layers use full attention, so 128k of KV costs ~1.3 GiB. The binding constraint is the 20.6 GiB of weights against 22 GiB of VRAM, handled with --n-cpu-moe. Two findings drive the config: - --n-cpu-moe strips experts from the first N layers, which -sm layer assigns to CUDA0, so CUDA1 inherits every heavy layer and OOMs at any offload level. -ts 24,16 rebalances it. - --threads 8 (physical cores) beats 16 by 44% on generation; the expert matmuls are bandwidth-bound and SMT siblings only contend. Ships ncmoe=10 (53 t/s @8k, 34 t/s @97k) over the faster ncmoe=8 to keep ~1.3 GiB spare on CUDA0, which is shared with the desktop. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-25 13:49:33 +02:00
commit 2c9cd04e96
6 changed files with 421 additions and 0 deletions
@@ -0,0 +1,67 @@
+import sys, struct
+
+GGUF_MAGIC = 0x46554747
+# value types
+T_UINT8,T_INT8,T_UINT16,T_INT16,T_UINT32,T_INT32,T_FLOAT32,T_BOOL,T_STRING,T_ARRAY,T_UINT64,T_INT64,T_FLOAT64 = range(13)
+
+f = open(sys.argv[1],'rb')
+
+def rd(n): return f.read(n)
+def u32(): return struct.unpack('<I', rd(4))[0]
+def u64(): return struct.unpack('<Q', rd(8))[0]
+def i32(): return struct.unpack('<i', rd(4))[0]
+def i64(): return struct.unpack('<q', rd(8))[0]
+def f32(): return struct.unpack('<f', rd(4))[0]
+def f64(): return struct.unpack('<d', rd(8))[0]
+def string():
+    n = u64()
+    return rd(n).decode('utf-8', errors='replace')
+
+def value(t):
+    if t == T_UINT8:  return struct.unpack('<B', rd(1))[0]
+    if t == T_INT8:   return struct.unpack('<b', rd(1))[0]
+    if t == T_UINT16: return struct.unpack('<H', rd(2))[0]
+    if t == T_INT16:  return struct.unpack('<h', rd(2))[0]
+    if t == T_UINT32: return u32()
+    if t == T_INT32:  return i32()
+    if t == T_FLOAT32:return f32()
+    if t == T_BOOL:   return struct.unpack('<?', rd(1))[0]
+    if t == T_STRING: return string()
+    if t == T_UINT64: return u64()
+    if t == T_INT64:  return i64()
+    if t == T_FLOAT64:return f64()
+    if t == T_ARRAY:
+        et = u32(); n = u64()
+        if et == T_STRING:
+            # don't materialize huge token lists
+            if n > 16:
+                for _ in range(n):
+                    ln = u64(); f.seek(ln, 1)
+                return f'<array string x{n}>'
+            return [string() for _ in range(n)]
+        sizes = {T_UINT8:1,T_INT8:1,T_UINT16:2,T_INT16:2,T_UINT32:4,T_INT32:4,T_FLOAT32:4,T_BOOL:1,T_UINT64:8,T_INT64:8,T_FLOAT64:8}
+        if n > 16:
+            f.seek(sizes[et]*n, 1)
+            return f'<array t{et} x{n}>'
+        return [value(et) for _ in range(n)]
+    raise ValueError(f'unknown type {t}')
+
+magic = u32()
+assert magic == GGUF_MAGIC, hex(magic)
+ver = u32()
+n_tensors = u64()
+n_kv = u64()
+print(f'gguf_version={ver} n_tensors={n_tensors} n_kv={n_kv}')
+print('---')
+kv = {}
+for _ in range(n_kv):
+    k = string(); t = u32(); v = value(t)
+    kv[k] = v
+
+for k, v in kv.items():
+    if k.startswith('tokenizer.ggml.') and k not in ('tokenizer.ggml.model','tokenizer.ggml.pre','tokenizer.ggml.bos_token_id','tokenizer.ggml.eos_token_id','tokenizer.ggml.padding_token_id','tokenizer.ggml.add_bos_token'):
+        continue
+    if k == 'tokenizer.chat_template':
+        print(f'{k} = <len {len(str(v))}>')
+        continue
+    print(f'{k} = {v}')