llama.cpp CUDA runner for Qwen3.6-35B-A3B at 128k context
Hybrid SSM+attention MoE (qwen35moe): only 10 of 40 layers use full attention, so 128k of KV costs ~1.3 GiB. The binding constraint is the 20.6 GiB of weights against 22 GiB of VRAM, handled with --n-cpu-moe. Two findings drive the config: - --n-cpu-moe strips experts from the first N layers, which -sm layer assigns to CUDA0, so CUDA1 inherits every heavy layer and OOMs at any offload level. -ts 24,16 rebalances it. - --threads 8 (physical cores) beats 16 by 44% on generation; the expert matmuls are bandwidth-bound and SMT siblings only contend. Ships ncmoe=10 (53 t/s @8k, 34 t/s @97k) over the faster ncmoe=8 to keep ~1.3 GiB spare on CUDA0, which is shared with the desktop. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark llama.cpp server: prefill (pp) and generation (tg) throughput."""
|
||||
import json, sys, urllib.request, argparse, random, string
|
||||
|
||||
def post(port, path, payload, timeout=1800):
|
||||
req = urllib.request.Request(
|
||||
f"http://localhost:{port}{path}",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
|
||||
def make_prompt(approx_tokens):
|
||||
# ~0.75 words/token for english-ish filler; use varied words to avoid trivial cache hits
|
||||
words = ["system", "kernel", "vector", "matrix", "gradient", "tensor", "buffer",
|
||||
"context", "attention", "expert", "router", "layer", "cache", "token",
|
||||
"inference", "throughput", "latency", "quantize", "offload", "memory"]
|
||||
rnd = random.Random(1234)
|
||||
n_words = int(approx_tokens * 0.75)
|
||||
return " ".join(rnd.choice(words) for _ in range(n_words))
|
||||
|
||||
def bench(port, prompt_tokens, n_predict, label):
|
||||
prompt = make_prompt(prompt_tokens)
|
||||
r = post(port, "/completion", {
|
||||
"prompt": prompt,
|
||||
"n_predict": n_predict,
|
||||
"temperature": 0.7,
|
||||
"cache_prompt": False,
|
||||
})
|
||||
t = r.get("timings", {})
|
||||
pp_n = t.get("prompt_n", 0); pp_ms = t.get("prompt_ms", 0)
|
||||
tg_n = t.get("predicted_n", 0); tg_ms = t.get("predicted_ms", 0)
|
||||
pp_tps = t.get("prompt_per_second") or 0
|
||||
tg_tps = t.get("predicted_per_second") or 0
|
||||
print(f"{label:<28} pp: {pp_n:>7} tok @ {pp_tps:>8.1f} t/s | tg: {tg_n:>4} tok @ {tg_tps:>6.2f} t/s",
|
||||
flush=True)
|
||||
return {"label": label, "pp_n": pp_n, "pp_tps": pp_tps, "tg_n": tg_n, "tg_tps": tg_tps,
|
||||
"pp_ms": pp_ms, "tg_ms": tg_ms}
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=18008)
|
||||
ap.add_argument("--label", default="run")
|
||||
ap.add_argument("--depths", default="512,8192,32768,131072")
|
||||
ap.add_argument("--n-predict", type=int, default=128)
|
||||
ap.add_argument("--json-out", default=None)
|
||||
a = ap.parse_args()
|
||||
|
||||
# Warm up: the first request after load pays one-off CUDA graph / kernel
|
||||
# setup that otherwise shows up as a bogus ~30% slower first data point.
|
||||
try:
|
||||
post(a.port, "/completion", {"prompt": make_prompt(64), "n_predict": 16,
|
||||
"temperature": 0, "cache_prompt": False})
|
||||
except Exception as e:
|
||||
print(f"warmup failed: {e}")
|
||||
|
||||
results = []
|
||||
for d in [int(x) for x in a.depths.split(",")]:
|
||||
try:
|
||||
results.append(bench(a.port, d, a.n_predict, f"{a.label} @{d//1024}k" if d >= 1024 else f"{a.label} @{d}"))
|
||||
except Exception as e:
|
||||
print(f"{a.label} @{d}: FAILED {type(e).__name__}: {e}")
|
||||
if a.json_out:
|
||||
with open(a.json_out, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
# Sweep llama.cpp runner configs for Qwen3.6-35B-A3B; report VRAM + throughput.
|
||||
#
|
||||
# Usage: ./sweep.sh <ncmoe>:<tensor-split>[:<threads>] ...
|
||||
# Example: ./sweep.sh 10:24,16:8 8:24,16:8
|
||||
#
|
||||
# Env: CTX (default 131072), DEPTHS (default 512,8192), PORT (default 18008)
|
||||
|
||||
HUB=${HUB:-/mnt/2TSAM990nvme/docker-volume-outsource/llm-models/hf/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF}
|
||||
SNAP=${SNAP:-snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf}
|
||||
CTX=${CTX:-131072}
|
||||
PORT=${PORT:-18008}
|
||||
NAME=qwen36-sweep
|
||||
BENCH=${BENCH:-$(dirname "$0")/bench.py}
|
||||
|
||||
run_cfg() {
|
||||
local spec="$1"
|
||||
local ncmoe="${spec%%:*}"
|
||||
local rest="${spec#*:}"
|
||||
local ts="${rest%%:*}"
|
||||
local threads="${rest#*:}"
|
||||
[ "$threads" = "$ts" ] && threads=8
|
||||
|
||||
echo "########## n-cpu-moe=$ncmoe -ts $ts threads=$threads ctx=$CTX ##########"
|
||||
docker rm -f $NAME >/dev/null 2>&1
|
||||
docker run -d --name $NAME --runtime nvidia --network host \
|
||||
-e NVIDIA_VISIBLE_DEVICES=0,1 \
|
||||
-e NVIDIA_DRIVER_CAPABILITIES=compute,utility \
|
||||
-e CUDA_DEVICE_ORDER=PCI_BUS_ID \
|
||||
-v "$HUB":/models:ro \
|
||||
ghcr.io/ggml-org/llama.cpp:server-cuda \
|
||||
-m /models/$SNAP \
|
||||
--ctx-size $CTX -ctk q8_0 -ctv q8_0 -fa on \
|
||||
-ngl 99 --n-cpu-moe "$ncmoe" -sm layer -ts "$ts" \
|
||||
--threads "$threads" -np 1 --no-mmap $EXTRA \
|
||||
--host 0.0.0.0 --port $PORT >/dev/null 2>&1
|
||||
|
||||
local ok=0
|
||||
for _ in $(seq 1 200); do
|
||||
if docker logs $NAME 2>&1 | grep -q "listening on"; then ok=1; break; fi
|
||||
if docker logs $NAME 2>&1 | grep -qiE "out of memory|CUDA error|failed to allocate|terminate called|error loading model"; then ok=2; break; fi
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^$NAME$"; then ok=3; break; fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [ "$ok" != "1" ]; then
|
||||
echo "RESULT ncmoe=$ncmoe ts=$ts threads=$threads -> FAILED"
|
||||
docker logs $NAME 2>&1 | grep -iE "out of memory|failed to allocate" | head -2
|
||||
docker rm -f $NAME >/dev/null 2>&1; echo; return 1
|
||||
fi
|
||||
|
||||
echo "--- VRAM used/free (MiB) ---"
|
||||
nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv,noheader
|
||||
python3 "$BENCH" --port $PORT --label "n${ncmoe}_ts${ts}_t${threads}" \
|
||||
--depths "${DEPTHS:-512,8192}" --n-predict 128
|
||||
docker rm -f $NAME >/dev/null 2>&1
|
||||
echo
|
||||
}
|
||||
|
||||
for c in "$@"; do run_cfg "$c"; done
|
||||
Reference in New Issue
Block a user