diff --git a/README.md b/README.md index 5299244..ae25c18 100644 --- a/README.md +++ b/README.md @@ -113,12 +113,36 @@ Generation falls off ~40% between empty and ~97k, which is the attention cost on 10 full-attention layers. Prefill stays flat around 420–490 t/s, so filling the whole 128k window takes roughly 4–5 minutes. -### 5. Rejected +### 5. Speculative decoding makes it slower — don't enable it + +Counter-intuitive but consistent, `--n-predict 400`, thinking disabled: + +| workload | none | ngram-mod | ngram-simple | ngram-cache | +|---|---|---|---|---| +| rewrite (output ≈ copy of prompt) | **55.6** | 40.6 | 39.7 | 47.0 | +| extend (structured code) | **55.3** | 32.7 | 27.2 | 38.1 | +| prose (novel text, control) | **56.1** | 54.7 | 52.9 | 48.4 | + +Every mode loses, and it loses *most* on exactly the structured workloads +speculation is supposed to win. The reason is the CPU-side experts: verifying a +K-token draft costs about K times the CPU expert work, because each token routes to +its own subset of 8 out of 256 experts, so there is no weight reuse to amortize. +On a fully GPU-resident model batch verification is nearly free; with +`--n-cpu-moe` it is not. Reconsider only if the model ever fits entirely in VRAM. + +### 6. Rejected - `-ub 256`: frees only ~80 MiB per GPU (not the 469 MiB an extra layer needs) and costs **38% prefill** (556 → 347 t/s). Keep the default `-ub 512`. - Lower `-ts` toward CUDA1 (e.g. `23,17`): would leave CUDA1 at ~60 MiB free. +### A note on the CUDA0 headroom + +Free VRAM on CUDA0 was observed drifting between ~1100 and ~1300 MiB across restarts +purely from desktop compositor churn — a ~200 MiB swing with nothing else changing. +That is why the default keeps >1 GiB spare there rather than the 388 MiB that +`ncmoe=8` leaves. + ## Deploy ```bash @@ -133,6 +157,7 @@ Loads in ~15 s (`--no-mmap`, weights read from NVMe). ```bash python3 bench/bench.py --port 18008 --depths 512,8192,32768 --n-predict 128 ./bench/sweep.sh 10:24,16:8 8:24,16:8 # :: +python3 bench/spec_bench.py --no-think # copy/code/prose generation workloads ``` `bench.py` issues a warmup request first — without it the first measurement reads diff --git a/bench/spec_bench.py b/bench/spec_bench.py new file mode 100755 index 0000000..7b0fa27 --- /dev/null +++ b/bench/spec_bench.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Measure generation throughput on workloads where ngram speculative decoding +can actually pay off: output that largely echoes the prompt (refactor / rewrite / +RAG-style restatement), plus a free-form control that it cannot help with.""" +import json, argparse, urllib.request + +CODE = ''' +class RingBuffer: + def __init__(self, capacity): + self.capacity = capacity + self.data = [None] * capacity + self.head = 0 + self.tail = 0 + self.size = 0 + + def push(self, item): + if self.size == self.capacity: + raise BufferError("ring buffer is full") + self.data[self.tail] = item + self.tail = (self.tail + 1) % self.capacity + self.size += 1 + + def pop(self): + if self.size == 0: + raise IndexError("pop from empty ring buffer") + item = self.data[self.head] + self.data[self.head] = None + self.head = (self.head + 1) % self.capacity + self.size -= 1 + return item +''' + +WORKLOADS = { + # Best case: output is mostly a copy of the input with small edits. + "rewrite": "Repeat this Python class verbatim, changing only the class name " + "to CircularBuffer. Output only code, no commentary:\n" + CODE, + # Realistic coding task: structurally predictable, not a copy. + "extend": "Here is a Python class:\n" + CODE + + "\nRewrite it with an added peek() method and type hints. Output only code.", + # Control: novel prose, nothing for an ngram matcher to hit. + "prose": "Write a short original story about a lighthouse keeper who collects storms.", +} + + +def run(port, name, prompt, n_predict, no_think): + body = {"messages": [{"role": "user", "content": prompt}], + "max_tokens": n_predict, "temperature": 0} + if no_think: + body["chat_template_kwargs"] = {"enable_thinking": False} + req = urllib.request.Request(f"http://localhost:{port}/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=1800) as r: + d = json.load(r) + t = d.get("timings", {}) + tps = t.get("predicted_per_second") or 0 + n = t.get("predicted_n") or 0 + print(f" {name:<10} {n:>5} tok @ {tps:>6.2f} t/s", flush=True) + return tps + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=18008) + ap.add_argument("--label", default="run") + ap.add_argument("--n-predict", type=int, default=400) + ap.add_argument("--no-think", action="store_true", + help="disable reasoning so the measurement is the answer, not the thinking") + a = ap.parse_args() + + print(f"[{a.label}]", flush=True) + for name, prompt in WORKLOADS.items(): + try: + run(a.port, name, prompt, a.n_predict, a.no_think) + except Exception as e: + print(f" {name:<10} FAILED {type(e).__name__}: {e}", flush=True)