From 09230bc499fe97be330f3fdc53d029fb3b2c4af1 Mon Sep 17 00:00:00 2001 From: DATA Date: Sat, 25 Jul 2026 15:03:49 +0200 Subject: [PATCH] Measure --reasoning-budget: a real cap, unlike --predict Counted the thinking block exactly via /tokenize. Server default stops it at 4095 tokens; a client sending reasoning_budget=16384 still gets 4095, so it cannot be raised per request. Changing it means editing the compose file. It truncates mid-derivation on a moderately hard problem, but the model recovered and answered correctly (empty solution set, verified by brute force), so there is no evidence 4096 is actually damaging output. Co-Authored-By: Claude Opus 5 --- README.md | 26 ++++++++++++- bench/reasoning_budget_probe.py | 68 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100755 bench/reasoning_budget_probe.py diff --git a/README.md b/README.md index 59bfe4c..35c39da 100644 --- a/README.md +++ b/README.md @@ -206,8 +206,30 @@ were silently cut off with `finish_reason: length`. Note the sibling `gemma4-26b-llama-runner` README describes `--predict` as a "harte Cap fuer Gesamt-Generierung" — by this measurement that is wrong there too. -`--reasoning-budget 4096` is a separate knob for the thinking block and was **not** -verified the same way; do not assume its semantics from the flag name either. +## `--reasoning-budget` dagegen ist ein echter Deckel + +Same flag family, opposite semantics — measured with `bench/reasoning_budget_probe.py`, +which counts the thinking block exactly via the server's `/tokenize` endpoint: + +| Request | Reasoning tokens | Answer | +|---|---|---| +| server default | **4095** | 1227 tok, complete | +| `reasoning_budget: 16384` in the request body | **4095** | 531 tok, complete | + +A client **cannot** raise it. Where `--predict` was a default anyone could override, +this one is a hard server-side ceiling, so changing it means editing the compose file +and restarting. + +It does bite: on a moderately hard number-theory problem the thinking was guillotined +mid-derivation (`...Since $x^2 \equiv`) at exactly the budget. But the model recovered +— it produced the correct answer (empty solution set, verified by brute force) and +rebuilt the parity argument cleanly in the visible response. So 4096 truncates +reasoning without necessarily damaging the result. + +Raising it does not weaken the runaway protection, since total generation is already +bounded by `--predict` / the client's `max_tokens`. For heavier math or agent +workloads, 16384 is the obvious next step; there is no measured evidence here that +4096 is actively breaking anything. ## Deploy diff --git a/bench/reasoning_budget_probe.py b/bench/reasoning_budget_probe.py new file mode 100755 index 0000000..e3c5707 --- /dev/null +++ b/bench/reasoning_budget_probe.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Probe what --reasoning-budget actually enforces. + +Sends a prompt hard enough to blow past the budget, then measures the thinking +block exactly via the server's /tokenize endpoint (word counts are too rough to +tell 4096 from 4500). Also tries a per-request override to distinguish a hard +cap from a default. +""" +import json, argparse, urllib.request + +# Deliberately open-ended search problem: forces long systematic enumeration +# rather than a one-line recall answer. +HARD = ( + "Bestimme alle Tripel positiver ganzer Zahlen (a,b,c) mit a<=b<=c, " + "a+b+c = 30 und a*b*c durch 30 teilbar, fuer die zusaetzlich a^2+b^2 = c^2+7 gilt. " + "Pruefe systematisch alle Faelle durch und begruende jeden Ausschluss einzeln." +) + + +def post(port, path, payload, timeout=3600): + req = urllib.request.Request(f"http://localhost:{port}{path}", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def ntok(port, text): + if not text: + return 0 + return len(post(port, "/tokenize", {"content": text}).get("tokens", [])) + + +def probe(port, label, extra): + body = {"messages": [{"role": "user", "content": HARD}], "temperature": 0.3} + body.update(extra) + d = post(port, "/v1/chat/completions", body) + ch = d["choices"][0] + msg = ch["message"] + think = msg.get("reasoning_content") or "" + answer = msg.get("content") or "" + t_think = ntok(port, think) + t_ans = ntok(port, answer) + print(f"[{label}]", flush=True) + print(f" reasoning tokens : {t_think}", flush=True) + print(f" answer tokens : {t_ans}", flush=True) + print(f" completion_tokens: {d.get('usage', {}).get('completion_tokens')}", flush=True) + print(f" finish_reason : {ch.get('finish_reason')}", flush=True) + print(f" answer non-empty : {bool(answer.strip())}", flush=True) + print(f" thinking tail : ...{think[-160:]!r}", flush=True) + return t_think + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=18008) + ap.add_argument("--case", choices=["default", "override", "off"], default="default") + ap.add_argument("--budget", type=int, default=16384) + a = ap.parse_args() + + if a.case == "default": + probe(a.port, "server default", {}) + elif a.case == "override": + probe(a.port, f"per-request reasoning_budget={a.budget}", + {"reasoning_budget": a.budget}) + else: + probe(a.port, "thinking disabled", + {"chat_template_kwargs": {"enable_thinking": False}})