Move runner port 18008 -> 18010
Follows the change already made on 4n4rch02. Updates the compose --port and healthcheck plus every default in the bench tooling and the README, so the scripts keep working without an explicit --port. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
llama.cpp CUDA runner for **Qwen3.6-35B-A3B** (UD-Q4_K_M) at **128k context** on 4n4rch02.
|
llama.cpp CUDA runner for **Qwen3.6-35B-A3B** (UD-Q4_K_M) at **128k context** on 4n4rch02.
|
||||||
|
|
||||||
Port **18008**, OpenAI-compatible API at `http://192.168.3.189:18008/v1`.
|
Port **18010**, OpenAI-compatible API at `http://192.168.3.189:18010/v1`.
|
||||||
|
|
||||||
## Host
|
## Host
|
||||||
|
|
||||||
@@ -262,7 +262,7 @@ Loads in ~15 s (`--no-mmap`, weights read from NVMe).
|
|||||||
## Benchmarks
|
## Benchmarks
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 bench/bench.py --port 18008 --depths 512,8192,32768 --n-predict 128
|
python3 bench/bench.py --port 18010 --depths 512,8192,32768 --n-predict 128
|
||||||
./bench/sweep.sh 10:24,16:8 8:24,16:8 # <ncmoe>:<tensor-split>:<threads>
|
./bench/sweep.sh 10:24,16:8 8:24,16:8 # <ncmoe>:<tensor-split>:<threads>
|
||||||
python3 bench/spec_bench.py --no-think # copy/code/prose generation workloads
|
python3 bench/spec_bench.py --no-think # copy/code/prose generation workloads
|
||||||
```
|
```
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ def bench(port, prompt_tokens, n_predict, label):
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--port", type=int, default=18008)
|
ap.add_argument("--port", type=int, default=18010)
|
||||||
ap.add_argument("--label", default="run")
|
ap.add_argument("--label", default="run")
|
||||||
ap.add_argument("--depths", default="512,8192,32768,131072")
|
ap.add_argument("--depths", default="512,8192,32768,131072")
|
||||||
ap.add_argument("--n-predict", type=int, default=128)
|
ap.add_argument("--n-predict", type=int, default=128)
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ def probe(port, label, extra):
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--port", type=int, default=18008)
|
ap.add_argument("--port", type=int, default=18010)
|
||||||
ap.add_argument("--case", choices=["default", "override", "off"], default="default")
|
ap.add_argument("--case", choices=["default", "override", "off"], default="default")
|
||||||
ap.add_argument("--budget", type=int, default=16384)
|
ap.add_argument("--budget", type=int, default=16384)
|
||||||
a = ap.parse_args()
|
a = ap.parse_args()
|
||||||
|
|||||||
+1
-1
@@ -61,7 +61,7 @@ def run(port, name, prompt, n_predict, no_think):
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--port", type=int, default=18008)
|
ap.add_argument("--port", type=int, default=18010)
|
||||||
ap.add_argument("--label", default="run")
|
ap.add_argument("--label", default="run")
|
||||||
ap.add_argument("--n-predict", type=int, default=400)
|
ap.add_argument("--n-predict", type=int, default=400)
|
||||||
ap.add_argument("--no-think", action="store_true",
|
ap.add_argument("--no-think", action="store_true",
|
||||||
|
|||||||
+2
-2
@@ -4,12 +4,12 @@
|
|||||||
# Usage: ./sweep.sh <ncmoe>:<tensor-split>[:<threads>] ...
|
# Usage: ./sweep.sh <ncmoe>:<tensor-split>[:<threads>] ...
|
||||||
# Example: ./sweep.sh 10:24,16:8 8:24,16:8
|
# Example: ./sweep.sh 10:24,16:8 8:24,16:8
|
||||||
#
|
#
|
||||||
# Env: CTX (default 131072), DEPTHS (default 512,8192), PORT (default 18008)
|
# Env: CTX (default 131072), DEPTHS (default 512,8192), PORT (default 18010)
|
||||||
|
|
||||||
HUB=${HUB:-/mnt/2TSAM990nvme/docker-volume-outsource/llm-models/hf/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF}
|
HUB=${HUB:-/mnt/2TSAM990nvme/docker-volume-outsource/llm-models/hf/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF}
|
||||||
SNAP=${SNAP:-snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf}
|
SNAP=${SNAP:-snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf}
|
||||||
CTX=${CTX:-131072}
|
CTX=${CTX:-131072}
|
||||||
PORT=${PORT:-18008}
|
PORT=${PORT:-18010}
|
||||||
NAME=qwen36-sweep
|
NAME=qwen36-sweep
|
||||||
BENCH=${BENCH:-$(dirname "$0")/bench.py}
|
BENCH=${BENCH:-$(dirname "$0")/bench.py}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -72,9 +72,9 @@ services:
|
|||||||
- --host
|
- --host
|
||||||
- 0.0.0.0
|
- 0.0.0.0
|
||||||
- --port
|
- --port
|
||||||
- "18008"
|
- "18010"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-sf", "http://localhost:18008/health"]
|
test: ["CMD", "curl", "-sf", "http://localhost:18010/health"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|||||||
Reference in New Issue
Block a user