services:
  llama-qwen36-a3b:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: llama-qwen36-a3b
    restart: unless-stopped
    runtime: nvidia
    network_mode: host
    environment:
      - NVIDIA_VISIBLE_DEVICES=0,1
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
      - CUDA_DEVICE_ORDER=PCI_BUS_ID
    cap_add:
      - IPC_LOCK
    ipc: host
    volumes:
      # HF hub repo dir, NOT the snapshot dir: the snapshot entry is a relative
      # symlink into ../../blobs/, so the mount has to contain both.
      - /mnt/2TSAM990nvme/docker-volume-outsource/llm-models/hf/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF:/models:ro
      - /mnt/1TVi550s3/datas-docker-space/slot-cache/qwen36-a3b:/slots
    command:
      - -m
      - /models/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf
      - --alias
      - qwen3.6-35b-a3b-q4-k-m
      - --ctx-size
      - "131072"
      # Only 10 of 40 layers are full-attention (full_attention_interval=4), so
      # 128k of KV costs just ~1.3 GiB at q8_0. The other 30 layers are SSM and
      # carry a constant-size recurrent state.
      - -ctk
      - q8_0
      - -ctv
      - q8_0
      - -fa
      - "on"
      - -ngl
      - "99"
      # Weights are 20.6 GiB vs 22 GiB total VRAM, so the experts of the first
      # N layers live in host RAM. See README for the ncmoe/-ts interaction.
      - --n-cpu-moe
      - "10"
      - -sm
      - layer
      - -ts
      - "24,16"
      # 8 = physical cores on the Ryzen 7 3700X. Using all 16 SMT threads costs
      # ~30% generation throughput (measured), because the CPU-side expert
      # matmuls are memory-bound and SMT siblings just contend for bandwidth.
      - --threads
      - "8"
      - -np
      - "1"
      - --no-mmap
      # Default only, NOT a ceiling: a client sending max_tokens overrides it
      # (verified — max_tokens=9000 returned 8893 tokens against --predict 8192).
      # It bounds clients that send no max_tokens at all, which would otherwise
      # generate until the 128k window is full: ~50 min blocking the single slot.
      - --predict
      - "32768"
      # Hard server-side ceiling on the thinking block — a client CANNOT raise it
      # per request (verified). Note the README measurement: on the test problem
      # 4096 and 16384 produced the SAME correct answer, but 16384 burned 3.3x the
      # tokens looping. 4096 is defensible; revisit if real workloads disagree.
      - --reasoning-budget
      - "16384"
      - --slot-save-path
      - /slots
      - --jinja
      - --reasoning-format
      - auto
      - --host
      - 0.0.0.0
      - --port
      - "18008"
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:18008/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 180s