DATAandClaude Opus 5
2c9cd04e96
Hybrid SSM+attention MoE (qwen35moe): only 10 of 40 layers use full attention, so 128k of KV costs ~1.3 GiB. The binding constraint is the 20.6 GiB of weights against 22 GiB of VRAM, handled with --n-cpu-moe. Two findings drive the config: - --n-cpu-moe strips experts from the first N layers, which -sm layer assigns to CUDA0, so CUDA1 inherits every heavy layer and OOMs at any offload level. -ts 24,16 rebalances it. - --threads 8 (physical cores) beats 16 by 44% on generation; the expert matmuls are bandwidth-bound and SMT siblings only contend. Ships ncmoe=10 (53 t/s @8k, 34 t/s @97k) over the faster ncmoe=8 to keep ~1.3 GiB spare on CUDA0, which is shared with the desktop. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
68 lines
2.4 KiB
Python
Executable File
68 lines
2.4 KiB
Python
Executable File
import sys, struct
|
|
|
|
GGUF_MAGIC = 0x46554747
|
|
# value types
|
|
T_UINT8,T_INT8,T_UINT16,T_INT16,T_UINT32,T_INT32,T_FLOAT32,T_BOOL,T_STRING,T_ARRAY,T_UINT64,T_INT64,T_FLOAT64 = range(13)
|
|
|
|
f = open(sys.argv[1],'rb')
|
|
|
|
def rd(n): return f.read(n)
|
|
def u32(): return struct.unpack('<I', rd(4))[0]
|
|
def u64(): return struct.unpack('<Q', rd(8))[0]
|
|
def i32(): return struct.unpack('<i', rd(4))[0]
|
|
def i64(): return struct.unpack('<q', rd(8))[0]
|
|
def f32(): return struct.unpack('<f', rd(4))[0]
|
|
def f64(): return struct.unpack('<d', rd(8))[0]
|
|
def string():
|
|
n = u64()
|
|
return rd(n).decode('utf-8', errors='replace')
|
|
|
|
def value(t):
|
|
if t == T_UINT8: return struct.unpack('<B', rd(1))[0]
|
|
if t == T_INT8: return struct.unpack('<b', rd(1))[0]
|
|
if t == T_UINT16: return struct.unpack('<H', rd(2))[0]
|
|
if t == T_INT16: return struct.unpack('<h', rd(2))[0]
|
|
if t == T_UINT32: return u32()
|
|
if t == T_INT32: return i32()
|
|
if t == T_FLOAT32:return f32()
|
|
if t == T_BOOL: return struct.unpack('<?', rd(1))[0]
|
|
if t == T_STRING: return string()
|
|
if t == T_UINT64: return u64()
|
|
if t == T_INT64: return i64()
|
|
if t == T_FLOAT64:return f64()
|
|
if t == T_ARRAY:
|
|
et = u32(); n = u64()
|
|
if et == T_STRING:
|
|
# don't materialize huge token lists
|
|
if n > 16:
|
|
for _ in range(n):
|
|
ln = u64(); f.seek(ln, 1)
|
|
return f'<array string x{n}>'
|
|
return [string() for _ in range(n)]
|
|
sizes = {T_UINT8:1,T_INT8:1,T_UINT16:2,T_INT16:2,T_UINT32:4,T_INT32:4,T_FLOAT32:4,T_BOOL:1,T_UINT64:8,T_INT64:8,T_FLOAT64:8}
|
|
if n > 16:
|
|
f.seek(sizes[et]*n, 1)
|
|
return f'<array t{et} x{n}>'
|
|
return [value(et) for _ in range(n)]
|
|
raise ValueError(f'unknown type {t}')
|
|
|
|
magic = u32()
|
|
assert magic == GGUF_MAGIC, hex(magic)
|
|
ver = u32()
|
|
n_tensors = u64()
|
|
n_kv = u64()
|
|
print(f'gguf_version={ver} n_tensors={n_tensors} n_kv={n_kv}')
|
|
print('---')
|
|
kv = {}
|
|
for _ in range(n_kv):
|
|
k = string(); t = u32(); v = value(t)
|
|
kv[k] = v
|
|
|
|
for k, v in kv.items():
|
|
if k.startswith('tokenizer.ggml.') and k not in ('tokenizer.ggml.model','tokenizer.ggml.pre','tokenizer.ggml.bos_token_id','tokenizer.ggml.eos_token_id','tokenizer.ggml.padding_token_id','tokenizer.ggml.add_bos_token'):
|
|
continue
|
|
if k == 'tokenizer.chat_template':
|
|
print(f'{k} = <len {len(str(v))}>')
|
|
continue
|
|
print(f'{k} = {v}')
|