0dee779a73
fast (Gemma 4 26B-A4B):
- Enable mmproj GPU loading (vision ~1s, 12x faster than CPU)
- KV f16 → q8_0 (save ~2.5 GB VRAM for mmproj)
- Tensor split 0.5,0.5 → 0.43,0.57 (13/17 layers)
- Remove --mlock/--poll/--prio/-t/-tb (no measurable impact)
- measured_tps 74.65 → 71.89 (trade 3.7% speed for vision)
balanced (Qwen 3.5 35B-A3B):
- Tensor split 0.5,0.5 → 0.48,0.52 (enables pipeline parallelism)
- Ubatch 128 → 256 (prefill +78%: 649 → 1,157 t/s)
- mmproj + --no-mmproj-offload (CPU vision, VRAM headroom)
- Remove useless flags same as fast
- measured_tps 61.62 → 64.16 (+4.1%)
Other:
- Document full retuning in docs/v3_{fast,balanced}_retuning_log.md
- Session report at .planning/reports/20260411-session-report.md
- Add bench utilities: bench_short/bench_long/test_ts_ratios
- Speculative decoding (E2B draft) experimented but rejected
(+14% gen vs -31% cold start + tokenizer mismatch + mmproj conflict)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
178 lines
3.3 KiB
JSON
178 lines
3.3 KiB
JSON
{
|
|
"version": "1.0",
|
|
"default_role": "fast",
|
|
"llama_server": {
|
|
"path": "llama_bin_run/llama-server.exe",
|
|
"internal_port": 8080,
|
|
"internal_host": "127.0.0.1"
|
|
},
|
|
"engine": {
|
|
"external_port": 8000,
|
|
"external_host": "0.0.0.0"
|
|
},
|
|
"roles": {
|
|
"fast": {
|
|
"display_name": "Gemma 4 26B (Fast)",
|
|
"model_path": "models/gemma-4-26B-A4B-it-Q4_K_M.gguf",
|
|
"measured_tps": 72.04,
|
|
"args": [
|
|
"--mmproj",
|
|
"models/gemma-4-26B-mmproj-F16.gguf",
|
|
"-ngl",
|
|
"999",
|
|
"-c",
|
|
"262144",
|
|
"-np",
|
|
"1",
|
|
"-fa",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q8_0",
|
|
"--cache-type-v",
|
|
"q8_0",
|
|
"-ub",
|
|
"512",
|
|
"-b",
|
|
"2048",
|
|
"-ts",
|
|
"0.43,0.57"
|
|
]
|
|
},
|
|
"balanced": {
|
|
"display_name": "Qwen 3.5 35B (Balanced)",
|
|
"model_path": "models/Qwen3.5-35B-A3B-Q4_K_M.gguf",
|
|
"measured_tps": 64.16,
|
|
"args": [
|
|
"--mmproj",
|
|
"models/mmproj-F16.gguf",
|
|
"--no-mmproj-offload",
|
|
"-ngl",
|
|
"999",
|
|
"-c",
|
|
"262144",
|
|
"-np",
|
|
"1",
|
|
"-fa",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"-ub",
|
|
"256",
|
|
"-b",
|
|
"512",
|
|
"-t",
|
|
"6",
|
|
"-tb",
|
|
"6",
|
|
"-ts",
|
|
"0.48,0.52"
|
|
]
|
|
},
|
|
"deep-coder": {
|
|
"display_name": "Gemma 4 31B (Deep Coder)",
|
|
"model_path": "models/gemma-4-31B-it-Q4_K_M.gguf",
|
|
"measured_tps": 16.0,
|
|
"args": [
|
|
"-ngl",
|
|
"999",
|
|
"-c",
|
|
"196608",
|
|
"-np",
|
|
"1",
|
|
"-fa",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"-ub",
|
|
"128",
|
|
"-b",
|
|
"512",
|
|
"-t",
|
|
"6",
|
|
"-tb",
|
|
"6",
|
|
"--prio",
|
|
"3",
|
|
"--mlock",
|
|
"--poll",
|
|
"50"
|
|
]
|
|
},
|
|
"deep-logic": {
|
|
"display_name": "Qwen 3.5 27B (Deep Logic)",
|
|
"model_path": "models/Qwen3.5-27B-Q4_K_M.gguf",
|
|
"measured_tps": 16.7,
|
|
"args": [
|
|
"-ngl",
|
|
"999",
|
|
"-c",
|
|
"262144",
|
|
"-np",
|
|
"1",
|
|
"-fa",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"-ub",
|
|
"512",
|
|
"-b",
|
|
"1024",
|
|
"-t",
|
|
"6",
|
|
"-tb",
|
|
"6",
|
|
"--prio",
|
|
"3",
|
|
"--mlock",
|
|
"--poll",
|
|
"50",
|
|
"-ts",
|
|
"0.5,0.5"
|
|
]
|
|
},
|
|
"ultra": {
|
|
"display_name": "Qwen 3.5 122B (Ultra Heavy)",
|
|
"model_path": "models/Q4_K_M/Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf",
|
|
"measured_tps": 8.95,
|
|
"args": [
|
|
"-ngl",
|
|
"999",
|
|
"-ncmoe",
|
|
"48",
|
|
"-c",
|
|
"262144",
|
|
"-np",
|
|
"1",
|
|
"-fa",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"-ub",
|
|
"512",
|
|
"-b",
|
|
"2048",
|
|
"-t",
|
|
"8",
|
|
"-tb",
|
|
"8",
|
|
"--prio",
|
|
"3",
|
|
"--poll",
|
|
"50",
|
|
"--main-gpu",
|
|
"1",
|
|
"-sm",
|
|
"none",
|
|
"--no-mmap"
|
|
]
|
|
}
|
|
}
|
|
} |