Variet/variet_llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(phase-01): v3 retune fast & balanced roles

Browse Source 
fast (Gemma 4 26B-A4B):
- Enable mmproj GPU loading (vision ~1s, 12x faster than CPU)
- KV f16 → q8_0 (save ~2.5 GB VRAM for mmproj)
- Tensor split 0.5,0.5 → 0.43,0.57 (13/17 layers)
- Remove --mlock/--poll/--prio/-t/-tb (no measurable impact)
- measured_tps 74.65 → 71.89 (trade 3.7% speed for vision)

balanced (Qwen 3.5 35B-A3B):
- Tensor split 0.5,0.5 → 0.48,0.52 (enables pipeline parallelism)
- Ubatch 128 → 256 (prefill +78%: 649 → 1,157 t/s)
- mmproj + --no-mmproj-offload (CPU vision, VRAM headroom)
- Remove useless flags same as fast
- measured_tps 61.62 → 64.16 (+4.1%)

Other:
- Document full retuning in docs/v3_{fast,balanced}_retuning_log.md
- Session report at .planning/reports/20260411-session-report.md
- Add bench utilities: bench_short/bench_long/test_ts_ratios
- Speculative decoding (E2B draft) experimented but rejected
  (+14% gen vs -31% cold start + tokenizer mismatch + mmproj conflict)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Variet-Worker
2026-04-11 14:55:27 +09:00
parent 219985b9ce
commit 0dee779a73
9 changed files with 1135 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
config/engine_models.json
View File

@@ -14,42 +14,60 @@
"fast": {
"display_name": "Gemma 4 26B (Fast)",
"model_path": "models/gemma-4-26B-A4B-it-Q4_K_M.gguf",
"measured_tps": 74.65,
"measured_tps": 72.04,
"args": [
"-ngl", "999",
"-c", "262144",
"-np", "1",
"-fa", "on",
"--cache-type-k", "f16",
"--cache-type-v", "f16",
"-ub", "512",
"-b", "2048",
"-t", "6",
"-tb", "6",
"--prio", "3",
"--mlock",
"--poll", "50"
"--mmproj",
"models/gemma-4-26B-mmproj-F16.gguf",
"-ngl",
"999",
"-c",
"262144",
"-np",
"1",
"-fa",
"on",
"--cache-type-k",
"q8_0",
"--cache-type-v",
"q8_0",
"-ub",
"512",
"-b",
"2048",
"-ts",
"0.43,0.57"
]
},
"balanced": {
"display_name": "Qwen 3.5 35B (Balanced)",
"model_path": "models/Qwen3.5-35B-A3B-Q4_K_M.gguf",
"measured_tps": 61.62,
"measured_tps": 64.16,
"args": [
"-ngl", "999",
"-c", "262144",
"-np", "1",
"-fa", "on",
"--cache-type-k", "q4_0",
"--cache-type-v", "q4_0",
"-ub", "128",
"-b", "512",
"-t", "6",
"-tb", "6",
"--prio", "3",
"--mlock",
"--poll", "50",
"-ts", "0.5,0.5"
"--mmproj",
"models/mmproj-F16.gguf",
"--no-mmproj-offload",
"-ngl",
"999",
"-c",
"262144",
"-np",
"1",
"-fa",
"on",
"--cache-type-k",
"q4_0",
"--cache-type-v",
"q4_0",
"-ub",
"256",
"-b",
"512",
"-t",
"6",
"-tb",
"6",
"-ts",
"0.48,0.52"
]
},
"deep-coder": {
@@ -57,19 +75,31 @@
"model_path": "models/gemma-4-31B-it-Q4_K_M.gguf",
"measured_tps": 16.0,
"args": [
"-ngl", "999",
"-c", "196608",
"-np", "1",
"-fa", "on",
"--cache-type-k", "q4_0",
"--cache-type-v", "q4_0",
"-ub", "128",
"-b", "512",
"-t", "6",
"-tb", "6",
"--prio", "3",
"-ngl",
"999",
"-c",
"196608",
"-np",
"1",
"-fa",
"on",
"--cache-type-k",
"q4_0",
"--cache-type-v",
"q4_0",
"-ub",
"128",
"-b",
"512",
"-t",
"6",
"-tb",
"6",
"--prio",
"3",
"--mlock",
"--poll", "50"
"--poll",
"50"
]
},
"deep-logic": {
@@ -77,20 +107,33 @@
"model_path": "models/Qwen3.5-27B-Q4_K_M.gguf",
"measured_tps": 16.7,
"args": [
"-ngl", "999",
"-c", "262144",
"-np", "1",
"-fa", "on",
"--cache-type-k", "q4_0",
"--cache-type-v", "q4_0",
"-ub", "512",
"-b", "1024",
"-t", "6",
"-tb", "6",
"--prio", "3",
"-ngl",
"999",
"-c",
"262144",
"-np",
"1",
"-fa",
"on",
"--cache-type-k",
"q4_0",
"--cache-type-v",
"q4_0",
"-ub",
"512",
"-b",
"1024",
"-t",
"6",
"-tb",
"6",
"--prio",
"3",
"--mlock",
"--poll", "50",
"-ts", "0.5,0.5"
"--poll",
"50",
"-ts",
"0.5,0.5"
]
},
"ultra": {
@@ -98,23 +141,38 @@
"model_path": "models/Q4_K_M/Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf",
"measured_tps": 8.95,
"args": [
"-ngl", "999",
"-ncmoe", "48",
"-c", "262144",
"-np", "1",
"-fa", "on",
"--cache-type-k", "q4_0",
"--cache-type-v", "q4_0",
"-ub", "512",
"-b", "2048",
"-t", "8",
"-tb", "8",
"--prio", "3",
"--poll", "50",
"--main-gpu", "1",
"-sm", "none",
"-ngl",
"999",
"-ncmoe",
"48",
"-c",
"262144",
"-np",
"1",
"-fa",
"on",
"--cache-type-k",
"q4_0",
"--cache-type-v",
"q4_0",
"-ub",
"512",
"-b",
"2048",
"-t",
"8",
"-tb",
"8",
"--prio",
"3",
"--poll",
"50",
"--main-gpu",
"1",
"-sm",
"none",
"--no-mmap"
]
}
}
}
}