c111b3a9b0
Phase 01 (LLM Tuning): - Gemma4 26B: 74.65 t/s (fast) - Qwen 35B: 61.62 t/s (balanced) - Gemma4 31B: 16.0 t/s (deep-coder) - Qwen 27B: 16.7 t/s (deep-logic) - Qwen 122B: 8.95 t/s (ultra, GPU 1 only) Phase 02 (API Engine): - FastAPI reverse proxy on port 8000 - /engine/switch hot-swap with 503 protection - config/engine_models.json as single source of truth - Replaced 4 individual .bat files with unified engine File cleanup: - scripts/ 85 files -> 9 + _archive/ - Root .bat files -> _archive/
73 lines
2.9 KiB
JavaScript
73 lines
2.9 KiB
JavaScript
import { exec, spawn } from 'child_process';
|
|
|
|
const delay = ms => new Promise(res => setTimeout(res, ms));
|
|
|
|
async function runTest(modelArgs, envVars, name) {
|
|
console.log(`\n===========================================`);
|
|
console.log(`Testing: ${name}`);
|
|
console.log(`Env: ${JSON.stringify(envVars)}`);
|
|
console.log(`Args: ${modelArgs}`);
|
|
|
|
return new Promise(async (resolve) => {
|
|
await new Promise(r => exec('taskkill /F /IM llama-server.exe', r));
|
|
await delay(2000);
|
|
|
|
const env = { ...process.env, ...envVars };
|
|
const server = spawn('llama_bin_run\\llama-server.exe', modelArgs.split(' '), {
|
|
detached: true,
|
|
stdio: 'ignore',
|
|
env
|
|
});
|
|
|
|
let ready = false;
|
|
|
|
for (let i = 0; i < 40; i++) {
|
|
try {
|
|
const res = await fetch('http://127.0.0.1:8000/health', { timeout: 2000 });
|
|
if (res.status === 200) {
|
|
ready = true;
|
|
break;
|
|
}
|
|
} catch (e) {}
|
|
await delay(3000);
|
|
}
|
|
|
|
if (!ready) {
|
|
console.log(`[${name}] FAILED TO BOOT`);
|
|
exec('taskkill /F /IM llama-server.exe');
|
|
resolve({ success: false });
|
|
return;
|
|
}
|
|
|
|
console.log(`[${name}] Server Ready! Running speed test...`);
|
|
exec('node scripts/quick_pptest.mjs', (err, stdout, stderr) => {
|
|
console.log(stdout || stderr);
|
|
exec('taskkill /F /IM llama-server.exe');
|
|
resolve({ success: true });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
// 1. 122B-A10B: Pure GPU offload (No n-cpu-moe at all)
|
|
// -ngl 999 will offload all 48 layers to the NVIDIA driver (triggering Shared VRAM fallback on Windows)
|
|
const args122B = `--model models\\Q4_K_M\\Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf -ngl 999 -c 32768 -np 1 -fa on --cache-type-k q4_0 --cache-type-v q4_0 -ub 512 -b 2048 -t 6 -tb 6 --prio 3 --fit off --port 8000 --host 0.0.0.0`;
|
|
await runTest(args122B, {}, "122B-A10B: Pure GPU (NVIDIA Shared Memory Fallback)");
|
|
|
|
// 2. 35B-A3B: Pure GPU tuning to hit 70 t/s
|
|
// Base configuration from previous full-gpu run:
|
|
const args35B = `--model models\\Qwen3.5-35B-A3B-Q4_K_M.gguf -ngl 999 -c 262144 -np 1 -fa on --cache-type-k q4_0 --cache-type-v q4_0 -ub 128 -b 512 -t 6 -tb 6 --prio 3 --fit off --port 8000 --host 0.0.0.0`;
|
|
|
|
// We already got ~64 t/s basically.
|
|
// Let's try MMQ for custom matrix multiplication which is often faster on Ampere for low batch generation
|
|
await runTest(args35B, { GGML_CUDA_FORCE_MMQ: "1" }, "35B-A3B: Force MMQ = 1");
|
|
|
|
// Try increasing threads to 12 just in case
|
|
const args35B_t12 = args35B.replace("-t 6 -tb 6", "-t 12 -tb 12");
|
|
await runTest(args35B_t12, { GGML_CUDA_FORCE_MMQ: "1" }, "35B-A3B: Threads 12 + MMQ");
|
|
|
|
console.log("\nALL TESTS COMPLETED");
|
|
}
|
|
|
|
main();
|