feat(phase-06): complete Hermes Agent windows fixes & deployment
This commit is contained in:
Variet-Worker
88
scripts/gemma4_test.py
Normal file
88
scripts/gemma4_test.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Gemma 4 26B-A4B Q4_K_M - 76.4 t/s 재현 테스트
|
||||
이전 최적값: ngl=999 t=6 ub=512 b=2048 ctk=f16 ctv=f16
|
||||
"""
|
||||
import subprocess, time, json, urllib.request, sys, os
|
||||
|
||||
try: sys.stdout.reconfigure(encoding='utf-8')
|
||||
except: pass
|
||||
|
||||
LLAMA = os.path.join(os.getcwd(), "llama_bin_run", "llama-server.exe")
|
||||
MODEL = os.path.join(os.getcwd(), "models", "gemma-4-26B-A4B-it-Q4_K_M.gguf")
|
||||
|
||||
subprocess.run(["taskkill", "/F", "/IM", "llama-server.exe"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(3)
|
||||
|
||||
cmd = [
|
||||
LLAMA, "--model", MODEL,
|
||||
"-ngl", "999", "-c", "262144", "-np", "1", "-fa", "on",
|
||||
"--cache-type-k", "f16", "--cache-type-v", "f16",
|
||||
"-ub", "512", "-b", "2048", "-t", "6", "-tb", "6",
|
||||
"--prio", "3", "--mlock", "--poll", "50",
|
||||
"--port", "8000", "--host", "0.0.0.0",
|
||||
]
|
||||
|
||||
print("[1/4] Starting Gemma4 26B Q4_K_M (76.4 t/s config)...")
|
||||
server = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
print("[2/4] Waiting for boot...")
|
||||
healthy = False
|
||||
for sec in range(180):
|
||||
time.sleep(1)
|
||||
if server.poll() is not None:
|
||||
print(f" !! CRASHED (exit code {server.returncode})")
|
||||
sys.exit(1)
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=1) as r:
|
||||
if json.loads(r.read()).get("status") == "ok":
|
||||
healthy = True; break
|
||||
except: pass
|
||||
if sec % 10 == 9: print(f" ... {sec+1}s")
|
||||
|
||||
if not healthy:
|
||||
print(" FAIL: boot timeout"); server.kill(); sys.exit(1)
|
||||
|
||||
print(f" OK!")
|
||||
try:
|
||||
v = subprocess.run(["nvidia-smi", "--query-gpu=index,memory.used,memory.total", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5)
|
||||
print(f" VRAM: {v.stdout.strip()}")
|
||||
except: pass
|
||||
|
||||
def bench(n):
|
||||
payload = json.dumps({"messages": [{"role": "user", "content": "Count from 1 to 50, each number on a new line."}], "max_tokens": n, "temperature": 0}).encode()
|
||||
req = urllib.request.Request("http://127.0.0.1:8000/v1/chat/completions", data=payload, headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
res = json.loads(r.read())
|
||||
el = time.time() - t0
|
||||
ct = res["usage"]["completion_tokens"]
|
||||
return ct / el, ct, el
|
||||
|
||||
try: bench(10)
|
||||
except: pass
|
||||
|
||||
print("[3/4] Running 5x benchmark (200 tokens)...")
|
||||
results = []
|
||||
for i in range(5):
|
||||
tps, tok, el = bench(200)
|
||||
results.append(tps)
|
||||
print(f" Run {i+1}: {tps:.2f} t/s ({tok} tok / {el:.2f}s)")
|
||||
|
||||
avg = sum(results) / len(results)
|
||||
best = max(results)
|
||||
worst = min(results)
|
||||
summary = f"""
|
||||
==================================================
|
||||
Gemma4 26B Q4_K_M 5-Run Results:
|
||||
AVG: {avg:.2f} t/s
|
||||
BEST: {best:.2f} t/s
|
||||
MIN: {worst:.2f} t/s
|
||||
Runs: {[f'{r:.2f}' for r in results]}
|
||||
==================================================
|
||||
"""
|
||||
print(summary)
|
||||
with open("scripts/gemma4_test_result.txt", "w") as f:
|
||||
f.write(summary)
|
||||
|
||||
server.kill()
|
||||
subprocess.run(["taskkill", "/F", "/IM", "llama-server.exe"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
67
scripts/qwen_split_challenge.py
Normal file
67
scripts/qwen_split_challenge.py
Normal file
@@ -0,0 +1,67 @@
|
||||
import subprocess
|
||||
import time
|
||||
import json
|
||||
import urllib.request
|
||||
import sys
|
||||
import os
|
||||
|
||||
try: sys.stdout.reconfigure(encoding='utf-8')
|
||||
except AttributeError: pass
|
||||
|
||||
MODEL = r"models\Qwen3.5-35B-A3B-Q4_K_M.gguf"
|
||||
LLAMA_SERVER = r"llama_bin_run\llama-server.exe"
|
||||
|
||||
subprocess.run(["taskkill", "/F", "/IM", "llama-server.exe"], capture_output=True)
|
||||
time.sleep(2)
|
||||
|
||||
cmd = [
|
||||
LLAMA_SERVER, "--model", MODEL,
|
||||
"-ngl", "999", "-c", "262144", "-np", "1", "-fa", "on",
|
||||
"--cache-type-k", "q4_0", "--cache-type-v", "q4_0",
|
||||
"-ub", "128", "-b", "512", "-t", "6", "-tb", "6",
|
||||
"--prio", "3", "--port", "8000", "--host", "0.0.0.0",
|
||||
"-ts", "0.44,0.56"
|
||||
]
|
||||
|
||||
print(f"🚀 Starting Challenge (0.44, 0.56) ...")
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
ready = False
|
||||
for i in range(120):
|
||||
try:
|
||||
req = urllib.request.Request("http://127.0.0.1:8000/health")
|
||||
with urllib.request.urlopen(req, timeout=1) as r:
|
||||
if json.loads(r.read()).get("status") == "ok":
|
||||
ready = True
|
||||
break
|
||||
except:
|
||||
pass
|
||||
print(f" booting... {i}s", end='\r', flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
if not ready:
|
||||
print("\n❌ FAILED to boot.")
|
||||
proc.kill()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n✅ Booted! Testing 200 tokens...")
|
||||
try:
|
||||
payload = json.dumps({
|
||||
"messages": [{"role": "user", "content": "Count from 1 to 50, each number on a new line."}],
|
||||
"max_tokens": 200, "temperature": 0
|
||||
}).encode()
|
||||
req = urllib.request.Request("http://127.0.0.1:8000/v1/chat/completions", data=payload, headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
res = json.loads(r.read())
|
||||
el = time.time() - t0
|
||||
ct = res["usage"]["completion_tokens"]
|
||||
tps = ct / el
|
||||
print("="*50)
|
||||
print(f"★ 0.44 / 0.56 RESULT: {tps:.2f} t/s ★")
|
||||
print(f" Tokens: {ct} | Time: {el:.2f}s")
|
||||
print("="*50)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Benchmark Error: {e}")
|
||||
|
||||
proc.kill()
|
||||
84
scripts/tune_models.mjs
Normal file
84
scripts/tune_models.mjs
Normal file
@@ -0,0 +1,84 @@
|
||||
import { exec, spawn } from 'child_process';
|
||||
|
||||
const delay = ms => new Promise(res => setTimeout(res, ms));
|
||||
|
||||
async function runTest(modelArgs, name) {
|
||||
console.log(`\n===========================================`);
|
||||
console.log(`Testing: ${name}`);
|
||||
console.log(`Args: ${modelArgs}`);
|
||||
|
||||
return new Promise(async (resolve) => {
|
||||
// Kill existing
|
||||
await new Promise(r => exec('taskkill /F /IM llama-server.exe', r));
|
||||
await delay(2000);
|
||||
|
||||
const server = spawn('llama_bin_run\\llama-server.exe', modelArgs.split(' '), {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
});
|
||||
|
||||
let ready = false;
|
||||
let oom = false;
|
||||
|
||||
for (let i = 0; i < 40; i++) {
|
||||
try {
|
||||
const res = await fetch('http://127.0.0.1:8000/health', { timeout: 2000 });
|
||||
if (res.status === 200) {
|
||||
ready = true;
|
||||
break;
|
||||
}
|
||||
} catch (e) {}
|
||||
await delay(3000);
|
||||
}
|
||||
|
||||
if (!ready) {
|
||||
console.log(`[${name}] FAILED TO BOOT (Likely OOM)`);
|
||||
exec('taskkill /F /IM llama-server.exe');
|
||||
resolve({ success: false });
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[${name}] Server Ready! Running benchmark...`);
|
||||
// Run pptest
|
||||
exec('node scripts/quick_pptest.mjs', (err, stdout, stderr) => {
|
||||
console.log(stdout || stderr);
|
||||
|
||||
// Extract TG and PP from TG-500
|
||||
const tgMatch = stdout.match(/TG-500 \| PP:\d+tok \d+\.\dt\/s \| TG:\d+tok (\d+\.\d+)t\/s/);
|
||||
const ppMatch = stdout.match(/10K-CODE \| PP:\d+tok (\d+\.\d+)t\/s/);
|
||||
|
||||
const tg = tgMatch ? parseFloat(tgMatch[1]) : 0;
|
||||
const pp = ppMatch ? parseFloat(ppMatch[1]) : 0;
|
||||
|
||||
exec('taskkill /F /IM llama-server.exe');
|
||||
resolve({ success: true, tg, pp });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// 1. Qwen 35B Tuning: We need 70 t/s. Let's try 1-3 layers of n-cpu-moe to unlock ub=512
|
||||
const args35B_base = `--model models\\Qwen3.5-35B-A3B-Q4_K_M.gguf -ngl 999 -c 262144 -np 1 -fa on --cache-type-k q4_0 --cache-type-v q4_0 -b 512 -t 6 -tb 6 --prio 3 --fit off --port 8000 --host 0.0.0.0`;
|
||||
|
||||
// Test 1: n-cpu-moe 1, ub 512
|
||||
await runTest(`${args35B_base} -ub 512 --n-cpu-moe 1`, "Qwen-35B: moe=1, ub=512");
|
||||
|
||||
// Test 2: n-cpu-moe 2, ub 512
|
||||
await runTest(`${args35B_base} -ub 512 --n-cpu-moe 2`, "Qwen-35B: moe=2, ub=512");
|
||||
|
||||
// Test 3: n-cpu-moe 4, ub 512
|
||||
await runTest(`${args35B_base} -ub 512 --n-cpu-moe 4`, "Qwen-35B: moe=4, ub=512");
|
||||
|
||||
// 2. 122B Tuning: Find optimal n-cpu-moe
|
||||
const args122B_base = `--model models\\Q4_K_M\\Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf -ngl 999 -c 32768 -np 1 -fa on --cache-type-k q4_0 --cache-type-v q4_0 -ub 512 -b 2048 -t 6 -tb 6 --prio 3 --fit off --port 8000 --host 0.0.0.0`;
|
||||
|
||||
// Since 48 leaves 16GB free, each layer is ~1.5GB total, meaning ~0.75GB per GPU.
|
||||
// Let's try 38, 35, 30
|
||||
await runTest(`${args122B_base} --n-cpu-moe 38`, "Qwen-122B: moe=38");
|
||||
await runTest(`${args122B_base} --n-cpu-moe 30`, "Qwen-122B: moe=30");
|
||||
await runTest(`${args122B_base} --n-cpu-moe 22`, "Qwen-122B: moe=22");
|
||||
|
||||
console.log("Tuning finished.");
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user