feat: Variet Engine v1.0 + 5-model tuning complete

Phase 01 (LLM Tuning): - Gemma4 26B: 74.65 t/s (fast) - Qwen 35B: 61.62 t/s (balanced) - Gemma4 31B: 16.0 t/s (deep-coder) - Qwen 27B: 16.7 t/s (deep-logic) - Qwen 122B: 8.95 t/s (ultra, GPU 1 only) Phase 02 (API Engine): - FastAPI reverse proxy on port 8000 - /engine/switch hot-swap with 503 protection - config/engine_models.json as single source of truth - Replaced 4 individual .bat files with unified engine File cleanup: - scripts/ 85 files -> 9 + _archive/ - Root .bat files -> _archive/
2026-04-07 18:08:58 +09:00
parent 7c7a899fd5
commit c111b3a9b0
414 changed files with 3402 additions and 68598 deletions
--- a/scripts/_archive/tuning/tune_122b.py
+++ b/scripts/_archive/tuning/tune_122b.py
@@ -0,0 +1,129 @@
+import subprocess, time, urllib.request, json, sys
+try: sys.stdout.reconfigure(encoding='utf-8')
+except: pass
+
+MODEL = "C:/Users/Variet-Worker/Desktop/variet-llm/models/Q4_K_M/Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf"
+BASE = "http://127.0.0.1:8000"
+
+# BEST SO FAR: GPU1 only + Expert CPU + 8t = 8.75 t/s (6.5GB / 12GB used)
+# 5.5GB VRAM remaining on GPU 1. Let's use it!
+# Strategy: keep some experts on GPU 1 using -ncmoe (n-cpu-moe)
+# n-cpu-moe = number of layers whose experts stay on CPU
+# Lower = more experts on GPU = more VRAM used = potentially faster
+
+BASE_CMD = [
+    r"llama_bin_run\llama-server.exe",
+    "--model", MODEL,
+    "-ngl", "999",
+    "-sm", "none", "--main-gpu", "1",
+    "-c", "4096", "-np", "1", "-fa", "on",
+    "--cache-type-k", "q4_0", "--cache-type-v", "q4_0",
+    "-ub", "512", "-b", "2048",
+    "-t", "8", "-tb", "8",
+    "--prio", "3", "--poll", "50",
+    "--no-mmap",
+    "--port", "8000", "--host", "0.0.0.0"
+]
+
+CONFIGS = [
+    # Baseline: all experts CPU (confirmed 8.75 t/s)
+    {"name": "Baseline: all expert CPU", "extra": ["-ot", ".*ffn_.*_exps.*=CPU"]},
+    # Try n-cpu-moe with GPU1 only: keep some experts on GPU
+    {"name": "n-cpu-moe=60 (4 layers expert GPU)", "extra": ["-ncmoe", "60"]},
+    {"name": "n-cpu-moe=56 (8 layers expert GPU)", "extra": ["-ncmoe", "56"]},
+    {"name": "n-cpu-moe=52 (12 layers expert GPU)", "extra": ["-ncmoe", "52"]},
+    {"name": "n-cpu-moe=48 (16 layers expert GPU)", "extra": ["-ncmoe", "48"]},
+]
+
+def kill():
+    subprocess.run("taskkill /F /IM llama-server.exe", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    time.sleep(4)
+
+def check_server(timeout=900):
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            req = urllib.request.Request(f"{BASE}/health")
+            resp = json.loads(urllib.request.urlopen(req, timeout=2).read())
+            if resp.get("status") in ("ok", "ready"):
+                return True
+        except: pass
+        time.sleep(5)
+    return False
+
+def bench(runs=3):
+    speeds = []
+    for i in range(runs):
+        payload = json.dumps({
+            "model": "m",
+            "messages": [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Write a Python fibonacci function with memoization."}
+            ],
+            "max_tokens": 200,
+            "temperature": 0.0
+        }).encode('utf-8')
+        req = urllib.request.Request(f"{BASE}/v1/chat/completions", data=payload, headers={"Content-Type": "application/json"})
+        t0 = time.time()
+        resp = json.loads(urllib.request.urlopen(req, timeout=600).read())
+        dt = time.time() - t0
+        tokens = resp.get("usage", {}).get("completion_tokens", 0)
+        speed = tokens / dt if dt > 0 else 0
+        speeds.append(speed)
+        print(f"    Run {i+1}: {speed:.2f} t/s ({tokens} tok / {dt:.1f}s)")
+    return sum(speeds)/len(speeds), max(speeds)
+
+def vram():
+    try:
+        out = subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits", shell=True).decode().strip()
+        return [int(x.strip()) for x in out.split('\n')]
+    except: return [0, 0]
+
+results = []
+for cfg in CONFIGS:
+    kill()
+    print(f"\n{'='*60}")
+    print(f"Testing: {cfg['name']}")
+    print(f"{'='*60}")
+    
+    cmd = BASE_CMD + cfg["extra"]
+    proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    
+    if not check_server(900):
+        print(f"  FAILED TO BOOT")
+        results.append({"name": cfg["name"], "status": "BOOT_FAIL"})
+        proc.terminate(); kill(); continue
+    
+    print("  Server ready! Warming up...")
+    try:
+        p = json.dumps({"model":"m","messages":[{"role":"system","content":"Hi"},{"role":"user","content":"Hi"}],"max_tokens":5}).encode()
+        urllib.request.urlopen(urllib.request.Request(f"{BASE}/v1/chat/completions",data=p,headers={"Content-Type":"application/json"}), timeout=120)
+    except: pass
+    
+    v = vram()
+    print(f"  VRAM: GPU0={v[0]}MB, GPU1={v[1]}MB, Total={sum(v)}MB")
+    
+    avg, best = bench(runs=3)
+    print(f"  >>> AVG: {avg:.2f} t/s | BEST: {best:.2f} t/s")
+    
+    results.append({
+        "name": cfg["name"], "avg_tps": round(avg,2), "best_tps": round(best,2),
+        "vram_gpu0": v[0], "vram_gpu1": v[1], "vram_total": sum(v), "status": "OK"
+    })
+    proc.terminate()
+
+kill()
+print(f"\n\n{'='*60}")
+print("FINAL RESULTS - GPU1 Expert Balance (Target: 10+ t/s)")
+print(f"{'='*60}")
+print(f"{'Config':<48} {'AVG':>6} {'BEST':>6} {'GPU1':>7}")
+print("-" * 72)
+for r in results:
+    if r["status"] == "OK":
+        print(f"  {r['name']:<46} {r['avg_tps']:>5} {r['best_tps']:>5} {r['vram_gpu1']:>5}MB")
+    else:
+        print(f"  {r['name']:<46} {'FAIL':>5}")
+
+with open("scripts/122b_final_results.json", "w", encoding="utf-8") as f:
+    json.dump(results, f, ensure_ascii=False, indent=2)
+print("\nSaved to scripts/122b_final_results.json")