chore: initial project setup with agent guide
This commit is contained in:
10
start_qwen_35b_api.bat
Normal file
10
start_qwen_35b_api.bat
Normal file
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
echo =========================================================
|
||||
echo Qwen3.5 35B-A3B API Server (MoE Optimized)
|
||||
echo [INFO] --cpu-moe: Expert weights forced to CPU RAM
|
||||
echo [INFO] -ngl 999: All attention layers on GPU
|
||||
echo [INFO] -np 1: Single slot for max speed
|
||||
echo =========================================================
|
||||
echo.
|
||||
llama_bin_run\llama-server.exe --model models\Qwen3.5-35B-A3B-Q4_K_M.gguf -ngl 999 --cpu-moe -c 4096 -np 1 -fa on --cache-type-k q4_0 --cache-type-v q4_0 -ub 512 -b 2048 -t 6 --mlock --prio 2 --port 8000 --host 0.0.0.0
|
||||
pause
|
||||
Reference in New Issue
Block a user