Variet/variet_llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7c7a899fd5226285bfc106963e5c76141e68ba1e
variet_llm/scripts/tune_results_qwen35b_256k.json

522 lines
9.5 KiB
JSON
Raw Blame History

[
{
"ngl": 999,
"cpu_moe": true,
"t": 6,
"tb": 6,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.169961832638464,
"best_tps": 26.533887071573073,
"vram_used": 4994,
"vram_total": 12288,
"label": "cpu_moe=True"
},
{
"ngl": 999,
"cpu_moe": false,
"t": 6,
"tb": 6,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 11.065030380022206,
"best_tps": 11.083028272674314,
"vram_used": 11949,
"vram_total": 12288,
"label": "cpu_moe=False"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 2,
"tb": 2,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 21.473286428302767,
"best_tps": 21.746637577851104,
"vram_used": 4994,
"vram_total": 12288,
"label": "t=2 | tb=2"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.552358479030676,
"best_tps": 27.314237654089343,
"vram_used": 4991,
"vram_total": 12288,
"label": "t=4 | tb=4"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 6,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.347068485327956,
"best_tps": 26.87924726131441,
"vram_used": 4993,
"vram_total": 12288,
"label": "t=4 | tb=6"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 6,
"tb": 6,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.331286039513458,
"best_tps": 26.81427299445741,
"vram_used": 5001,
"vram_total": 12288,
"label": "t=6 | tb=6"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 6,
"tb": 8,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.391160513711274,
"best_tps": 26.735573238878736,
"vram_used": 5001,
"vram_total": 12288,
"label": "t=6 | tb=8"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 8,
"tb": 8,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 25.32340666199144,
"best_tps": 25.87949347494079,
"vram_used": 4995,
"vram_total": 12288,
"label": "t=8 | tb=8"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 10,
"tb": 10,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 23.752277317850815,
"best_tps": 24.98242898809555,
"vram_used": 5011,
"vram_total": 12288,
"label": "t=10 | tb=10"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 12,
"tb": 12,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 21.75032196383532,
"best_tps": 23.18963400077116,
"vram_used": 5104,
"vram_total": 12288,
"label": "t=12 | tb=12"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 128,
"b": 512,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 13.27593572827031,
"best_tps": 13.337407402920235,
"vram_used": 4391,
"vram_total": 12288,
"label": "ub=128 | b=512"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 256,
"b": 1024,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.638687188233188,
"best_tps": 27.361082444434413,
"vram_used": 4495,
"vram_total": 12288,
"label": "ub=256 | b=1024"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 256,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.29069503392877,
"best_tps": 26.63368832924803,
"vram_used": 4490,
"vram_total": 12288,
"label": "ub=256 | b=2048"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 1024,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.518331831441134,
"best_tps": 26.972021321271527,
"vram_used": 4984,
"vram_total": 12288,
"label": "ub=512 | b=1024"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.401541912276873,
"best_tps": 26.46530849236633,
"vram_used": 4990,
"vram_total": 12288,
"label": "ub=512 | b=2048"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 26.892711500590455,
"best_tps": 26.892711500590455,
"vram_used": 5006,
"vram_total": 12288,
"label": "ub=512 | b=4096"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 1024,
"b": 2048,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 12.600209659679201,
"best_tps": 12.759356030807627,
"vram_used": 12020,
"vram_total": 12288,
"label": "ub=1024 | b=2048"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 1024,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 6.023959262370547,
"best_tps": 8.284882268188156,
"vram_used": 11931,
"vram_total": 12288,
"label": "ub=1024 | b=4096"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 12.96992950856374,
"best_tps": 12.96992950856374,
"vram_used": 12022,
"vram_total": 12288,
"label": "ctk=q4_0 | ctv=q4_0"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q8_0",
"ctv": "q8_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 11.420078920350697,
"best_tps": 13.524778595767653,
"vram_used": 12030,
"vram_total": 12288,
"label": "ctk=q8_0 | ctv=q8_0"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "f16",
"ctv": "f16",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 11.978106511464183,
"best_tps": 13.729190013094977,
"vram_used": 11518,
"vram_total": 12288,
"label": "ctk=f16 | ctv=f16"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 50,
"avg_tps": 16.164278220452957,
"best_tps": 22.645890325274323,
"vram_used": 11623,
"vram_total": 12288,
"label": "mmap=True | poll=50 | prio=2"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": false,
"prio": 2,
"poll": 50,
"avg_tps": 16.555542780023114,
"best_tps": 23.333815015033892,
"vram_used": 9062,
"vram_total": 12288,
"label": "mmap=False | poll=50 | prio=2"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 0,
"avg_tps": 13.003619379106329,
"best_tps": 13.031594557134142,
"vram_used": 11994,
"vram_total": 12288,
"label": "mmap=True | poll=0 | prio=2"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 2,
"poll": 100,
"avg_tps": 5.7762452690702935,
"best_tps": 5.795560155803046,
"vram_used": 11953,
"vram_total": 12288,
"label": "mmap=True | poll=100 | prio=2"
},
{
"ngl": 999,
"cpu_moe": true,
"t": 4,
"tb": 4,
"ub": 512,
"b": 4096,
"ctk": "q4_0",
"ctv": "q4_0",
"fa": "on",
"mlock": true,
"mmap": true,
"prio": 3,
"poll": 50,
"avg_tps": 12.59406799687573,
"best_tps": 14.966737641114795,
"vram_used": 11996,
"vram_total": 12288,
"label": "mmap=True | poll=50 | prio=3"
}
]
Reference in New Issue View Git Blame Copy Permalink