lingtiny30vq3 / scripts /run_experiments.sh
HelloSun's picture
LUT-VQ3-64: machine-tuned (threads/NUMA/ubatch), full 128k ctx data, one-click build, results
511ba50 verified
Raw History Blame Contribute Delete
2.26 kB
#!/bin/bash
# One-shot experiment suite for Ling-3.0-tiny VQ3 vs standard quantizations
# Usage: run_experiments.sh (from work root)
set -u
cd "$(dirname "$0")/../.."
BIN=./llama.cpp/build/bin
CORP=corpora/wikitext-2-raw/wiki.test.raw
R=ling-vq3/results
mkdir -p "$R"
T=64
CHUNKS_PPL=256
CHUNKS_KLD=128
declare -A MODELS=(
[bf16]=models/Ling-3.0-tiny-bf16.gguf
[q8_0]=models/Ling-3.0-tiny-Q8_0.gguf
[q4_k_m]=models/Ling-3.0-tiny-Q4_K_M.gguf
[base_q8dq6]=models/Ling-3.0-tiny-base-Q8dQ6.gguf
[vq3]=models/Ling-3.0-tiny-VQ3.gguf
)
echo "=== [1/5] PPL (wikitext-2 test, $CHUNKS_PPL chunks) ==="
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
m=${MODELS[$k]}
echo "--- PPL $k ---"
$BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_PPL -t $T -ngl 0 \
> "$R/ppl_$k.log" 2>&1
grep -E "Final estimate|error" "$R/ppl_$k.log" | tail -2
done
echo "=== [2/5] Baseline logits from BF16 (KLD reference, $CHUNKS_KLD chunks) ==="
$BIN/llama-perplexity -m models/Ling-3.0-tiny-bf16.gguf -f "$CORP" -c 512 --chunks $CHUNKS_KLD \
-t $T -ngl 0 --save-all-logits "$R/logits_bf16.bin" > "$R/logits_bf16.log" 2>&1
grep -E "Final estimate|error" "$R/logits_bf16.log" | tail -2
echo "=== [3/5] KLD vs BF16 ==="
for k in q8_0 q4_k_m base_q8dq6 vq3; do
m=${MODELS[$k]}
echo "--- KLD $k ---"
$BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_KLD -t $T -ngl 0 \
--kl-divergence --kl-divergence-base "$R/logits_bf16.bin" > "$R/kld_$k.log" 2>&1
grep -E "Final estimate|error" "$R/kld_$k.log" | tail -2
done
echo "=== [4/5] llama-bench pp512/tg128 ==="
$BIN/llama-bench -t $T -p 512 -n 128 -r 2 -mlo 0 \
-m models/Ling-3.0-tiny-bf16.gguf \
-m models/Ling-3.0-tiny-Q8_0.gguf \
-m models/Ling-3.0-tiny-Q4_K_M.gguf \
-m models/Ling-3.0-tiny-base-Q8dQ6.gguf \
-m models/Ling-3.0-tiny-VQ3.gguf > "$R/bench.tsv" 2> "$R/bench.err"
cat "$R/bench.tsv"
echo "=== [5/5] Peak RSS via short generation ==="
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
m=${MODELS[$k]}
/usr/bin/time -v $BIN/llama-cli -m "$m" -p "Hello" -n 8 -t $T --no-warmup -ngl 0 --temp 0 \
> /dev/null 2> "$R/rss_$k.txt"
rss=$(grep "Maximum resident" "$R/rss_$k.txt" | awk '{print $6}')
echo "peak RSS $k: $rss KB"
done
echo "=== ALL DONE ==="