#!/bin/bash # One-shot experiment suite for Ling-3.0-tiny VQ3 vs standard quantizations # Usage: run_experiments.sh (from work root) set -u cd "$(dirname "$0")/../.." BIN=./llama.cpp/build/bin CORP=corpora/wikitext-2-raw/wiki.test.raw R=ling-vq3/results mkdir -p "$R" T=64 CHUNKS_PPL=256 CHUNKS_KLD=128 declare -A MODELS=( [bf16]=models/Ling-3.0-tiny-bf16.gguf [q8_0]=models/Ling-3.0-tiny-Q8_0.gguf [q4_k_m]=models/Ling-3.0-tiny-Q4_K_M.gguf [base_q8dq6]=models/Ling-3.0-tiny-base-Q8dQ6.gguf [vq3]=models/Ling-3.0-tiny-VQ3.gguf ) echo "=== [1/5] PPL (wikitext-2 test, $CHUNKS_PPL chunks) ===" for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do m=${MODELS[$k]} echo "--- PPL $k ---" $BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_PPL -t $T -ngl 0 \ > "$R/ppl_$k.log" 2>&1 grep -E "Final estimate|error" "$R/ppl_$k.log" | tail -2 done echo "=== [2/5] Baseline logits from BF16 (KLD reference, $CHUNKS_KLD chunks) ===" $BIN/llama-perplexity -m models/Ling-3.0-tiny-bf16.gguf -f "$CORP" -c 512 --chunks $CHUNKS_KLD \ -t $T -ngl 0 --save-all-logits "$R/logits_bf16.bin" > "$R/logits_bf16.log" 2>&1 grep -E "Final estimate|error" "$R/logits_bf16.log" | tail -2 echo "=== [3/5] KLD vs BF16 ===" for k in q8_0 q4_k_m base_q8dq6 vq3; do m=${MODELS[$k]} echo "--- KLD $k ---" $BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_KLD -t $T -ngl 0 \ --kl-divergence --kl-divergence-base "$R/logits_bf16.bin" > "$R/kld_$k.log" 2>&1 grep -E "Final estimate|error" "$R/kld_$k.log" | tail -2 done echo "=== [4/5] llama-bench pp512/tg128 ===" $BIN/llama-bench -t $T -p 512 -n 128 -r 2 -mlo 0 \ -m models/Ling-3.0-tiny-bf16.gguf \ -m models/Ling-3.0-tiny-Q8_0.gguf \ -m models/Ling-3.0-tiny-Q4_K_M.gguf \ -m models/Ling-3.0-tiny-base-Q8dQ6.gguf \ -m models/Ling-3.0-tiny-VQ3.gguf > "$R/bench.tsv" 2> "$R/bench.err" cat "$R/bench.tsv" echo "=== [5/5] Peak RSS via short generation ===" for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do m=${MODELS[$k]} /usr/bin/time -v $BIN/llama-cli -m "$m" -p "Hello" -n 8 -t $T --no-warmup -ngl 0 --temp 0 \ > /dev/null 2> "$R/rss_$k.txt" rss=$(grep "Maximum resident" "$R/rss_$k.txt" | awk '{print $6}') echo "peak RSS $k: $rss KB" done echo "=== ALL DONE ==="