Slayer1B-Poetry / tutorials /average_checkpoints.py
kacperwikiel's picture
Add five native Slayer notebooks: generation, CPT, SFT, RL and measured checkpoint averaging
05c768b verified
Raw History Blame Contribute Delete
4.39 kB
"""Average compatible native Slayer full checkpoints in FP32, with provenance."""
from pathlib import Path
import argparse,hashlib,json,math
import torch
from huggingface_hub import hf_hub_download
REPO='SlayerLab/Slayer1B-1.09-checkpoints'
REVISION='32a7304dc2208acb3fb4a6e957498116552a4b18'
def sha256(path):
h=hashlib.sha256()
with open(path,'rb') as f:
for chunk in iter(lambda:f.read(8*1024*1024),b''):h.update(chunk)
return h.hexdigest()
def average(paths,weights,output):
if not paths or len(paths)!=len(weights):raise ValueError('One weight per checkpoint is required')
if not all(math.isfinite(w) and w>=0 for w in weights) or sum(weights)<=0:raise ValueError('Need finite nonnegative weights with positive total')
if len(set(map(str,paths)))!=len(paths):raise ValueError('Duplicate checkpoint paths')
weights=[w/sum(weights) for w in weights]
accumulator={};config=None;keys=None;dtypes={};sources=[]
for path,weight in zip(paths,weights):
checkpoint=torch.load(path,map_location='cpu',weights_only=True,mmap=True)
state=checkpoint['model']
if config is None:
config=dict(checkpoint['config']);keys=set(state)
elif checkpoint['config']!=config or set(state)!=keys:raise ValueError('Configuration or parameter keys differ')
if not torch.equal(state['tok.weight'],state['head.weight']):raise ValueError('Source embeddings/head are not tied-equivalent')
for key,tensor in state.items():
if not isinstance(tensor,torch.Tensor):raise ValueError('Expected tensor state')
if key not in accumulator:
dtypes[key]=tensor.dtype
accumulator[key]=torch.zeros_like(tensor,dtype=torch.float32) if tensor.is_floating_point() else tensor.clone()
elif accumulator[key].shape!=tensor.shape or dtypes[key]!=tensor.dtype:raise ValueError(f'Shape/dtype mismatch: {key}')
if tensor.is_floating_point():
if not torch.isfinite(tensor).all():raise ValueError(f'Nonfinite source: {key}')
accumulator[key].add_(tensor,alpha=weight)
elif not torch.equal(accumulator[key],tensor):raise ValueError(f'Nonfloat buffers differ: {key}')
sources.append({'file':Path(path).name,'sha256':sha256(path),'step':int(checkpoint['step']),'weight':weight})
del state,checkpoint
print('Accumulated',sources[-1],flush=True)
for key,tensor in accumulator.items():
if tensor.is_floating_point() and not torch.isfinite(tensor).all():raise ValueError('Nonfinite average')
# Preserve aliasing explicitly in the output checkpoint.
accumulator['head.weight']=accumulator['tok.weight']
merged={'model':accumulator,'config':config,'step':0,
'merge_metadata':{'method':'normalized arithmetic weight average in FP32','sources':sources,
'step_note':'0 denotes a derived model, not an optimizer update'}}
output=Path(output);output.parent.mkdir(parents=True,exist_ok=True)
temp=output.with_suffix(output.suffix+'.tmp');torch.save(merged,temp);temp.replace(output)
metadata={**merged['merge_metadata'],'output_sha256':sha256(output)}
output.with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
return metadata
def from_hub(steps,weights,output,revision=REVISION):
sums=Path(hf_hub_download(REPO,'SHA256SUMS',revision=revision)).read_text()
expected={line.split()[1]:line.split()[0] for line in sums.splitlines() if line.strip()}
paths=[]
for step in steps:
name=f'wagi_{step}.pt'
path=hf_hub_download(REPO,name,revision=revision)
if sha256(path)!=expected[name]:raise ValueError(f'Checksum mismatch: {name}')
paths.append(path)
metadata=average(paths,weights,output)
metadata.update(repo=REPO,revision=revision,source_filenames=[f'wagi_{s}.pt' for s in steps])
Path(output).with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
return metadata
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--steps',type=int,nargs='+',default=[45000,50000,55000]);p.add_argument('--weights',type=float,nargs='+');p.add_argument('--output',default='outputs/slayer1b-average.pt');a=p.parse_args()
print(json.dumps(from_hub(a.steps,a.weights or [1.]*len(a.steps),a.output),indent=2))