File size: 4,391 Bytes
05c768b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Average compatible native Slayer full checkpoints in FP32, with provenance."""
from pathlib import Path
import argparse,hashlib,json,math
import torch
from huggingface_hub import hf_hub_download
REPO='SlayerLab/Slayer1B-1.09-checkpoints'
REVISION='32a7304dc2208acb3fb4a6e957498116552a4b18'
def sha256(path):
h=hashlib.sha256()
with open(path,'rb') as f:
for chunk in iter(lambda:f.read(8*1024*1024),b''):h.update(chunk)
return h.hexdigest()
def average(paths,weights,output):
if not paths or len(paths)!=len(weights):raise ValueError('One weight per checkpoint is required')
if not all(math.isfinite(w) and w>=0 for w in weights) or sum(weights)<=0:raise ValueError('Need finite nonnegative weights with positive total')
if len(set(map(str,paths)))!=len(paths):raise ValueError('Duplicate checkpoint paths')
weights=[w/sum(weights) for w in weights]
accumulator={};config=None;keys=None;dtypes={};sources=[]
for path,weight in zip(paths,weights):
checkpoint=torch.load(path,map_location='cpu',weights_only=True,mmap=True)
state=checkpoint['model']
if config is None:
config=dict(checkpoint['config']);keys=set(state)
elif checkpoint['config']!=config or set(state)!=keys:raise ValueError('Configuration or parameter keys differ')
if not torch.equal(state['tok.weight'],state['head.weight']):raise ValueError('Source embeddings/head are not tied-equivalent')
for key,tensor in state.items():
if not isinstance(tensor,torch.Tensor):raise ValueError('Expected tensor state')
if key not in accumulator:
dtypes[key]=tensor.dtype
accumulator[key]=torch.zeros_like(tensor,dtype=torch.float32) if tensor.is_floating_point() else tensor.clone()
elif accumulator[key].shape!=tensor.shape or dtypes[key]!=tensor.dtype:raise ValueError(f'Shape/dtype mismatch: {key}')
if tensor.is_floating_point():
if not torch.isfinite(tensor).all():raise ValueError(f'Nonfinite source: {key}')
accumulator[key].add_(tensor,alpha=weight)
elif not torch.equal(accumulator[key],tensor):raise ValueError(f'Nonfloat buffers differ: {key}')
sources.append({'file':Path(path).name,'sha256':sha256(path),'step':int(checkpoint['step']),'weight':weight})
del state,checkpoint
print('Accumulated',sources[-1],flush=True)
for key,tensor in accumulator.items():
if tensor.is_floating_point() and not torch.isfinite(tensor).all():raise ValueError('Nonfinite average')
# Preserve aliasing explicitly in the output checkpoint.
accumulator['head.weight']=accumulator['tok.weight']
merged={'model':accumulator,'config':config,'step':0,
'merge_metadata':{'method':'normalized arithmetic weight average in FP32','sources':sources,
'step_note':'0 denotes a derived model, not an optimizer update'}}
output=Path(output);output.parent.mkdir(parents=True,exist_ok=True)
temp=output.with_suffix(output.suffix+'.tmp');torch.save(merged,temp);temp.replace(output)
metadata={**merged['merge_metadata'],'output_sha256':sha256(output)}
output.with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
return metadata
def from_hub(steps,weights,output,revision=REVISION):
sums=Path(hf_hub_download(REPO,'SHA256SUMS',revision=revision)).read_text()
expected={line.split()[1]:line.split()[0] for line in sums.splitlines() if line.strip()}
paths=[]
for step in steps:
name=f'wagi_{step}.pt'
path=hf_hub_download(REPO,name,revision=revision)
if sha256(path)!=expected[name]:raise ValueError(f'Checksum mismatch: {name}')
paths.append(path)
metadata=average(paths,weights,output)
metadata.update(repo=REPO,revision=revision,source_filenames=[f'wagi_{s}.pt' for s in steps])
Path(output).with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
return metadata
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--steps',type=int,nargs='+',default=[45000,50000,55000]);p.add_argument('--weights',type=float,nargs='+');p.add_argument('--output',default='outputs/slayer1b-average.pt');a=p.parse_args()
print(json.dumps(from_hub(a.steps,a.weights or [1.]*len(a.steps),a.output),indent=2))
|