Add five native Slayer notebooks: generation, CPT, SFT, RL and measured checkpoint averaging
05c768b verified Download tutorials/average_checkpoints.py from SlayerLab/Slayer1B-Poetry: direct link, hf CLI and curl.
- Browser
- Download file 4.39 kB
-
https://huggingface.co/SlayerLab/Slayer1B-Poetry/resolve/main/tutorials/average_checkpoints.py
- Command line
-
hf download hf://SlayerLab/Slayer1B-Poetry/tutorials/average_checkpoints.py
-
curl -L -o average_checkpoints.py https://huggingface.co/SlayerLab/Slayer1B-Poetry/resolve/main/tutorials/average_checkpoints.py
4.39 kB
| """Average compatible native Slayer full checkpoints in FP32, with provenance.""" | |
| from pathlib import Path | |
| import argparse,hashlib,json,math | |
| import torch | |
| from huggingface_hub import hf_hub_download | |
| REPO='SlayerLab/Slayer1B-1.09-checkpoints' | |
| REVISION='32a7304dc2208acb3fb4a6e957498116552a4b18' | |
| def sha256(path): | |
| h=hashlib.sha256() | |
| with open(path,'rb') as f: | |
| for chunk in iter(lambda:f.read(8*1024*1024),b''):h.update(chunk) | |
| return h.hexdigest() | |
| def average(paths,weights,output): | |
| if not paths or len(paths)!=len(weights):raise ValueError('One weight per checkpoint is required') | |
| if not all(math.isfinite(w) and w>=0 for w in weights) or sum(weights)<=0:raise ValueError('Need finite nonnegative weights with positive total') | |
| if len(set(map(str,paths)))!=len(paths):raise ValueError('Duplicate checkpoint paths') | |
| weights=[w/sum(weights) for w in weights] | |
| accumulator={};config=None;keys=None;dtypes={};sources=[] | |
| for path,weight in zip(paths,weights): | |
| checkpoint=torch.load(path,map_location='cpu',weights_only=True,mmap=True) | |
| state=checkpoint['model'] | |
| if config is None: | |
| config=dict(checkpoint['config']);keys=set(state) | |
| elif checkpoint['config']!=config or set(state)!=keys:raise ValueError('Configuration or parameter keys differ') | |
| if not torch.equal(state['tok.weight'],state['head.weight']):raise ValueError('Source embeddings/head are not tied-equivalent') | |
| for key,tensor in state.items(): | |
| if not isinstance(tensor,torch.Tensor):raise ValueError('Expected tensor state') | |
| if key not in accumulator: | |
| dtypes[key]=tensor.dtype | |
| accumulator[key]=torch.zeros_like(tensor,dtype=torch.float32) if tensor.is_floating_point() else tensor.clone() | |
| elif accumulator[key].shape!=tensor.shape or dtypes[key]!=tensor.dtype:raise ValueError(f'Shape/dtype mismatch: {key}') | |
| if tensor.is_floating_point(): | |
| if not torch.isfinite(tensor).all():raise ValueError(f'Nonfinite source: {key}') | |
| accumulator[key].add_(tensor,alpha=weight) | |
| elif not torch.equal(accumulator[key],tensor):raise ValueError(f'Nonfloat buffers differ: {key}') | |
| sources.append({'file':Path(path).name,'sha256':sha256(path),'step':int(checkpoint['step']),'weight':weight}) | |
| del state,checkpoint | |
| print('Accumulated',sources[-1],flush=True) | |
| for key,tensor in accumulator.items(): | |
| if tensor.is_floating_point() and not torch.isfinite(tensor).all():raise ValueError('Nonfinite average') | |
| # Preserve aliasing explicitly in the output checkpoint. | |
| accumulator['head.weight']=accumulator['tok.weight'] | |
| merged={'model':accumulator,'config':config,'step':0, | |
| 'merge_metadata':{'method':'normalized arithmetic weight average in FP32','sources':sources, | |
| 'step_note':'0 denotes a derived model, not an optimizer update'}} | |
| output=Path(output);output.parent.mkdir(parents=True,exist_ok=True) | |
| temp=output.with_suffix(output.suffix+'.tmp');torch.save(merged,temp);temp.replace(output) | |
| metadata={**merged['merge_metadata'],'output_sha256':sha256(output)} | |
| output.with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n') | |
| return metadata | |
| def from_hub(steps,weights,output,revision=REVISION): | |
| sums=Path(hf_hub_download(REPO,'SHA256SUMS',revision=revision)).read_text() | |
| expected={line.split()[1]:line.split()[0] for line in sums.splitlines() if line.strip()} | |
| paths=[] | |
| for step in steps: | |
| name=f'wagi_{step}.pt' | |
| path=hf_hub_download(REPO,name,revision=revision) | |
| if sha256(path)!=expected[name]:raise ValueError(f'Checksum mismatch: {name}') | |
| paths.append(path) | |
| metadata=average(paths,weights,output) | |
| metadata.update(repo=REPO,revision=revision,source_filenames=[f'wagi_{s}.pt' for s in steps]) | |
| Path(output).with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n') | |
| return metadata | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser();p.add_argument('--steps',type=int,nargs='+',default=[45000,50000,55000]);p.add_argument('--weights',type=float,nargs='+');p.add_argument('--output',default='outputs/slayer1b-average.pt');a=p.parse_args() | |
| print(json.dumps(from_hub(a.steps,a.weights or [1.]*len(a.steps),a.output),indent=2)) | |