"""Average compatible native Slayer full checkpoints in FP32, with provenance.""" from pathlib import Path import argparse,hashlib,json,math import torch from huggingface_hub import hf_hub_download REPO='SlayerLab/Slayer1B-1.09-checkpoints' REVISION='32a7304dc2208acb3fb4a6e957498116552a4b18' def sha256(path): h=hashlib.sha256() with open(path,'rb') as f: for chunk in iter(lambda:f.read(8*1024*1024),b''):h.update(chunk) return h.hexdigest() def average(paths,weights,output): if not paths or len(paths)!=len(weights):raise ValueError('One weight per checkpoint is required') if not all(math.isfinite(w) and w>=0 for w in weights) or sum(weights)<=0:raise ValueError('Need finite nonnegative weights with positive total') if len(set(map(str,paths)))!=len(paths):raise ValueError('Duplicate checkpoint paths') weights=[w/sum(weights) for w in weights] accumulator={};config=None;keys=None;dtypes={};sources=[] for path,weight in zip(paths,weights): checkpoint=torch.load(path,map_location='cpu',weights_only=True,mmap=True) state=checkpoint['model'] if config is None: config=dict(checkpoint['config']);keys=set(state) elif checkpoint['config']!=config or set(state)!=keys:raise ValueError('Configuration or parameter keys differ') if not torch.equal(state['tok.weight'],state['head.weight']):raise ValueError('Source embeddings/head are not tied-equivalent') for key,tensor in state.items(): if not isinstance(tensor,torch.Tensor):raise ValueError('Expected tensor state') if key not in accumulator: dtypes[key]=tensor.dtype accumulator[key]=torch.zeros_like(tensor,dtype=torch.float32) if tensor.is_floating_point() else tensor.clone() elif accumulator[key].shape!=tensor.shape or dtypes[key]!=tensor.dtype:raise ValueError(f'Shape/dtype mismatch: {key}') if tensor.is_floating_point(): if not torch.isfinite(tensor).all():raise ValueError(f'Nonfinite source: {key}') accumulator[key].add_(tensor,alpha=weight) elif not torch.equal(accumulator[key],tensor):raise ValueError(f'Nonfloat buffers differ: {key}') sources.append({'file':Path(path).name,'sha256':sha256(path),'step':int(checkpoint['step']),'weight':weight}) del state,checkpoint print('Accumulated',sources[-1],flush=True) for key,tensor in accumulator.items(): if tensor.is_floating_point() and not torch.isfinite(tensor).all():raise ValueError('Nonfinite average') # Preserve aliasing explicitly in the output checkpoint. accumulator['head.weight']=accumulator['tok.weight'] merged={'model':accumulator,'config':config,'step':0, 'merge_metadata':{'method':'normalized arithmetic weight average in FP32','sources':sources, 'step_note':'0 denotes a derived model, not an optimizer update'}} output=Path(output);output.parent.mkdir(parents=True,exist_ok=True) temp=output.with_suffix(output.suffix+'.tmp');torch.save(merged,temp);temp.replace(output) metadata={**merged['merge_metadata'],'output_sha256':sha256(output)} output.with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n') return metadata def from_hub(steps,weights,output,revision=REVISION): sums=Path(hf_hub_download(REPO,'SHA256SUMS',revision=revision)).read_text() expected={line.split()[1]:line.split()[0] for line in sums.splitlines() if line.strip()} paths=[] for step in steps: name=f'wagi_{step}.pt' path=hf_hub_download(REPO,name,revision=revision) if sha256(path)!=expected[name]:raise ValueError(f'Checksum mismatch: {name}') paths.append(path) metadata=average(paths,weights,output) metadata.update(repo=REPO,revision=revision,source_filenames=[f'wagi_{s}.pt' for s in steps]) Path(output).with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n') return metadata if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('--steps',type=int,nargs='+',default=[45000,50000,55000]);p.add_argument('--weights',type=float,nargs='+');p.add_argument('--output',default='outputs/slayer1b-average.pt');a=p.parse_args() print(json.dumps(from_hub(a.steps,a.weights or [1.]*len(a.steps),a.output),indent=2))