File size: 4,391 Bytes
05c768b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
"""Average compatible native Slayer full checkpoints in FP32, with provenance."""
from pathlib import Path
import argparse,hashlib,json,math
import torch
from huggingface_hub import hf_hub_download
REPO='SlayerLab/Slayer1B-1.09-checkpoints'
REVISION='32a7304dc2208acb3fb4a6e957498116552a4b18'

def sha256(path):
    h=hashlib.sha256()
    with open(path,'rb') as f:
        for chunk in iter(lambda:f.read(8*1024*1024),b''):h.update(chunk)
    return h.hexdigest()

def average(paths,weights,output):
    if not paths or len(paths)!=len(weights):raise ValueError('One weight per checkpoint is required')
    if not all(math.isfinite(w) and w>=0 for w in weights) or sum(weights)<=0:raise ValueError('Need finite nonnegative weights with positive total')
    if len(set(map(str,paths)))!=len(paths):raise ValueError('Duplicate checkpoint paths')
    weights=[w/sum(weights) for w in weights]
    accumulator={};config=None;keys=None;dtypes={};sources=[]
    for path,weight in zip(paths,weights):
        checkpoint=torch.load(path,map_location='cpu',weights_only=True,mmap=True)
        state=checkpoint['model']
        if config is None:
            config=dict(checkpoint['config']);keys=set(state)
        elif checkpoint['config']!=config or set(state)!=keys:raise ValueError('Configuration or parameter keys differ')
        if not torch.equal(state['tok.weight'],state['head.weight']):raise ValueError('Source embeddings/head are not tied-equivalent')
        for key,tensor in state.items():
            if not isinstance(tensor,torch.Tensor):raise ValueError('Expected tensor state')
            if key not in accumulator:
                dtypes[key]=tensor.dtype
                accumulator[key]=torch.zeros_like(tensor,dtype=torch.float32) if tensor.is_floating_point() else tensor.clone()
            elif accumulator[key].shape!=tensor.shape or dtypes[key]!=tensor.dtype:raise ValueError(f'Shape/dtype mismatch: {key}')
            if tensor.is_floating_point():
                if not torch.isfinite(tensor).all():raise ValueError(f'Nonfinite source: {key}')
                accumulator[key].add_(tensor,alpha=weight)
            elif not torch.equal(accumulator[key],tensor):raise ValueError(f'Nonfloat buffers differ: {key}')
        sources.append({'file':Path(path).name,'sha256':sha256(path),'step':int(checkpoint['step']),'weight':weight})
        del state,checkpoint
        print('Accumulated',sources[-1],flush=True)
    for key,tensor in accumulator.items():
        if tensor.is_floating_point() and not torch.isfinite(tensor).all():raise ValueError('Nonfinite average')
    # Preserve aliasing explicitly in the output checkpoint.
    accumulator['head.weight']=accumulator['tok.weight']
    merged={'model':accumulator,'config':config,'step':0,
            'merge_metadata':{'method':'normalized arithmetic weight average in FP32','sources':sources,
                              'step_note':'0 denotes a derived model, not an optimizer update'}}
    output=Path(output);output.parent.mkdir(parents=True,exist_ok=True)
    temp=output.with_suffix(output.suffix+'.tmp');torch.save(merged,temp);temp.replace(output)
    metadata={**merged['merge_metadata'],'output_sha256':sha256(output)}
    output.with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
    return metadata

def from_hub(steps,weights,output,revision=REVISION):
    sums=Path(hf_hub_download(REPO,'SHA256SUMS',revision=revision)).read_text()
    expected={line.split()[1]:line.split()[0] for line in sums.splitlines() if line.strip()}
    paths=[]
    for step in steps:
        name=f'wagi_{step}.pt'
        path=hf_hub_download(REPO,name,revision=revision)
        if sha256(path)!=expected[name]:raise ValueError(f'Checksum mismatch: {name}')
        paths.append(path)
    metadata=average(paths,weights,output)
    metadata.update(repo=REPO,revision=revision,source_filenames=[f'wagi_{s}.pt' for s in steps])
    Path(output).with_suffix('.merge.json').write_text(json.dumps(metadata,indent=2)+'\n')
    return metadata

if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('--steps',type=int,nargs='+',default=[45000,50000,55000]);p.add_argument('--weights',type=float,nargs='+');p.add_argument('--output',default='outputs/slayer1b-average.pt');a=p.parse_args()
    print(json.dumps(from_hub(a.steps,a.weights or [1.]*len(a.steps),a.output),indent=2))