Repository navigation
Expand file tree
/
Copy pathqwen4_iq2.py
More file actions
216 lines (200 loc) · 9.89 KB
/
Copy pathqwen4_iq2.py
File metadata and controls
216 lines (200 loc) · 9.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
#!/usr/bin/env python3
"""Build calibrated IQ2_XXS gate/up or padded Q2_K down trunk tensors.
The two stages allow quantization beside the original BF16 checkpoint and
assembly beside an existing qwen4exp model. All other tensors are byte-copied.
"""
import argparse
import hashlib
import json
import math
from pathlib import Path
import struct
import time
from qwen4_pack import (SourceDB, QwenGGUFImatrix, GGMLQuantizer,
Q2_IMATRIX_SHA256, encode_weighted_experts)
from qwen4_pack_to_qwen4exp import Reader, kv_bytes, w_str, tnbytes
def tensor_bytes(kind, dims):
n = math.prod(dims)
if kind in (10, 16):
if dims[0] % 256:
raise ValueError('K-quant weight rows must be block aligned')
return n // 256 * (84 if kind == 10 else 66)
return tnbytes(kind, n)
def digest(path):
h = hashlib.sha256()
with path.open('rb') as f:
for chunk in iter(lambda: f.read(8 << 20), b''):
h.update(chunk)
return h.hexdigest()
def save(path, value):
tmp = path.with_suffix('.tmp')
tmp.write_text(json.dumps(value, indent=2) + '\n')
tmp.replace(path)
def quantize(args):
args.experts_dir.mkdir(parents=True, exist_ok=True)
db = SourceDB(args.source)
imatrix = QwenGGUFImatrix(args.imatrix, Q2_IMATRIX_SHA256)
quant = GGMLQuantizer(args.library)
quant.lib.ds4q_quantize_init(16) # Initialize shared lookup tables before workers.
manifest = args.experts_dir / 'experts.json'
down = args.projection == 'down'
qtype = 'Q2_K' if down else 'IQ2_XXS'
identity = {'source': str(args.source.resolve()), 'imatrix': imatrix.provenance(),
'library_sha256': digest(args.library), 'format': qtype}
if down:
identity['padding'] = {'logical_input': 640, 'physical_input': 768}
report = json.loads(manifest.read_text()) if manifest.exists() else {**identity, 'tensors': {}}
for key, value in identity.items():
if report[key] != value:
raise SystemExit(f'Resume provenance mismatch: {key}')
for layer in range(48):
source = f'model.language_model.layers.{layer}.mlp.experts.' + ('down_proj' if down else 'gate_up_proj')
info = db.tensors[source]
shape = (512, 2560, 640) if down else (512, 1280, 2560)
if info['dtype'] != 'BF16' or info['shape'] != shape:
raise SystemExit(f'Unexpected source layout: {source}: {info}')
names = [f'blk.{layer}.ffn_{part}_exps.weight' for part in (('down',) if down else ('gate', 'up'))]
if all(n in report['tensors'] and (args.experts_dir / n).is_file() and
digest(args.experts_dir / n) == report['tensors'][n]['sha256'] for n in names):
print(f'Layer {layer}: verified existing tensors', flush=True)
continue
values = db.read(source)
source_hash = hashlib.sha256(values).hexdigest()
for part, name in enumerate(names):
start = time.monotonic()
raw = encode_weighted_experts(values if down else values[:, part * 640:(part + 1) * 640, :],
qtype, imatrix, name, quant, args.threads,
pad_last_to=768 if down else 0)
if len(raw) != (512 * 2560 * 3 * 84 if down else 512 * 640 * 10 * 66):
raise SystemExit(f'Invalid output size for {name}')
path = args.experts_dir / name
tmp = path.with_suffix('.incomplete')
tmp.write_bytes(raw)
tmp.replace(path)
report['tensors'][name] = {'sha256': hashlib.sha256(raw).hexdigest(),
'bytes': len(raw), 'source_tensor': source, 'source_sha256': source_hash,
'seconds': time.monotonic() - start}
save(manifest, report)
print(f'{name}: {len(raw)} bytes, {time.monotonic() - start:.1f}s', flush=True)
del values
imatrix.close()
def assemble(args):
if args.out.exists() or Path(str(args.out) + '.incomplete').exists():
raise SystemExit('Output already exists; choose a new path')
manifest = json.loads((args.experts_dir / 'experts.json').read_text())
down = args.projection == 'down'
expected = {f'blk.{i}.ffn_{p}_exps.weight' for i in range(48) for p in (('down',) if down else ('gate', 'up'))}
if set(manifest['tensors']) != expected:
raise SystemExit(f'Expected all {len(expected)} calibrated trunk tensors')
if manifest['format'] != ('Q2_K' if down else 'IQ2_XXS'):
raise SystemExit('Quantization format does not match projection')
if down and manifest.get('padding') != {'logical_input': 640, 'physical_input': 768}:
raise SystemExit('Invalid down projection padding provenance')
base = Reader(str(args.template))
if base.kv.get('general.architecture') != 'qwen4exp' or base.kv.get('qwen4exp.block_count') != 49:
raise SystemExit('Template must be a combined 49-layer qwen4exp GGUF')
if 'per_layer_token_embd.weight' in base.tensors:
raise SystemExit('Template must use an external PLE sidecar')
if down:
for layer in range(48):
for part in ('gate', 'up'):
kind, dims, _ = base.tensors[f'blk.{layer}.ffn_{part}_exps.weight']
if kind != 16 or dims != [2560, 640, 512]:
raise SystemExit('Down experiment requires the existing IQ2_XXS trunk recipe')
alignment = base.kv.get('general.alignment', 32)
align = lambda n: (n + alignment - 1) // alignment * alignment
metadata = {k: (base.kv_types[k], v) for k, v in base.kv.items()}
metadata['general.name'] = (8, 'Qwen3.8 Flash Next IQ2_XXS imatrix trunk, ' +
('padded Q2_K down' if down else 'MXFP4 down') + ', MTP')
metadata['general.file_type'] = (4, 19) # LLAMA_FTYPE_MOSTLY_IQ2_XXS
metadata['ds4.iq2.imatrix_sha256'] = (8, manifest['imatrix']['sha256'])
if down:
metadata['ds4.qwen4.down.logical_input'] = (4, 640)
metadata['ds4.qwen4.down.physical_input'] = (4, 768)
plan = []
offset = 0
for name, (kind, dims, old_offset) in base.tensors.items():
replace = name in expected
if replace and (kind != (39 if down else 12) or dims != ([640, 2560, 512] if down else [2560, 640, 512])):
raise SystemExit(f'Unexpected template expert layout: {name}')
if replace:
kind = 10 if down else 16
if down:
dims = [768, 2560, 512]
size = tensor_bytes(kind, dims)
if replace:
path = args.experts_dir / name
if path.stat().st_size != size or digest(path) != manifest['tensors'][name]['sha256']:
raise SystemExit(f'Invalid calibrated payload: {name}')
plan.append((name, kind, dims, offset, size, old_offset))
offset = align(offset + size)
prefix = b'GGUF' + struct.pack('<IQQ', 3, len(plan), len(metadata))
prefix += b''.join(kv_bytes(k, t, v) for k, (t, v) in metadata.items())
for name, kind, dims, pos, size, old in plan:
prefix += w_str(name) + struct.pack('<I', len(dims))
prefix += struct.pack('<' + 'Q' * len(dims), *dims) + struct.pack('<IQ', kind, pos)
data_start = align(len(prefix))
args.out.parent.mkdir(parents=True, exist_ok=True)
tmp = Path(str(args.out) + '.incomplete')
records = {}
with tmp.open('wb') as out:
out.write(prefix)
out.write(bytes(data_start - len(prefix)))
for name, kind, dims, pos, size, old in plan:
out.seek(data_start + pos)
src = (args.experts_dir / name).open('rb') if name in expected else base.f
if src is base.f:
src.seek(base.data_start + old)
h = hashlib.sha256()
left = size
while left:
chunk = src.read(min(left, 8 << 20))
if not chunk:
raise SystemExit(f'Short tensor read: {name}')
out.write(chunk)
h.update(chunk)
left -= len(chunk)
if src is not base.f:
src.close()
records[name] = {'type': kind, 'bytes': size, 'sha256': h.hexdigest(),
'replaced': name in expected}
out.truncate(data_start + offset)
# Read back every payload before publishing the completed artifact.
check = Reader(str(tmp))
for name, rec in records.items():
check.f.seek(check.data_start + check.tensors[name][2])
h = hashlib.sha256()
left = rec['bytes']
while left:
chunk = check.f.read(min(left, 8 << 20))
if not chunk:
raise SystemExit(f'Short verification read: {name}')
h.update(chunk)
left -= len(chunk)
if h.hexdigest() != rec['sha256']:
raise SystemExit(f'Written tensor mismatch: {name}')
check.f.close()
base.f.close()
tmp.replace(args.out)
save(Path(str(args.out) + '.json'), {'template': str(args.template.resolve()),
'quantization': manifest, 'tensors': records, 'bytes': args.out.stat().st_size,
'sha256': digest(args.out)})
print(f'Verified {args.out}: {args.out.stat().st_size} bytes', flush=True)
def main():
ap = argparse.ArgumentParser(description=__doc__)
sub = ap.add_subparsers(dest='stage', required=True)
q = sub.add_parser('quantize')
q.add_argument('--source', type=Path, required=True)
q.add_argument('--imatrix', type=Path, required=True)
q.add_argument('--library', type=Path, required=True)
q.add_argument('--threads', type=int, default=8)
a = sub.add_parser('assemble')
a.add_argument('--template', type=Path, required=True)
a.add_argument('--out', type=Path, required=True)
for p in (q, a):
p.add_argument('--experts-dir', type=Path, required=True)
p.add_argument('--projection', choices=('gate-up', 'down'), default='gate-up')
args = ap.parse_args()
(quantize if args.stage == 'quantize' else assemble)(args)
if __name__ == '__main__':
main()