Repository navigation
Expand file tree
/
Copy pathtrain.py
More file actions
2750 lines (2590 loc) · 142 KB
/
Copy pathtrain.py
File metadata and controls
2750 lines (2590 loc) · 142 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""
Training mini-AGI.
read point it at files and let it read them, continually. This is the
one that matters, and the one the run uses.
stream the same mechanism over a packed corpus: batch 1, a KV cache,
one chunk of characters at a time. Peak memory is set by
--chunk and nothing else, so --context is nearly free to extend.
probe what depth the model chooses, per character.
Everything here runs at batch 1 behind a cache. The fixed-window regime that
preceded it - several windows per step, one autograd graph over each whole
window - cost about three times as much at an eighth of the reach: 6857 MB at
4k characters against 3090 MB at 32k, because the retained activation graph
was bounded by the window rather than by the chunk.
"""
import os
import re
import sys
import json
import collections
import math
import time
import argparse
from dataclasses import asdict
# BEFORE torch, because the allocator reads this once at CUDA init and ignores
# it afterwards. Without it PyTorch's caching allocator keeps its free blocks in
# fixed segments, and a run that alternates between a few large short-lived
# tensors - which is exactly what a checkpointed expert dispatch does, three
# batched matmuls recomputed in the backward - strands memory it cannot hand
# back. Measured on an 8,192 window: the backward asked for 384 MiB with 394
# MiB free and 861 MiB reserved but unallocated. There was plenty of memory;
# there was no contiguous piece of it. Set it here rather than in a shell so it
# is a property of the program, not of how it happened to be launched.
# PyTorch 2.9 renamed the variable and warns about the old name; earlier
# versions read only the old one.
def _alloc_var():
try:
from importlib.metadata import version
major, minor = (int(v) for v in version("torch").split(".")[:2])
return "PYTORCH_ALLOC_CONF" if (major, minor) >= (2, 9) else "PYTORCH_CUDA_ALLOC_CONF"
except Exception:
10000
# noqa: BLE001
return "PYTORCH_CUDA_ALLOC_CONF"
if not any(v in os.environ for v in ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")):
os.environ[_alloc_var()] = "expandable_segments:True"
import numpy as np
import torch
import torch.nn as nn
from minagi.recur import RecurConfig, RecurCoder, load_recur
from minagi.model import watch_logits
from minagi.pool import PooledMLP, AutoGrow, split_trunk_pool
from minagi.stream import StreamSet, Evaluator, detach_caches, ramp_context
from minagi.plasticity import Plasticity
from minagi.governor import Governor, DEFAULTS as CHECKS, steps_of
from minagi.lanes import LaneSchedule
from minagi.optim import GradSNR
from minagi import heldout
from minagi import store as weights_store
def lr_at(step, total, base, warmup, floor_frac=0.1):
if step < warmup:
return base * (step + 1) / warmup
prog = (step - warmup) / max(1, total - warmup)
return base * (floor_frac + (1 - floor_frac) * 0.5 * (1 + math.cos(math.pi * prog)))
def _resync_opt(opt, model, args):
"""Drop parameters the model no longer has, adopt the ones it gained.
Growth appends experts and replaces the gate tensor; pruning removes them
and replaces it again. Either way the optimiser is left holding tensors the
model does not have - Adam would keep stepping them and keep their moments
alive, a leak on a pool that grows and shrinks all run.
"""
live = {id(p) for p in model.parameters()}
for g in opt.param_groups:
for p_ in g["params"]:
if id(p_) not in live:
opt.state.pop(p_, None)
g["params"] = [p for p in g["params"] if id(p) in live]
known = {id(p) for g in opt.param_groups for p in g["params"]}
fresh = [p for p in model.parameters() if id(p) not in known]
mats = [p for p in fresh if p.dim() >= 2]
vecs = [p for p in fresh if p.dim() < 2]
# a group added mid-run needs the anchor the schedule scales from, or it
# would sit at whatever rate it was born with while everything else decays
base = args.lr
from minagi.muon import adopt
for on, ps in (adopt(opt, model, mats) if mats else []):
opt.add_param_group({"params": ps, "name": "pool", "muon": on,
"weight_decay": args.wd,
"lr": base, "base_lr": base})
if vecs:
opt.add_param_group({"params": vecs, "name": "pool",
"weight_decay": 0.0,
"lr": base, "base_lr": base})
def _auto_device():
"""
The device to use when --device is not given: CUDA (or ROCm, which torch
also calls cuda) where there is one, else Apple Silicon's GPU (MPS), else
the CPU. An MPS forward computes in fp32 - minagi/precision.py turns
autocast on only for CUDA, ROCm, XPU and a CPU with bf16 arithmetic.
"""
if torch.cuda.is_available():
return "cuda"
mps = getattr(torch.backends, "mps", None)
if mps is not None and mps.is_available():
return "mps"
return "cpu"
def cmd_stream(args):
"""
Train the way the model runs: batch 1, a KV cache, one chunk at a time.
The old loop built one autograd graph over the whole window. That cost
about four times what reading the same window costs - 6857 MB against 1704
- and the difference was entirely activations retained across 26 block
applications. So the model could be served at a context it could not be
trained at, and training kept hitting OOM on a card with room to spare.
Here the graph is bounded by --chunk and the reach is bounded by the cache,
which is linear and cheap. Batch 1 gives up gradient averaging; what it
buys is roughly an order of magnitude of context on the same card.
"""
device = torch.device(args.device)
torch.manual_seed(args.seed)
rng = np.random.default_rng(args.seed)
sources = []
for part in args.mix.split(","):
name, _, w = part.partition(":")
sources.append((name.strip(), float(w) if w else 1.0))
wdir = args.weights_dir
man = None
if os.path.exists(os.path.join(wdir, "manifest.json")):
with open(os.path.join(wdir, "manifest.json")) as f:
man = json.load(f)
cfgd = dict(man["cfg"]) if man and man.get("cfg") else {}
cfgd.update(vocab_size=265, block=args.context, use_pool=True)
cfgd["pool_max"] = args.pool_max
cfg = RecurConfig(**{k: v for k, v in cfgd.items()
if k in RecurConfig.__dataclass_fields__})
# The capacity bound comes from config.yaml rather than from the
# checkpoint, because it is a property of the machine the model is running
# on - how much VRAM the dispatch may use - not of the model. A directory
# written before this existed carries no value for it and would otherwise
# get the dataclass default, which is right but silent; taking it from the
# config every load means the number in the file is the number in effect.
try:
from minagi.config import load as _load_cfg, get as _get_cfg
cfg.pool_capacity_factor = float(
_get_cfg(_load_cfg(), "pool.capacity_factor",
cfg.pool_capacity_factor))
except Exception:
pass
model = RecurCoder(cfg).to(device)
if man and int(man["n_experts"]) != model.pool.n_experts():
have, want = model.pool.n_experts(), int(man["n_experts"])
if want > have:
model.pool.add_experts(want - have, device=device)
print(f" pool resized {have} -> {want} to match the directory")
print(f"params {model.n_params()/1e6:.2f}M | {model.pool.n_experts()} "
f"experts | block-applications {cfg.n_layer_effective}")
print(f"batch 1 | chunk {args.chunk} | context up to {cfg.block:,}")
print(f" the graph spans {args.chunk} characters; the cache spans the "
f"context. only the first costs activations.")
pool_ids = set()
sites = [m for m in model.modules() if isinstance(m, PooledMLP)]
pool_ids = {id(q) for e in model.pool.experts for q in e.parameters()}
pool_ids |= {id(st.router.weight) for st in sites}
pool_ids |= {id(st.depth_emb) for st in sites}
pool_ids.add(id(model.pool.gate))
trunk = [q for q in model.parameters() if id(q) not in pool_ids]
pool = [q for q in model.parameters() if id(q) in pool_ids]
opt = torch.optim.AdamW(
[{"params": [q for q in trunk if q.dim() >= 2], "name": "trunk",
"weight_decay": args.wd},
{"params": [q for q in trunk if q.dim() < 2], "name": "trunk",
"weight_decay": 0.0},
{"params": [q for q in pool if q.dim() >= 2], "name": "pool",
"weight_decay": args.wd},
{"params": [q for q in pool if q.dim() < 2], "name": "pool",
"weight_decay": 0.0}],
lr=args.lr, betas=(0.9, 0.95), fused=(device.type == "cuda"))
print(f"trunk learns at {args.trunk_lr_mult:g}x the pool's rate "
f"({sum(q.numel() for q in trunk)/1e6:.2f}M trunk, "
f"{sum(q.numel() for q in pool)/1e6:.2f}M pool)")
start_step = 0
if man:
weights_store.load(model, wdir, opt=opt, device=device, verbose=True)
start_step = int(man.get("step", -1)) + 1
print(f"resuming from {wdir} at step {start_step}")
grower = None
if args.grow_k > 0:
# args.pool_max, not cfg.pool_max - the latter is the router width
grower = AutoGrow(grow_k=args.grow_k, max_experts=args.pool_max,
mem_frac_max=args.grow_mem_frac)
model.pool.born.fill_(float(start_step))
model.pool.gate_seen.copy_(model.pool.gate.data.abs())
print(f"auto-growth armed: {model.pool.n_experts()} experts, ceiling "
f"{args.pool_max}, VRAM brake at {100*args.grow_mem_frac:.0f}%")
streams = StreamSet(model, sources, args.chunk, args.ctx_start, device,
rng, replay=args.replay,
replay_burst=args.replay_burst)
evaluator = Evaluator(model, [n for n, _ in sources], args.chunk,
args.ctx_start, device)
tot = sum(w for _, w in sources)
for name, w in sources:
n_ch = len(np.memmap(os.path.join(name, "train.bin"),
dtype=np.uint16, mode="r"))
print(f" {name:<22}{100*w/tot:>4.0f}% {n_ch/1e6:>7.1f}M characters")
if args.ctx_start < args.context:
print(f"context ramps {args.ctx_start:,} -> {args.context:,} over the "
f"first third of the run")
os.makedirs(args.out, exist_ok=True)
# A run keeps its own history rather than leaving it in a log. A log is one
# shell redirect away from being overwritten by the next run, and two
# processes writing one log here once produced a number that looked like a
# model bug and was not. Appended a line at a time, so an interrupted run
# keeps everything up to the interruption.
hist_path = os.path.join(args.out, "history.jsonl")
hist = open(hist_path, "a", buffering=1, encoding="utf-8")
def record(kind, step=None, **kw):
hist.write(json.dumps({"kind": kind, "step": step, **kw}) + "\n")
record("start", weights_dir=wdir, context=cfg.block, chunk=args.chunk,
accum=args.accum, lr=args.lr, trunk_lr_mult=args.trunk_lr_mult,
steps=args.steps, start_step=start_step,
experts=model.pool.n_experts(), params=model.n_params(),
mix=[{"name": n, "weight": w} for n, w in sources])
# the directory's own score, not infinity - resuming must not let a worse
# state call itself the best merely because this process just started
best = weights_store.best_val(wdir)
if best < float("inf"):
print(f"best on disk: val {best:.4f} - weights/ advances only past it")
reverts = 0
t0, seen = time.time(), 0
model.train()
for step in range(start_step, args.steps):
lr = lr_at(step, args.steps, args.lr, args.warmup)
for g in opt.param_groups:
g["lr"] = lr * (args.trunk_lr_mult if g["name"] == "trunk" else 1.0)
ctx_now = ramp_context(step - start_step, max(1, args.steps - start_step),
args.ctx_start, args.context)
if ctx_now != streams.context:
streams.set_context(ctx_now)
evaluator.context = ctx_now
print(f" context -> {ctx_now:,} characters", flush=True)
# Accumulate over consecutive chunks before stepping. Each chunk's
# graph is freed by its own backward, so this costs no memory beyond
# one chunk - but the gradient the optimiser sees averages
# accum x chunk characters instead of chunk. That is the whole gap
# between this and the batch trainer: at chunk 512 a step saw 512
# characters where a batch step saw 32768, and with accumulation the
# two can be made identical without giving up the memory profile.
opt.zero_grad(set_to_none=True)
for _ in range(args.accum):
loss, src = streams.step(learn=True, aux_weight=cfg.pool_aux)
(loss / args.accum).backward()
detach_caches(streams.active.caches)
seen += args.chunk
gn = torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
opt.step()
if step % args.log_every == 0:
el = max(time.time() - t0, 1e-9)
vram = (f" vram {torch.cuda.max_memory_allocated()/1e6:.0f}MB"
if device.type == "cuda" else "")
record("step", step=step, loss=float(loss), lr=lr,
grad_norm=float(gn), chars=seen, chars_per_s=seen / el,
context=streams.context, experts=model.pool.n_experts(),
vram_mb=(torch.cuda.max_memory_allocated() / 1e6
if device.type == "cuda" else None))
rp = (f" replay {streams.replays}" if args.replay > 0 else "")
print(f"step {step:>6}/{args.steps} loss {float(loss):.4f} "
f"lr {lr:.2e} gn {float(gn):.2f} ctx {streams.context//1024}k "
f"{seen/el/1e3:.1f}k char/s{vram}{rp}", flush=True)
if (step + 1) % args.eval_every == 0 or step + 1 == args.steps:
v = evaluator.run(args.eval_chunks)
se = v.pop("stderr", 0.0)
val = float(np.mean(list(v.values())))
record("val", step=step, val=val, stderr=se, ppl=math.exp(val),
per_domain=dict(v), experts=model.pool.n_experts())
print(f" val {val:.4f} +/-{se:.4f} ppl {math.exp(val):.2f} "
+ " ".join(f"{k.replace('data_','').replace('_char','')} "
f"{x:.3f}" for k, x in v.items()), flush=True)
if grower is not None:
gone = model.pool.prune(step, survival=args.grow_min_age)
if gone:
_resync_opt(opt, model, args)
record("pruned", step=step, n=gone,
to=model.pool.n_experts())
print(f" pruned {gone} -> {model.pool.n_experts()} experts",
flush=True)
rec = grower.step(val, model.pool, step)
if rec["grew"]:
_resync_opt(opt, model, args)
record("grew", step=step, n=rec["grew"], to=rec["experts"],
reason=rec.get("reason"))
print(f" GREW +{rec['grew']} -> {rec['experts']} experts "
f"| {rec['reason']}", flush=True)
elif rec.get("reason"):
record("held", step=step, reason=rec["reason"],
experts=model.pool.n_experts())
print(f" [pool] {rec['reason']}", flush=True)
# The directory holds the best state the model has reached, so it
# is written when a run improves on it and left alone otherwise.
# Keeping a second copy aside as a snapshot would double 236 MB
# that grows with the pool, to hold a copy of what is already here.
if val < best:
best = val
weights_store.save(model, wdir, step=step, val=val, opt=opt,
cfg=asdict(cfg), verbose=True)
record("saved", step=step, val=val)
print(f" improved - weights/ advanced to val {val:.4f}",
flush=True)
elif args.revert_factor > 0 and val > best * args.revert_factor:
# A run can diverge: extending the context too fast once took
# validation from 0.7375 to 5.4490 in a thousand steps. Without
# this the live directory would carry that away with it.
print(f" DIVERGED: val {val:.4f} is {val/best:.1f}x the best "
f"{best:.4f} - reloading weights/ and halving the "
f"learning rate", flush=True)
with open(os.path.join(wdir, "manifest.json")) as _f:
man_b = json.load(_f)
n_back = int(man_b["n_experts"])
if n_back != model.pool.n_experts():
have = model.pool.n_experts()
if n_back > have:
model.pool.add_experts(n_back - have, device=device)
else:
model.pool.experts = nn.ModuleList(
list(model.pool.experts)[:n_back])
model.pool.gate = nn.Parameter(
model.pool.gate.data[:n_back].clone())
for b in ("use", "age", "born", "gate_seen"):
setattr(model.pool, b,
getattr(model.pool, b)[:n_back].clone())
model.pool.invalidate()
_resync_opt(opt, model, args)
weights_store.load(model, wdir, opt=opt, device=device)
args.lr *= 0.5
streams.set_context(max(args.ctx_start, streams.context // 2))
evaluator.context = streams.context
reverts += 1
print(f" back to val {man_b['val']:.4f} at "
f"{model.pool.n_experts()} experts, lr now "
f"{args.lr:.2e}, context back to {streams.context:,}",
flush=True)
if reverts >= args.max_reverts:
print(f" {reverts} reverts - stopping rather than "
f"thrashing", flush=True)
break
model.train()
record("done", minutes=(time.time() - t0) / 60, best=best,
experts=model.pool.n_experts())
hist.close()
print(f"\ndone in {(time.time()-t0)/60:.1f}m best val {best:.4f}")
print(f"history: {hist_path}")
return 0
class _Tracer:
"""
Record what the real training forward did, for the first few chunks.
Not a reconstruction: this sits inside the training loop, around the same
forward whose loss is about to be backpropagated, with the same KV cache,
the same precision, the same chunk and the same admitted experts. Whatever it
writes down is what actually happened.
The capture context is exited before backward on purpose. Gradient
checkpointing replays the forward during backward, and a capture still
open would record every call site twice.
"""
def __init__(self, path, chunks, swap_chunks=160):
self.path, self.want = path, chunks
self.want_swaps = max(swap_chunks, chunks)
self.ids, self.wts, self.meta, self.text = [], [], [], []
self.swaps = []
def full(self):
"""Routes are heavy, so only the first few chunks keep them."""
return len(self.ids) >= self.want
def done(self):
"""Swap counts are cheap and run on well past the routes."""
return self.full() and len(self.swaps) >= self.want_swaps
def add_swap(self, lane, moved, first):
"""
One line per training step: how many experts its window had to load.
`first` marks the chunk that opens a lane's window - the only place
the subject actually changes, so the place loads should concentrate.
"""
self.swaps.append({"subject": lane.name, "moved": int(moved),
"first_of_window": bool(first)})
def add(self, got, pool, lane, moved, step, loss, seen, ids):
import numpy as _np
if self.ids and ids.shape[1] != self.text[0].shape[0]:
return # a short tail chunk would not stack
self.ids.append(_np.stack([g[0].numpy() for g in got]).astype("int16"))
self.wts.append(_np.stack([g[1].numpy() for g in got]).astype("float16"))
self.text.append(ids[0].detach().cpu().numpy().astype("int16"))
self.meta.append({
"subject": lane.name, "moved": int(moved), "step": int(step),
"loss": float(loss), "seen": int(seen),
"resident": [int(e) for e in pool.slots if e >= 0],
"experts": int(pool.n_experts()),
})
written = False
def write(self, cfg, precision, chunk):
import json as _json
import numpy as _np
self.written = True
_np.savez_compressed(
self.path,
ids=_np.stack(self.ids), weights=_np.stack(self.wts),
text=_np.stack(self.text),
meta=_json.dumps({"chunks": self.meta, "swaps": self.swaps,
"chunk": chunk,
"top_k": cfg.pool_top_k,
"max_steps": cfg.max_steps,
"n_recur": cfg.n_recur, "n_coda": cfg.n_coda,
"n_prelude": cfg.n_prelude,
"context": cfg.block,
"precision": precision}))
print(f" routing trace -> {self.path} "
f"({len(self.ids)} chunks of real training)", flush=True)
class _Lane:
"""
One subject, sampled a window at a time.
Each visit opens a random file and a random offset inside it, then reads
one context window forward from there. The window is contiguous on
purpose: a lone chunk carries no history, so attention would have nothing
to reach back over and the context window would be decoration.
"""
def __init__(self, name, paths, rng):
self.name, self.paths, self.rng = name, paths, rng
self.r = None
self.visits = 0
def open(self, model, chunk, block, device, span=None):
"""
Open a file for this visit.
`block` is the context window; `span` is how much will be read before
the reader moves to another subject, which may be several windows. The
offset has to leave room for the whole visit, not just the first
window, or the file runs out part way through and the visit is cut
short.
"""
from minagi.stream import FileReader
from minagi.ingest import as_stream
span = max(span or block, block)
for _ in range(8): # a few tries for short files
path = self.paths[int(self.rng.integers(0, len(self.paths)))]
try:
data = as_stream(path)
except OSError:
# The corpus can change under a run that lasts days. The
# self-knowledge lane in particular is REGENERATED from the
# source, which deletes and rewrites every file in it, and the
# path list was taken at startup. A vanished file is a reason
# to pick another one, not to end a week of reading.
continue
if len(data) < 8:
continue
r = FileReader(model, data, path, chunk, block, device)
if len(data) > span + 1:
r.pos = int(self.rng.integers(0, len(data) - span - 1))
self.r = r
self.visits += 1
return r
return None
def rest(self, model):
"""
Drop the KV cache between visits.
A visit is exactly one context window, which is where FileReader
throws its cache away anyway - so this costs nothing, and it is what
keeps a single window of keys and values resident rather than one per
lane. At 26 block-applications a window is 1.7 GB, so four lanes
holding their own would not fit on the card at all.
"""
if self.r is not None:
self.r.seen = 0
self.r = None
def _lanes(files, seed, roots=None, resume=0):
"""
One lane per subject, where a subject is a TOP-LEVEL folder of the corpus.
Not the immediate parent: a corpus of any size grows subdirectories - a
hundred and fifty thousand articles cannot sit in
10BC8
one directory - and
grouping by the parent turns every one of those into its own subject. It
also matches what a reader means by their own files: pointing at ~/notes
with folders inside should give one subject per folder, not per nested
directory.
"""
roots = [os.path.abspath(r) for r in (roots or [])]
def subject(path):
p = os.path.abspath(path)
for r in roots:
if p.startswith(r + os.sep):
rel = os.path.relpath(p, r).split(os.sep)
# the first component under the root, or the root itself for
# a file lying directly in it
return rel[0] if len(rel) > 1 else os.path.basename(r)
return os.path.basename(os.path.dirname(p)) or p
by = {}
for f in files:
by.setdefault(subject(f), []).append(f)
# `resume` is how many characters the model has already read, and leaving
# it out was the single most expensive bug in this project.
#
# A lane picks a random file and a random offset inside it, from its own
# generator. Seeded by `seed + j` alone, that generator is rebuilt
# identically at every start, so every session replayed the SAME files at
# the SAME offsets in the SAME order, from the beginning. Measured over 43
# sessions: the median one read 3.2M characters, the longest 66M, and the
# counter reported 500M - but the UNIQUE text was about the length of the
# longest session, because the short ones were all re-reading the opening
# of the same sequence.
#
# It showed up as a restart spike nobody could place. Train loss fell hard
# (0.675 -> 0.446 at the last one) because the model was being handed text
# it had already memorised, and held-out rose at the same moment (0.925 ->
# 1.105) because it was being pushed to overfit that slice further. The
# learning rate was identical across the seam, nothing was pruned, and
# nothing was added - which is what made it look like lost state rather
# than repeated data.
#
# Mixing the read count into the seed means a restart continues into text
# the model has not seen instead of starting the same tape again. It stays
# reproducible from a given checkpoint, and needs nothing persisted: the
# character count is already in the manifest.
return [_Lane(k, sorted(by[k]), np.random.default_rng([seed, j, resume]))
for j, k in enumerate(sorted(by))]
def _lane_weights(spec, names):
"""
Fixed weights for the subjects, from "subject=weight,..." or a mapping,
each weight above 0 and at most 1; a subject not named weighs 1. Empty
when nothing is set - and then the schedule decides (minagi/lanes.py).
A subject at 0.25 gets a quarter of the reading of one at 1.
"""
if isinstance(spec, dict):
pairs = list(spec.items())
else:
pairs = [p.split("=", 1) for p in str(spec or "").split(",")
if p.strip()]
weights = {}
for pair in pairs:
if len(pair) != 2:
raise SystemExit(f"--lane-weights: {'='.join(pair)!r} is not "
f"subject=weight")
name, w = str(pair[0]).strip(), float(pair[1])
if name not in names:
raise SystemExit(f"--lane-weights names {name!r}, which is not a "
f"subject here: {', '.join(names)}")
if not 0 < w <= 1:
raise SystemExit(f"--lane-weights: {name} at {w:g} - a weight is "
f"above 0 and at most 1")
weights[name] = w
return weights
def _window_bounds(saved, start, block, ceiling):
"""The window a session reads at, and the most it may grow to: (now,
most). It starts where the checkpoint left it (`saved`), or at `start`
for a model that has not grown one. The most is the smaller of `block`,
how far the rotary tables were built, and `ceiling` - --context, how far
this card can afford - and it binds on a resumed window too: lowering
--context must bring a grown window down, or a run that no longer fits
the card dies on its first step."""
most = max(64, min(int(block), int(ceiling)))
return max(64, min(int(saved or start), most)), most
def _window_gain(by_pos, ctx, chunk):
"""How much better the model predicts deep into the window than early in
it: the mean loss over the first half of the window's chunk positions
minus the mean over its last quarter, from `by_pos` (chunk position ->
losses). Positive means it is using the distance it has, and would use
more; near zero, the far end of the window is carried for nothing. None
with fewer than 8 losses at either end."""
n = max(2, int(ctx) // int(chunk))
early = [l for p, ls in by_pos.items() if p < n * 0.5 for l in ls]
late = [l for p, ls in by_pos.items() if p >= n * 0.75 for l in ls]
if len(early) < 8 or len(late) < 8:
return None
return float(np.mean(early) - np.mean(late))
def _visit_chunks(ctx, chunk, passage):
"""How many chunks make up one visit to a file.
`passage` is in characters and says how far the reader walks through
one file before moving to another subject. The window slides along it
a chunk at a time, so a visit is passage/chunk steps - and it must be
at least one window, or the window never fills and the model is asked
to read with less context than it has.
This used to be `min_visit_chunks`, in chunks, rounded up to whole
context windows - so the number in the config was not the number in
effect. At 16 with a 32,768 window it silently became 64.
"""
least = max(1, -(-int(ctx) // int(chunk))) # one full window
want = max(1, -(-int(passage) // int(chunk)))
return max(least, want)
def _may_grow(gap, max_gap):
"""Whether held-out lets the pool grow: not once train and held-out have
separated by `max_gap` or more - that is memorising, and more capacity
would make it worse. A gap not yet measured, or `max_gap` 0, holds
nothing back. Everything else growth asks - is capacity idle, are recent
additions earning, is there room - is asked of the network as it is,
inside AutoGrow."""
return gap is None or max_gap <= 0 or gap < max_gap
def _truncate_history(path, chars):
"""Drop history rows past `chars` - they belong to a branch that was
abandoned, and every reader of this file assumes it moves forward."""
if not path or not os.path.exists(path) or chars <= 0:
return
try:
with open(path) as f:
rows = f.read().splitlines()
keep, dropped = [], 0
for line in rows:
try:
if json.loads(line).get("chars", 0) > chars:
dropped += 1
continue
except json.JSONDecodeError:
pass # keep anything unparseable
keep.append(line)
if dropped:
with open(path, "w") as f:
f.write("\n".join(keep) + ("\n" if keep else ""))
print(f" history: dropped {dropped} row(s) past "
f"{chars/1e6:.1f}M - superseded by this resume", flush=True)
except OSError as e:
print(f" (history not trimmed: {e})", flush=True)
def confirm_drop(n, wdir, ask=input):
"""
Whether to go ahead with --drop-quietest N. It deletes experts, files and
all, and nothing but a backup brings them back - and a flag left in a
command copied from the last restart deletes another N. So it is asked at
launch, before anything loads, while whoever started the run is still
there, and the answer is the number typed back: a reflexive "y" does not
count. With no one to ask - no input at all - the answer is no.
"""
have = ""
try:
with open(os.path.join(wdir, "manifest.json")) as f:
have = f" of the {int(json.load(f)['n_experts'])}"
except (OSError, ValueError, KeyError, TypeError):
pass
print(f"\n --drop-quietest {n} deletes the {n} quietest{have} experts in "
f"{wdir}/, files and all. It cannot be undone.", flush=True)
try:
said = ask(f" Type {n} to delete them; anything else keeps them: ")
except EOFError:
said = ""
if said.strip() == str(n):
return True
print(" kept: nothing will be dropped", flush=True)
return False
def cmd_read(args):
"""
Point the model at files and let it read them.
This is the whole interface for learning on your own data. The alphabet is
the 256 byte values, so a file needs no preparation - no tokenizer to fit,
no corpus to build, no step to re-run when the data changes. Give it a
directory and it reads what is there, in order, once or several times, and
is different afterwards.
python3 train.py read ~/notes
python3 train.py read src/ docs/ --passes 3
It is the same path `stream` uses - same chunking, same cache, same
gradient step. What differs is where the characters come from, and that a
file is read from its beginning to its end rather than sampled at random,
because a document has an order and the model should see it.
"""
if args.drop_quietest and not confirm_drop(int(args.drop_quietest),
args.weights_dir):
args.drop_quietest = 0
# EVERY CADENCE IS CONFIGURED IN CHARACTERS AND USED IN STEPS.
#
# A step is `--chunk` characters, so anything counted in steps silently
# changes meaning the moment the chunk changes - `growth.every: 400` was
# 204,800 characters at chunk 512 and would have been 1.6 million at 4,096
# without a line of the config moving. Characters are what the model
# actually reads; steps are an implementation detail of how often we stop
# to update. Convert once, here, where the chunk is known.
def _in_steps(chars, least=1):
return max(least, int(round(int(chars) / max(1, args.chunk))))
grow_every_steps = _in_steps(args.grow_every)
survival_steps = _in_steps(args.prune_survival)
ctx_every_steps = _in_steps(args.context_every)
ctx_grow_every_steps = _in_steps(args.context_grow_every)
from minagi.ingest import collect, summarise
from minagi.stream import FolderEvaluator
from minagi.precision import set_compute_dtype
device = torch.device(args.device)
torch.manual_seed(args.seed)
set_compute_dtype(args.precision)
files = collect(args.paths)
if not files:
print(f"no readable text under {', '.join(args.paths)}", file=sys.stderr)
return 1
info = summarise(files)
print(f"{info['files']} files, {info['characters']/1e6:.2f}M characters"
+ (f", {args.passes} passes" if args.passes > 1 else ""))
wdir = args.weights_dir
# No model here yet is a model that has not been made yet, not an error.
# Everything about its shape is in config.yaml, so there is nothing to ask.
#
# The test is core.npz, NOT whether the directory exists. The repository
# ships weights/manifest.json - it records what the released model is - so
# weights/ is already there on a fresh clone, and testing the directory
# sent every new reader straight into a missing core.npz.
if not os.path.exists(os.path.join(wdir, "core.npz")):
from minagi.create import create
leftovers = ([f for f in os.listdir(wdir) if f != "manifest.json"]
if os.path.isdir(wdir) else [])
if leftovers:
# Something is in there, but not a model. Wiping it is not this
# function's decision to make.
raise SystemExit(
f"{wdir} has no core.npz but is not empty: "
f"{', '.join(sorted(leftovers)[:6])}. That is a partial or "
f"foreign weights directory - move it aside, or point "
f"--weights-dir somewhere else, and it will be created.")
if os.path.isdir(wdir):
print(f"{wdir} holds only a manifest - creating a fresh model "
f"from config.yaml (the manifest describes the released "
f"model, not this one, and is replaced)")
else:
print(f"{wdir} does not exist - creating a fresh model from "
f"config.yaml")
create(wdir, force=True)
print()
model, cfg, pool, man = build_paged(wdir, device, args.resident,
args.ram_capacity, args.context)
# the best held-out this run has seen, for the optional notifier below.
# Reads the manifest, so it cannot move above build_paged.
best_val = float(man.get("val") or float("inf"))
print(f" {pool.n_experts()} experts on disk, {pool.n_resident()} resident, "
f"{pool.vram_params()/1e6:.1f}M in VRAM of "
f"{pool.n_params()/1e6:.1f}M total")
from minagi.precision import describe
print(f" computing in {describe(device)}; weights stay fp32 everywhere, "
f"Adam's moments store as bf16")
# The checkpoint carries whatever depth policy it was trained under, but
# this is a knob about how to spend compute now, not a property of the
# weights - so config.yaml wins over the manifest every time. Applied
# before the first held-out check, which the halting settings change.
cfg.train_steps_mean = float(args.train_steps_mean)
cfg.min_steps = max(1, min(int(args.min_steps), cfg.max_steps))
if args.bptt_window:
cfg.bptt_window = int(args.bptt_window)
if args.ponder_beta:
cfg.ponder_beta = float(args.ponder_beta)
if args.halt_prior:
cfg.halt_prior = float(args.halt_prior)
if args.halt_thresh:
cfg.halt_thresh = float(args.halt_thresh)
cfg.halt_freeze = bool(args.halt_freeze)
# The settings above are counted in ROWS, and the CHECKPOINT decides how
# many rows a step is - n_recur + n_coda. So a config written for one
# architecture is silently wrong on another, and the failure is not always
# loud: bptt_window 16 against max_steps 6 makes the detach condition
# n < -10, which never fires, so NOTHING is detached and the graph holds
# every block instead of the last few. That is how a config meant for a
# 24-row model put a 6-row model out of memory.
warn = []
if cfg.bptt_window > cfg.max_steps:
warn.append(f"bptt_window {cfg.bptt_window} exceeds max_steps "
f"{cfg.max_steps}, so nothing would be detached and the "
f"whole chain would be held. Clamping to {cfg.max_steps}.")
cfg.bptt_window = cfg.max_steps
if cfg.train_steps_mean and cfg.train_steps_mean >= cfg.max_steps:
warn.append(f"train_steps_mean {cfg.train_steps_mean:g} is at or above "
f"max_steps {cfg.max_steps}, so depth sampling saves "
f"nothing - every step runs the ceiling.")
if cfg.halt_prior and 1.0 / cfg.halt_prior > cfg.max_steps * 1.5:
warn.append(f"halt_prior {cfg.halt_prior:g} pulls toward "
f"{1/cfg.halt_prior:.1f} rows but the model only has "
f"{cfg.max_steps}.")
if warn:
print(f" [config] this checkpoint has {cfg.n_recur + cfg.n_coda} "
f"block(s) per row and {cfg.max_steps} rows; config.yaml looks "
f"written for a different shape:", flush=True)
for w in warn:
print(f" - {w}", flush=True)
print(f" gradient reaches back {cfg.bptt_window} of {cfg.max_steps} rows")
print(f" depth: up to {cfg.max_steps} rows, halting prior pulls toward "
f"{1/max(cfg.halt_prior,1e-9):.1f} rows, inference stops at "
f"{cfg.halt_thresh:g} cumulative")
if cfg.train_steps_mean > 0:
import torch as _t
_n = (_t.poisson(_t.full((20000,), cfg.train_steps_mean)).int() + 1
).clamp(cfg.min_steps, cfg.max_steps).float().mean()
print(f" recurrence depth is SAMPLED while training: mean {_n:.2f} of "
f"{cfg.max_steps} ({100*_n/cfg.max_steps:.0f}% of the recurrent "
f"compute). "
+ (f"A character stops when it halts, in training and in "
f"writing alike." if cfg.halt_freeze else
f"Inference still runs all {cfg.max_steps} and lets halting "
f"decide which row each character keeps."))
_temp = float(getattr(pool, "select_temperature", 0.0) or 0.0)
if _temp > 0:
print(f" experts are DRAWN at selection temperature {_temp:g}: each row "
f"draws {cfg.pool_top_k} from the text's probabilities until the "
f"{pool.n_resident()} slots are full, and every character draws "
f"its {cfg.pool_top_k} from its own - the text itself is always "
f"read and written greedily")
else:
print(f" every character ranks all {pool.n_experts()} experts and asks "
f"for its top {cfg.pool_top_k}; each forward uses the "
f"{pool.n_resident()} its text asks for most")
if getattr(pool, "balance", 0) > 0:
print(f" while training, the router pays for the probability it puts "
f"on each expert in proportion to that expert's share of recent "
f"admissions (balance {pool.balance:g}), so the whole pool gets "
f"used")
if getattr(pool, "route_bias", 0) > 0:
if getattr(pool, "route_floor", 0) > 0:
print(f" an expert on trial that takes less than "
f"{pool.route_floor:g} of an even share of choices is lifted "
f"toward the characters, {pool.route_bias:g} a step, until "
f"its own row earns it the share (route_floor)")
else:
print(f" every expert's choices are steered toward its rank's "
f"share, {pool.route_bias:g} a step (route_bias)")
if args.no_pool_checkpoint:
n_off = 0
for _s in model.modules():
if isinstance(_s, PooledMLP):
_s.grad_checkpoint = False
n_off += 1
print(f" pool activations kept, not recomputed ({n_off} call sites) "
f"- faster, and it needs the memory depth was using")
trunk, pool_ps = _split_trunk_pool(model)
# AdamW, or Muon for the weight matrices (training.optimizer) - the one
# factory every learner builds its optimiser with (minagi/muon.py)
from minagi import muon
opt = muon.build(model, trunk, pool_ps, args.lr,
args.lr * args.trunk_lr_mult, args.wd,
kind=args.optimizer, fused=(device.type == "cuda"),
which=args.muon_params)
print(f" {muon.describe(opt)}")
if hasattr(pool, "keep_second_moment"):
# said now, before the start's held-out check reads experts into RAM:
# where Muon steps them, their files' second moments are never read
pool.keep_second_moment(muon.keeps_second(opt, pool.w1))
snr = GradSNR()
print(f" trunk learns at {args.trunk_lr_mult:g}x the pool's rate "
f"({sum(q.numel() for q in trunk)/1e6:.1f}M trunk, "
f"{sum(q.numel() for q in pool_ps)/1e6:.1f}M pool)")
# Scored at the window the model is actually reading at. A fixed
# reference window would be steadier, but it would stop describing the
# model as it grows; and because the window moves one character at a time,
# two consecutive scores are effectively at the same length anyway. The
# sample log states the window with every entry so no comparison is made
# blind.
# held-out is configured in CHARACTERS per domain and read in chunks
eval_steps = steps_of(args.eval_chunks, args.chunk)
sample_eval_steps = steps_of(args.sample_eval_chunks, args.chunk)
ev = (FolderEvaluator(model, args.held_out, args.chunk, cfg.block, device)
if args.held_out and os.path.isdir(args.held_out) else None)
# THE CHECK AND WHAT IT DOES TO THE RATE go through minagi/governor.py,
# the one path chat and the benchmarks' learning take as well. Its
# controller is put in below, once restored.
gov = (Governor(None, ev, args.chunk, every=args.sample_every,
check_chars=args.sample_eval_chunks,
eval_chars=args.eval_chunks)
if ev is not None else None)
checks_kept = ({"every": float(args.sample_every),
"held_out": args.held_out,
"check_chars": int(args.sample_eval_chunks),
"eval_chars": int(args.eval_chunks)}
if gov is not None and args.sample_every
else (man or {}).get("checks"))
before = None
# THE LEVEL CHECKS REPORT AT. Every evaluation here reads whole blocks
# spread over all of each subject. A check reads a few, and a few blocks
# sit a little above or below the whole set by an amount that stays put
# while they do - so the start reads the larger evaluation and the
# checks' own blocks side by side, and every check this session reports
# what it reads plus the difference between the two (minagi/heldout.py).
# The check as it was read before checks were spread, from the top of
# each subject, is read as well, so the log says how far those sat from
# this level.
level, held, parts = {}, None, None
if ev is not None:
v, se, c = gov.start()
before = float(np.mean(list(v.values())))
top = ev.run(sample_eval_steps); top.pop("stderr", None)
level = gov.level
# The first session to count whole blocks only reads the old blocks
# too, once, so the log can move everything before it onto the new
# level (minagi/heldout.py, part_shift).
try:
with open(args.sample_log, errors="ignore") as f:
counted = heldout.part_shift(f.read()) is not None
except (OSError, TypeError):
counted = True # no log: nothing before to move
if not counted and args.sample_every:
old = FolderEvaluator(model, args.held_out, args.chunk, cfg.block,
device, whole=False)
parts = old.run(eval_steps, spread=True)
parts.pop("stderr", None)
print(f" held-out now counts whole blocks only: {before:.4f}; "
f"with each file's leftover part-block, as before, "
f"{np.mean(list(parts.values())):.4f} - the log draws "
f"earlier checks moved by the difference, subject by "
f"subject")
# the best this run has seen was read on the old blocks too
best_val -= float(np.mean(list(parts.values()))) - before
held = (dict(v), c, top, parts)
print(f" before: {before:.4f} +/-{se:.4f} on {args.held_out}, "
f"{-(-eval_steps // ev.per)} blocks of each subject spread "
f"over all of it")
print(f" checks read {-(-sample_eval_steps // ev.per)} blocks of "
f"each: {np.mean(list(c.values())):.4f} here, so they report "
f"{before - np.mean(list(c.values())):+.4f} from what they "
f"read; from the top of each subject, as checks read before, "
f"{np.mean(list(top.values())):.4f}")
# the best this run has seen includes what it has just measured
best_val = min(best_val, before)
pool.attach_optimiser(opt)
# ADAM'S MOMENTS. They are written to optim.npz at every checkpoint and
# were never read back here: this path builds a fresh optimiser and only
# ever SAVED it. So every restart threw away 108.9M moment values, 99.9% of