mini-AGI/config.yaml at main · doublegdub/mini-AGI

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

# mini-AGI — the settings worth changing.

# Read when a model is created and every time it reads. Anything not here is

# either derived or was not worth a knob.

model:

# THIS BLOCK BUILDS A MODEL; IT DOES NOT CHANGE ONE. Down as far as the

# depth rule, these are fixed in the manifest when the weights are created.

# Editing them later does nothing - the shapes would no longer match.

d_model: 512 # width of the trunk

n_head: 8 # attention heads

d_ff: 1408 # the trunk's own feed-forward

n_prelude: 2 # dense blocks before the recurrent core; run once

n_recur: 1 # blocks per row

n_coda: 0 # readout blocks after the last row

# ------------------------------------------------------------------ DEPTH

# The model runs one row at a time and decides after each whether to stop.

# A row is n_recur + n_coda block applications.

# RESCALING RULE: max_steps, halt_prior, bptt_window and train_steps_mean

# are all counted in ROWS, so all four move together if n_recur or n_coda

# changes what a row is. Missing one is silent.

max_steps: 24 # most rows a character may take

min_steps: 1 # rows it must run before halting may stop it

halt_prior: 0.072 # halting is held to a mean of 1/this rows

halt_thresh: 0.9 # at inference, stop at the first row past this

halt_freeze: true # a halted character stops being computed, in

# training and writing alike; later characters read

# its final state at the deeper rows

ponder_beta: 0.01 # weight on the halting KL

bptt_window: 24 # rows gradient reaches back through. Also the

# activation memory, so it sets peak VRAM

train_steps_mean: 12.8 # depth sampled while training: poisson(this) + 1,

# clipped. 0 runs the ceiling every step

# The window attention reaches back over. Reading starts at context_start

# and grows only when the model is still gaining from the far end of what

# it has; context_end is the ceiling and binds at load and at every growth.

context_start: 2048 # where a fresh model begins

context_end: 4096 # and the furthest it may ever reach

context_step: 1 # characters added per growth

context_grow_every_chars: 100_000 # characters between growths

context_gain_min: 0.015 # stop growing below this gain from depth

context_every_chars: 65536 # characters between asking

pool:

# experts, width, depth and top_k are fixed in the manifest, like the model

# block. resident, ram_cache and capacity_factor are run-time and can be

# changed freely.

experts: 64 # how many exist at the start; it grows from here

width: 2048 # hidden units in one expert

depth: 1 # SwiGLU blocks stacked inside one expert

top_k: 8 # how many one character routes through

# HOW EXPERTS ARE CHOSEN - one rule: the text chooses. Every character

# ranks the whole pool with the router and asks for its top_k, and every

# forward's first pass adds its characters' requests to the vote of its

# text - everything read since position 0. A forward - a training window, a

# chunk of held-out file, a prompt, each character of a reply - may use at

# most `resident` different experts: the ones its text has voted for most,

# and a character whose request was not admitted takes its best admitted

# expert instead. A character being written routes among what its prompt

# and the reply so far asked for, as it would inside a training window.

# Three tiers: disk holds every expert, RAM caches the recently wanted,

# VRAM holds what the current forward admitted. Only VRAM is scarce.

resident: 32 # experts one forward may use - the card's slots, each

# holding weights, gradient and Adam's moments

ram_cache: 96 # RAM - experts kept in system memory

capacity_factor: 1.5 # most of a batch one expert may take, as a multiple

# of its fair share; the rest is dropped. 0 removes

# THE BALANCE TERM - how the whole pool gets used. Left alone the router

# keeps choosing the same experts: reading held-out on its own it put 59 of

# 118 on a card across every subject. This term in the loss charges the

# router, in every forward that trains, for the probability it puts on each

# expert over the whole pool, in proportion to that expert's share of recent

# admissions (the last ~1,000 training forwards). Busy experts get dearer and

# idle ones cheaper, so the router's own rows move toward the idle - and

# reading and writing use them too, not only training. A text is still free

# to want few experts; what gets evened out is use across texts.

# The weight sets how hard it leans against learning the text on the experts

# in use. An expert no text admits gets no other gradient, so Adam moves its

# row at the usual pace whatever the weight. 0 turns it off. In a 15-minute

# test 0.01 took the experts in use on held-out from 59 to 114, at a cost of

# 0.023 in held-out loss over those minutes.

balance: 3.5e-4

# THE TEMPERATURE OF EXPERT SELECTION - which experts compute a character,

# NOT which character comes out. Characters are always chosen greedily; any

# randomness in what the model writes comes from here.

# 0 threshold selection, the rule as it has run: the card holds the

# 32 experts the text's first row asks for most, and every character

# takes its own top 8 of those

# > 0 drawn: at each row, 8 experts are drawn from the text's

# probabilities (the router's full distribution, summed over every

# character in the context window) in proportion to p^(1/T), until

# the card's 32 are full, and every character draws its own 8 from

# its own probabilities over the card, the same way

# 1 draws in proportion to the router's probabilities, 0.5 to their

# squares, 0.2 to their fifth powers. Close to 0 the draws approach each

# row's top 8 - which is not quite 0 itself: a drawn card fills 8 a row, so

# the first row has only 8 experts where 0 gives it all 32.

select_temperature: 0

training:

save_every: 5 # minutes between checkpoints

precision: bf16 # what the forward computes in. Weights, gradients

# and the expert files stay fp32

# THE LEARNING RATE IS NOT SCHEDULED. minagi/plasticity.py watches held-out

# and moves the rate in both directions: easing it down without evidence of

# improvement, stepping it back up when held-out jumps and stays there.

chunk: 2048 # characters between steps within the window

lr: 3.0e-4

trunk_lr_mult: 0.1 # the trunk is shared by every domain and is where

# forgetting happens; it learns slower on purpose

weight_decay: 0.1

clip: 1.0

growth:

# Every `every_chars`, add `k` experts if all five brakes are off:

# ROOM there is disk and VRAM for it

# USED the capacity it already has is being asked for

# EARNING the experts added recently are being asked for

# FITS not too many are already inside their trial

# HONEST train and held-out have not separated

every_chars: 2_000_000 # characters between growth decisions

k: 1 # experts added per decision

max_gap: 0.4 # HONEST: train minus held-out. 0 disables

# USED. Staleness is a share of prune.survival_chars: 0.0 when a forward

# admits an expert, 1.0 when it has gone long enough unadmitted to be

# deleted. Past prune.dying_at it counts as dying.

dying_frac_max: 0.35 # stop growing once this share of the pool is dying

# FITS.

max_in_flight: 50 # experts allowed inside their trial at once

# EARNING. Of the experts past their trial and still young, this share must

# have earned a gate.

keep_ratio_min: 0.35

recent_mult: 4.0 # "still young" is age < this x the trial

# ROOM. An expert is 25.2 MB, so 10 GB is about 397 of them.

max_disk_gb: 10.0 # ceiling on the pool on disk. 0 for no limit

mem_frac: 0.95 # refuse to grow past this share of the card

birth_gate: 0.001 # gate a new expert starts at - a scale, not a

# verdict. Small so it cannot shout before it earns

prune:

# USE IT OR LOSE IT. An expert is deleted when nothing has admitted it for

# this long, and it is never deleted inside its first survival_chars.

survival_chars: 100_000_000

dying_at: 0.65 # how far through that window an expert must get,

# unaddressed, to count as dying for growth's brake

data:

train: data/train

held_out: data/val

weights: weights # created on the first run if it is not there

shuffle_seed: 0 # which file and which offset each window comes from

# Each sub-folder of `train` is a subject, and reading rotates between them.

# A window never spans two subjects, but a subject may take several in a row.

passage: 32768 # characters read from one file before moving on

decoding:

# Generation is greedy - no random numbers - so two samples taken at the

# same point in training are the same text. What holds off the loops greedy

# decoding usually falls into is a per-character trace that rises when a

# character is used and decays after.

adapt_strength: 2.5 # logits of suppression for a character just used

adapt_decay: 0.88 # half-life of 5.4 characters; 0 disables it

rep_penalty: 1.0 # the weaker alternative, off at 1.0

plots:

enabled: true # redraw the graphs whenever the sample log grows

last: 100 # millions of characters the recent-progress graph

# covers, counted back from wherever reading has got

FilesExpand file tree

config.yaml

Latest commit

History

config.yaml

File metadata and controls