Repository navigation
Expand file tree
/
Copy pathtoy_cpu.yaml
More file actions
139 lines (127 loc) · 4.16 KB
/
Copy pathtoy_cpu.yaml
File metadata and controls
139 lines (127 loc) · 4.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# miniVERL toy recipe: genuine on-policy distillation on a CPU, no network.
#
# Everything here is local:
# * the tokenizer is miniVERL's ~190-entry reversible toy tokenizer,
# * both models are ~170k-parameter transformers built from this file,
# * the calculator environment generates and verifies its own tasks.
#
# The pipeline is the real one -- student rollouts with tool execution, teacher
# scoring of exactly those states, compressed top-k targets written to a
# provenance-checked cache, and a masked reverse-KL update on assistant tokens
# only. It just runs in seconds instead of hours.
#
# miniverl validate recipes/toy_cpu.yaml
# miniverl train recipes/toy_cpu.yaml
schema_version: 1
run:
name: toy-cpu-opd
mode: opd # sft | offline_kd | opd
# This seed is reported, not chosen silently: at this model size the outcome is
# strongly seed-dependent (81.2% with 1234, 0.0% with 20260727 under an
# otherwise identical 600-step supervised budget). That variance is the reason
# the toy backend is a machinery harness and not a capability claim.
seed: 1234
output_dir: runs
deterministic: true
notes: CPU-only smoke recipe. No downloads, no CUDA, no network.
models:
backend: toy
device: cpu
student:
model_id: toy-student
dtype: float32
quantization: none
lora:
enabled: false # the toy model is trained in full; it is tiny
toy:
hidden_size: 96
num_layers: 3
num_heads: 4
intermediate_size: 192
max_position_embeddings: 1024
teacher:
model_id: toy-teacher
dtype: float32
mode: standard # standard | privileged_context
toy_teacher_seed: 99
# The toy teacher is fitted on oracle traces before it supervises anything.
# A randomly initialized teacher would be a uniform-noise oracle.
# Measured on this environment: 48 oracle traces memorize (train loss 6e-4)
# and generalize at 25%; 256 traces reach 87.5%. Copying operands out of the
# prompt only emerges with task diversity.
toy_pretrain_steps: 800
toy_pretrain_lr: 0.003
toy:
hidden_size: 160
num_layers: 4
num_heads: 4
intermediate_size: 320
max_position_embeddings: 1024
environment:
name: calculator
difficulty: easy
params:
prompt_style: compact # the toy tokenizer is near-character-level
protocol_version: v1 # frozen historical v0.2 prompt
train_tasks: 256
eval_tasks: 24
test_tasks: 24
split_seed: 7
rollout:
max_turns: 3
max_new_tokens_per_turn: 40
max_total_tokens: 512
temperature: 1.0
top_p: 1.0
max_parse_errors: 2
max_repeated_calls: 2
selection:
selector: all_model_tokens # all_model_tokens | tool_and_final | uniform_ratio | hybrid
critical_weight: 1.0
other_weight: 1.0
loss:
mode: bucketed_topk_tail # bucketed_topk_tail | exact_full_vocab
divergence: reverse_kl # forward_kl | reverse_kl | jsd
temperature: 1.0
scale_by_temperature_squared: true
top_k: 16
tail_epsilon: 1.0e-9
chunk_size: 128
sampled_token_nll_weight: 0.0
train:
cycles: 40
rollouts_per_cycle: 8
# One optimizer step per rollout batch keeps every update strictly on-policy.
gradient_accumulation_steps: 8
opd_freshness: strict
learning_rate: 0.003
weight_decay: 0.0
max_grad_norm: 1.0
lr_schedule: cosine
optimizer: adamw
# Measured on this environment: the toy student needs roughly 600 supervised
# steps before it can copy operands out of the prompt. At 300 steps it has
# the tool-call format and none of the arithmetic. See docs/limitations.md.
sft_warmup_cycles: 600 # oracle cold start, then on-policy distillation
save_every_cycles: 0
eval_every_cycles: 20
memory:
strategy: resident # resident | swap | auto
oom_retries: 2
min_chunk_size: 8
cache:
entries_per_shard: 16
strict_policy_version: true
reuse_across_policy_versions: false
dtype: float32
keep_cycles: 2
eval:
enabled: true
split: eval
tasks: null # null -> environment.eval_tasks
temperature: 0.0 # greedy: evaluation is reproducible token for token
seed: 0
report:
enabled: true
max_trajectories: 4
max_tokens_per_trajectory: 240