Repository navigation
Expand file tree
/
Copy pathtest_cli.py
More file actions
876 lines (687 loc) · 30.8 KB
/
Copy pathtest_cli.py
File metadata and controls
876 lines (687 loc) · 30.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
37
10BC8
5
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
"""The command line surface of ``miniverl``.
This file protects the contract a user touches first: every command must be
reachable and self-documenting, ``--json`` must emit a *parsable* document with
the documented keys, the read-only commands must work without torch, the
inspection commands must refuse a path that is not what they claim to read, and
``train --dry-run`` must not create a single file. The end-to-end block runs
the embedded toy demo and then walks the whole ``inspect`` / ``cache`` /
``report`` / ``export-benchmark`` chain over its output, so a break anywhere in
the artifact contract fails here instead of in a user's first session.
"""
from __future__ import annotations
import json
import multiprocessing
import re
import subprocess
import sys
import threading
from pathlib import Path
from typing import Any
import pytest
import yaml
from typer.main import get_command
# `Result` comes from typer, not click. typer 0.27 dropped click as a dependency,
# so importing it from `click.testing` only worked where click happened to be
# installed for some other reason -- which was true locally and false in CI.
from typer.testing import CliRunner, Result
from miniverl import __version__
from miniverl.cli import app
from tests.conftest import HAS_CUDA, requires_torch
REPO_ROOT = Path(__file__).resolve().parents[2]
TOY_RECIPE = REPO_ROOT / "recipes" / "toy_cpu.yaml"
#: Every documented command, spelled exactly as it is typed.
EXPECTED_COMMANDS = {
"profiles list",
"profiles show",
"profiles schema",
"compat explain",
"compat check",
"hardware record",
"hardware validate",
"qualification validate",
"data sample",
"align",
"alignment-suite prepare",
"alignment-suite validate",
"alignment-suite report",
"doctor",
"evidence show",
"evidence validate",
"validate",
"demo",
"train",
"eval",
"benchmark",
"import-verl",
"convert-dataset",
"export-verl",
"prepare-offline-kd",
"pilot",
"plan",
"qualify-teacher",
"inspect",
"report",
"run",
"export-adapter",
"export-benchmark",
"schema",
"bridge doctor",
"bridge compile-opd",
"bridge materialize",
"cache stats",
"cache validate",
}
#: What a finished run must contain, relative to the run directory.
DEMO_ARTIFACTS = (
"config.original.yaml",
"config.validated.yaml",
"config.resolved.yaml",
"manifest.json",
"environment.json",
"metrics.jsonl",
"events.jsonl",
"trajectories.jsonl",
"teacher-cache/index.json",
"eval.json",
"report.html",
)
_STATUSES = {"ok", "warn", "missing", "fail"}
def _invoke(*args: str) -> Result:
"""Run the CLI in-process with a fresh runner."""
return CliRunner().invoke(app, list(args))
def test_sample_parquet_rows_are_distinct_when_requested(tmp_path: Path) -> None:
pyarrow = pytest.importorskip("pyarrow.parquet")
target = tmp_path / "sample.parquet"
result = _invoke("data", "sample", "--out", str(target), "--rows", "64")
assert result.exit_code == 0, result.output
prompts = [str(row["prompt"]) for row in pyarrow.read_table(target).to_pylist()]
assert len(prompts) == 64
assert len(set(prompts)) == 64
def test_bridge_compile_opd_is_an_offline_deterministic_smoke(tmp_path: Path) -> None:
source = REPO_ROOT / "examples" / "verl-opd-v0.8-single-gpu.yaml"
output = tmp_path / "compiled-plan.json"
result = _invoke(
"bridge",
"compile-opd",
"--config",
str(source),
"--set",
"distillation.distillation_loss.topk=64",
"--out",
str(output),
"--json",
)
assert result.exit_code == 0, result.output
payload = json.loads(result.stdout)
assert payload["profile"] == "verl-opd-v0.8-single-gpu-v1"
assert payload["executable"] is True
assert payload["source"]["distillation"]["distillation_loss"]["topk"] == 64
assert json.loads(output.read_text(encoding="utf-8")) == payload
def _probe_run_lock(output_root: str, run_id: str, result_queue) -> None: # type: ignore[no-untyped-def]
"""Spawn-safe probe used to prove an automatic report retains ownership."""
from miniverl.errors import RunLockedError
from miniverl.utils.locking import RunLock
try:
with RunLock(Path(output_root), run_id):
result_queue.put("acquired")
except RunLockedError:
result_queue.put("locked")
def _payload(result: Result) -> Any:
"""Assert the command succeeded and parse its ``--json`` document.
``stdout`` only: log records go to stderr, so mixing the two would make the
JSON unparsable and hide a real regression behind a decoding error.
"""
assert result.exit_code == 0, result.output
return json.loads(result.stdout)
def _collapse(text: str) -> str:
"""Undo rich's console wrapping so a wrapped message can be searched."""
return re.sub(r"\s+", " ", text)
def _walk_commands(*, groups: bool) -> list[tuple[str, ...]]:
"""Enumerate the app's commands as argument tuples.
``groups=True`` also yields the sub-command groups themselves (``cache``),
which have their own help page. The root is never yielded.
"""
paths: list[tuple[str, ...]] = []
def walk(prefix: tuple[str, ...], command: Any) -> None:
children = getattr(command, "commands", None)
if not children:
paths.append(prefix)
return
if groups and prefix:
paths.append(prefix)
for name, child in sorted(children.items()):
walk((*prefix, name), child)
walk((), get_command(app))
return paths
def _top_level_names() -> list[str]:
"""Names listed under ``Commands:`` in the root help."""
return sorted(get_command(app).commands)
def _write_recipe(path: Path, mutate: dict[str, Any]) -> Path:
"""Copy the toy recipe with ``mutate`` applied to its ``train`` block."""
data = yaml.safe_load(TOY_RECIPE.read_text(encoding="utf-8"))
data["train"].update(mutate)
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
return path
def _write_alignment_recipe(path: Path, *, with_pilot: bool = False) -> Path:
data = yaml.safe_load(TOY_RECIPE.read_text(encoding="utf-8"))
data["run"].update({"name": "alignment-dry-run", "mode": "opd"})
data["models"]["teacher"]["mode"] = "privileged_context"
data["environment"].update({"name": "tool_policy", "difficulty": "easy"})
data["environment"]["params"] = {"prompt_style": "compact", "protocol_version": "v2"}
data["alignment"] = {
"method": "standard_opd",
"teacher_mode": "policy_conditioned",
"policy": {"id": "minipolicy", "revision": "v1", "sha256": "a" * 64},
}
if with_pilot:
data["alignment"]["pilot"] = {
"sample_size": 48,
"teacher_policy_competence": 0.9,
"fresh_state_gap": 0.08,
"hard_soft_gap": 0.08,
"policy_sensitive_token_fraction": 0.8,
}
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
return path
COMMAND_PATHS = _walk_commands(groups=False)
HELP_PATHS = _walk_commands(groups=True)
# ------------------------------------------------------------------ discovery
def test_version_flag_prints_name_and_version() -> None:
result = _invoke("--version")
assert result.exit_code == 0
assert result.stdout.strip() == f"miniverl {__version__}"
def test_short_version_flag_matches_long_flag() -> None:
assert _invoke("-V").stdout == _invoke("--version").stdout
def test_root_help_lists_every_command() -> None:
result = _invoke("--help")
assert result.exit_code == 0
for name in _top_level_names():
assert name in result.stdout, f"{name} missing from --help"
def test_root_help_describes_the_current_product_without_promising_v08() -> None:
result = _invoke("--help")
assert result.exit_code == 0
collapsed = _collapse(result.stdout)
assert "documented subset of verl-style OPD on one consumer GPU" in collapsed
assert "teacher scoring and actor update locally" in collapsed
assert "inspectable PEFT artifacts" in collapsed
assert "distributed verl execution" not in collapsed
def test_command_set_matches_the_documented_set() -> None:
assert {" ".join(path) for path in COMMAND_PATHS} == EXPECTED_COMMANDS
@pytest.mark.parametrize("path", HELP_PATHS, ids=[" ".join(path) for path in HELP_PATHS])
def test_every_command_has_help(path: tuple[str, ...]) -> None:
result = _invoke(*path, "--help")
assert result.exit_code == 0, result.output
assert "Usage:" in result.stdout
assert result.stdout.strip()
def test_no_arguments_shows_help_and_exits_nonzero() -> None:
result = _invoke()
assert result.exit_code != 0
assert "Usage:" in result.output
# --------------------------------------------------------------------- doctor
def test_doctor_json_reports_checks_capabilities_and_verdict(tmp_path: Path) -> None:
target = tmp_path / "probe-runs"
payload = _payload(_invoke("doctor", "--json", "--output", str(target)))
assert set(payload) >= {"miniverl_version", "checks", "capabilities", "verdict"}
assert payload["miniverl_version"] == __version__
assert payload["checks"], "doctor reported no checks at all"
for check in payload["checks"]:
assert set(check) == {"name", "status", "detail", "hint"}
assert check["status"] in _STATUSES
names = [check["name"] for check in payload["checks"]]
assert any(name.startswith("dependency:") for name in names)
assert "output directory" in names
verdict = payload["verdict"]
assert set(verdict) == {"core_commands", "cpu_training", "gpu_training", "qlora_4bit"}
assert all(isinstance(value, bool) for value in verdict.values())
# The test suite itself is running, so the base dependencies are importable.
assert verdict["core_commands"] is True
assert verdict["gpu_training"] is HAS_CUDA
capabilities = payload["capabilities"]
assert capabilities["environments"], "no environments registered"
assert capabilities["cuda_available"] is HAS_CUDA
def test_doctor_probes_the_requested_directory_and_cleans_up(tmp_path: Path) -> None:
target = tmp_path / "nested" / "runs"
payload = _payload(_invoke("doctor", "--json", "--output", str(target)))
status = next(c for c in payload["checks"] if c["name"] == "output directory")
assert status["status"] == "ok"
assert target.is_dir()
# The writability probe file must not survive the check.
assert list(target.iterdir()) == []
def test_doctor_renders_a_table_without_json(tmp_path: Path) -> None:
result = _invoke("doctor", "--output", str(tmp_path / "probe-runs"))
assert result.exit_code == 0
collapsed = _collapse(result.stdout)
assert "core commands" in collapsed
assert "4-bit QLoRA" in collapsed
# ------------------------------------------------------------------- validate
def test_validate_toy_recipe_json() -> None:
payload = _payload(_invoke("validate", str(TOY_RECIPE), "--json"))
assert payload["valid"] is True
assert payload["mode"] == "opd"
assert payload["is_on_policy"] is True
assert payload["backend"] == "toy"
assert payload["environment"] == "calculator"
assert payload["eval_tasks"] > 0
# One optimizer step per rollout batch is what makes the toy recipe strictly
# on-policy; the planned step count must follow from it.
assert payload["optimizer_steps_per_cycle"] == 1
assert payload["planned_optimizer_steps"] == payload["optimizer_steps_per_cycle"] * (
payload["cycles"] + payload["sft_warmup_cycles"]
)
assert payload["warnings"] == []
def test_validate_renders_a_table_without_json() -> None:
result = _invoke("validate", str(TOY_RECIPE))
assert result.exit_code == 0
collapsed = _collapse(result.stdout)
assert "valid" in collapsed
assert "toy-cpu-opd" in collapsed
def test_validate_missing_recipe_exits_nonzero(tmp_path: Path) -> None:
missing = tmp_path / "nope.yaml"
result = _invoke("validate", str(missing))
assert result.exit_code != 0
assert "not found" in _collapse(result.output)
def test_validate_out_of_range_field_exits_nonzero_and_names_it(tmp_path: Path) -> None:
recipe = _write_recipe(tmp_path / "bad-range.yaml", {"cycles": -1})
result = _invoke("validate", str(recipe))
assert result.exit_code != 0
assert "train.cycles" in _collapse(result.output)
def test_validate_unknown_field_exits_nonzero_and_names_it(tmp_path: Path) -> None:
recipe = _write_recipe(tmp_path / "bad-field.yaml", {"nonexistent_knob": 3})
result = _invoke("validate", str(recipe))
assert result.exit_code != 0
assert "nonexistent_knob" in _collapse(result.output)
def test_validate_json_reports_the_failing_location(tmp_path: Path) -> None:
recipe = _write_recipe(tmp_path / "bad-range.yaml", {"cycles": -1})
result = _invoke("validate", str(recipe), "--json")
assert result.exit_code != 0
payload = json.loads(result.stdout)
assert payload["valid"] is False
assert ["train", "cycles"] in [error["loc"] for error in payload["errors"]]
# ---------------------------------------------------------------------- train
def test_train_dry_run_plans_without_writing_anything(tmp_path: Path) -> None:
target = tmp_path / "run-output"
default_output = Path.cwd() / "runs"
default_existed = default_output.exists()
payload = _payload(
_invoke("train", str(TOY_RECIPE), "--dry-run", "--json", "--output", str(target))
)
assert payload["dry_run"] is True
assert payload["downloads_required"] is False
assert payload["mode"] == "opd"
assert payload["backend"] == "toy"
assert payload["output_dir"] == str(target)
assert payload["planned_optimizer_steps"] > 0
assert payload["planned_rollouts"] > 0
assert payload["resolved_config"]["models"]["backend"] == "toy"
assert not target.exists()
assert list(tmp_path.iterdir()) == []
assert default_output.exists() is default_existed
def test_train_dry_run_rejects_a_missing_recipe(tmp_path: Path) -> None:
result = _invoke("train", str(tmp_path / "absent.yaml"), "--dry-run")
assert result.exit_code != 0
assert "not found" in _collapse(result.output)
def test_align_dry_run_exposes_the_six_stage_workflow_without_writes(tmp_path: Path) -> None:
recipe = _write_alignment_recipe(tmp_path / "alignment.yaml")
target = tmp_path / "runs"
payload = _payload(
_invoke("align", str(recipe), "--dry-run", "--json", "--output", str(target))
)
assert payload["dry_run"] is True
assert payload["method"] == "standard_opd"
assert [stage["name"] for stage in payload["workflow"]["stages"]] == [
"base_model",
"sft_checkpoint",
"teacher_reference_construction",
"alignment",
"evaluation",
"alignment_card",
]
assert not target.exists()
def test_pilot_is_bounded_explainable_and_does_not_load_models(tmp_path: Path) -> None:
recipe = _write_alignment_recipe(tmp_path / "pilot.yaml", with_pilot=True)
payload = _payload(_invoke("pilot", str(recipe), "--json"))
assert payload["rules_version"] == "alignment-pilot-v1"
assert payload["recommendation"] == "standard_opd"
assert payload["reasons"]
def test_pilot_consumes_the_external_early_stop_without_recommending_a_method() -> None:
result = REPO_ROOT / "benchmarks/results/alignment-external-v1.json"
payload = _payload(_invoke("pilot", "--study-result", str(result), "--json"))
assert payload["study_status"] == "terminated_at_checkpoint_selection"
assert payload["recommendation"] == "do_not_continue"
assert payload["recommendation_scope"] == "do_not_continue_this_study"
assert payload["method_recommendation"] == "insufficient_evidence"
assert payload["universal_claim"] is False
assert payload["evidence"]["sha256"]
assert not any(
method in json.dumps(payload).lower() for method in ("continued sft", "dpo", "kd", "opd")
)
def test_pilot_consumes_the_packaged_external_study_without_a_checkout() -> None:
payload = _payload(_invoke("pilot", "--builtin-study", "alignment-external-v1", "--json"))
assert payload["study_status"] == "terminated_at_checkpoint_selection"
assert payload["method_recommendation"] == "insufficient_evidence"
assert payload["evidence"]["builtin_study"] == "alignment-external-v1"
assert payload["evidence"]["sha256"]
def test_evidence_show_and_validate_use_packaged_bytes() -> None:
shown = _payload(_invoke("evidence", "show", "alignment-external-v1", "--json"))
validated = _payload(_invoke("evidence", "validate", "alignment-external-v1", "--json"))
assert shown["study_id"] == "alignment-external-v1"
assert shown["result"]["study_status"] == "terminated_at_checkpoint_selection"
assert validated["valid"] is True
assert validated["study_id"] == "alignment-external-v1"
assert validated["task_rows"] == 512
assert validated["problems"] == []
def test_pilot_rejects_an_inconsistent_external_result(tmp_path: Path) -> None:
source = json.loads(
(REPO_ROOT / "benchmarks/results/alignment-external-v1.json").read_text(encoding="utf-8")
)
source["selected_checkpoint"] = "update-004"
broken = tmp_path / "broken-result.json"
broken.write_text(json.dumps(source), encoding="utf-8")
result = _invoke("pilot", "--study-result", str(broken), "--json")
assert result.exit_code != 0
assert "selected_checkpoint" in result.output
@pytest.mark.parametrize(
"options",
[
("--resume", "run", "--resume-from", "run/checkpoints/final"),
("--resume", "run", "--overwrite"),
("--resume-from", "run/checkpoints/final", "--run-id", "other"),
],
)
def test_train_rejects_mutually_exclusive_run_lifecycle_options(
options: tuple[str, ...],
) -> None:
result = _invoke("train", str(TOY_RECIPE), "--dry-run", *options)
assert result.exit_code != 0
assert "cannot be combined" in _collapse(result.output) or "mutually exclusive" in _collapse(
result.output
)
# --------------------------------------------------------------------- schema
def test_schema_emits_a_json_schema() -> None:
payload = _payload(_invoke("schema"))
assert payload["$schema"].startswith("https://json-schema.org/")
assert payload["$id"].endswith("benchmark-result.schema.json")
assert {"miniverl_version", "name", "created_at", "arms"} <= set(payload["properties"])
assert "miniverl_version" in payload["required"]
def test_schema_out_writes_the_same_document(tmp_path: Path) -> None:
target = tmp_path / "schema.json"
result = _invoke("schema", "--out", str(target))
assert result.exit_code == 0
assert json.loads(target.read_text(encoding="utf-8")) == _payload(_invoke("schema"))
# ------------------------------------------------------------- failure paths
def test_inspect_missing_file_exits_nonzero(tmp_path: Path) -> None:
result = _invoke("inspect", str(tmp_path / "trajectories.jsonl"))
assert result.exit_code != 0
assert "not found" in _collapse(result.output)
def test_cache_stats_without_index_exits_nonzero(tmp_path: Path) -> None:
empty = tmp_path / "teacher-cache"
empty.mkdir()
result = _invoke("cache", "stats", str(empty))
assert result.exit_code != 0
assert "index.json" in _collapse(result.output)
def test_cache_validate_without_index_exits_nonzero(tmp_path: Path) -> None:
empty = tmp_path / "teacher-cache"
empty.mkdir()
result = _invoke("cache", "validate", str(empty), "--json")
assert result.exit_code != 0
payload = json.loads(result.stdout)
assert payload["valid"] is False
assert payload["problems"]
def test_report_on_a_non_run_directory_exits_nonzero(tmp_path: Path) -> None:
stray = tmp_path / "not-a-run"
stray.mkdir()
(stray / "notes.txt").write_text("hello", encoding="utf-8")
result = _invoke("report", str(stray))
assert result.exit_code != 0
assert "manifest.json" in _collapse(result.output)
def test_report_on_a_missing_directory_exits_nonzero(tmp_path: Path) -> None:
result = _invoke("report", str(tmp_path / "absent"))
assert result.exit_code != 0
assert "not found" in _collapse(result.output)
# ------------------------------------------------- behaviour on a bare install
#: Runs ``miniverl demo`` in a child interpreter where ``import torch`` fails the
#: way it fails on ``pip install miniverl`` without the ``[train]`` extra.
_NO_TORCH_PROBE = """
import sys
from importlib.abc import MetaPathFinder
class _Blocker(MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname.split(".")[0] == "torch":
raise ModuleNotFoundError("No module named 'torch'", name=fullname)
return None
sys.meta_path.insert(0, _Blocker())
from typer.testing import CliRunner
from miniverl.cli import app
result = CliRunner().invoke(app, ["demo", "--output", sys.argv[1], "--fast"])
print("EXIT", result.exit_code)
print("EXCEPTION", type(result.exception).__name__ if result.exception else "none")
print("OUTPUT", " | ".join(result.output.splitlines()))
"""
def test_demo_without_torch_names_the_missing_extra(tmp_path: Path) -> None:
completed = subprocess.run(
[sys.executable, "-c", _NO_TORCH_PROBE, str(tmp_path / "run")],
capture_output=True,
text=True,
timeout=300,
check=False,
cwd=REPO_ROOT,
)
report = completed.stdout
assert "EXIT 1" in report, report
# typer.Exit surfaces as SystemExit, which is the intended clean exit path.
assert "EXCEPTION ModuleNotFoundError" not in report, (
"the missing extra escaped as a raw ModuleNotFoundError"
)
assert "EXCEPTION ImportError" not in report, report
assert "miniverl[train]" in report, "the error must name the extra to install"
# ------------------------------------------------------- end-to-end toy flow
@pytest.fixture(scope="module")
def demo_run(tmp_path_factory: pytest.TempPathFactory) -> Path:
"""Run ``miniverl demo --fast`` once and hand back its run directory."""
pytest.importorskip("torch")
root = tmp_path_factory.mktemp("demo-flow") / "run"
result = _invoke("demo", "--output", str(root), "--fast", "--json")
assert result.exit_code == 0, result.output
payload = json.loads(result.stdout)
assert payload["run_dir"] == str(root)
assert payload["mode"] == "opd"
assert payload["global_step"] > 0
return root
@requires_torch
@pytest.mark.torch
def test_demo_writes_every_documented_artifact(demo_run: Path) -> None:
for relative in DEMO_ARTIFACTS:
assert (demo_run / relative).is_file(), f"{relative} missing from {demo_run}"
checkpoints = demo_run / "checkpoints"
assert checkpoints.is_dir()
assert any(checkpoints.iterdir()), "no checkpoint was written"
@requires_torch
@pytest.mark.torch
def test_automatic_report_remains_under_the_training_run_lock(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import miniverl.reporting as reporting
target = tmp_path / "automatic-report-lock"
report_entered = threading.Event()
release_report = threading.Event()
real_write_report = reporting.write_report
def blocked_write_report(*args, **kwargs): # type: ignore[no-untyped-def]
report_entered.set()
assert release_report.wait(timeout=30)
return real_write_report(*args, **kwargs)
monkeypatch.setattr(reporting, "write_report", blocked_write_report)
result_holder: list[Result] = []
cli_thread = threading.Thread(
target=lambda: result_holder.append(
_invoke("demo", "--output", str(target), "--fast", "--json")
)
)
cli_thread.start()
assert report_entered.wait(timeout=30)
context = multiprocessing.get_context("spawn")
result_queue = context.Queue()
probe = context.Process(
target=_probe_run_lock,
args=(str(target.parent), target.name, result_queue),
)
probe.start()
probe.join(timeout=30)
assert not probe.is_alive()
assert result_queue.get(timeout=5) == "locked"
release_report.set()
cli_thread.join(timeout=60)
assert not cli_thread.is_alive()
assert result_holder and result_holder[0].exit_code == 0, (
result_holder[0].output if result_holder else "CLI returned no result"
)
@requires_torch
@pytest.mark.torch
def test_automatic_report_failure_preserves_completed_manifest_and_releases_lock(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import miniverl.reporting as reporting
from miniverl.utils.locking import RunLock
target = tmp_path / "report-write-failure"
monkeypatch.setattr(
reporting,
"write_report",
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("injected report write failure")),
)
result = _invoke("demo", "--output", str(target), "--fast", "--json")
assert result.exit_code != 0
assert isinstance(result.exception, OSError)
manifest = json.loads((target / "manifest.json").read_text(encoding="utf-8"))
assert manifest["status"] == "completed"
with RunLock(target.parent, target.name):
pass
@requires_torch
@pytest.mark.torch
def test_inspect_json_accounts_for_every_token(demo_run: Path) -> None:
payload = _payload(_invoke("inspect", str(demo_run / "trajectories.jsonl"), "--json"))
assert payload["trajectories"] > 0
fraction = payload["model_token_fraction"]
assert 0.0 < fraction < 1.0, "a tool-using rollout is neither all context nor all model"
provenance = payload["provenance_check"]
excluded = provenance["context_span_types_excluded_from_loss"]
trainable = set(provenance["trainable_span_types"])
assert "tool_result" in excluded, "tool output must never be trainable"
assert trainable.isdisjoint(excluded)
by_span = payload["tokens_by_span_type"]
assert sum(by_span.values()) == payload["tokens"]
assert payload["tokens"] == payload["model_tokens"] + payload["context_tokens"]
assert payload["model_tokens"] == sum(
count for name, count in by_span.items() if name in trainable
)
assert payload["model_tokens"] / payload["tokens"] == pytest.approx(fraction)
@requires_torch
@pytest.mark.torch
def test_inspect_unknown_trajectory_id_exits_nonzero(demo_run: Path) -> None:
result = _invoke("inspect", str(demo_run / "trajectories.jsonl"), "--trajectory", "no-such-id")
assert result.exit_code != 0
assert "no trajectories matched" in _collapse(result.output)
@requires_torch
@pytest.mark.torch
def test_inspect_renders_spans_without_json(demo_run: Path) -> None:
trajectories = demo_run / "trajectories.jsonl"
first = json.loads(trajectories.read_text(encoding="utf-8").splitlines()[0])
result = _invoke(
"inspect", str(trajectories), "--trajectory", first["trajectory_id"], "--spans"
)
assert result.exit_code == 0, result.output
assert "span" in _collapse(result.stdout)
@requires_torch
@pytest.mark.torch
def test_cache_stats_json_shows_real_compression(demo_run: Path) -> None:
payload = _payload(_invoke("cache", "stats", str(demo_run / "teacher-cache"), "--json"))
assert payload["compression_ratio"] > 0.0
assert payload["problems"] == []
assert payload["trajectories"] > 0
assert payload["selected_positions"] > 0
assert payload["checksums_verified"] is True
assert payload["actual_bytes"] < payload["theoretical_full_logit_bytes"]
assert payload["shards"], "a written cache must have at least one shard"
@requires_torch
@pytest.mark.torch
def test_cache_validate_json_accepts_the_written_cache(demo_run: Path) -> None:
payload = _payload(_invoke("cache", "validate", str(demo_run / "teacher-cache"), "--json"))
assert payload["valid"] is True
assert payload["problems"] == []
assert payload["entries"] > 0
assert payload["shards"] > 0
@requires_torch
@pytest.mark.torch
def test_report_html_is_self_contained(demo_run: Path) -> None:
payload = _payload(_invoke("report", str(demo_run), "--json"))
html_path = Path(payload["html"])
assert html_path.is_file()
assert Path(payload["markdown"]).is_file()
assert payload["json"] is None
assert payload["summary"]["run_id"] == demo_run.name
html = html_path.read_text(encoding="utf-8")
assert "<html" in html.lower()
assert "<style" in html, "the stylesheet must be inlined, not linked"
assert not re.search(r"https?://", html), "the report must not reference the network"
assert 'src="//' not in html, "protocol-relative reference"
for attribute, value in re.findall(r'\b(src|href)="([^"]*)"', html):
assert value.startswith(("#", "data:")), f"external {attribute}: {value}"
@requires_torch
@pytest.mark.torch
def test_export_benchmark_matches_the_published_schema(demo_run: Path) -> None:
from miniverl.evaluation.schema import BenchmarkResult
payload = _payload(_invoke("export-benchmark", str(demo_run), "--json"))
written = Path(payload["written"])
assert written.is_file()
assert json.loads(written.read_text(encoding="utf-8")) == payload["result"]
result = BenchmarkResult.model_validate(payload["result"])
assert result.miniverl_version == __version__
assert len(result.arms) == 1
arm = result.arms[0]
assert arm.run_dir == demo_run.name, "run_dir must be sanitized to a bare name"
assert 0.0 <= arm.success_rate <= 1.0
assert arm.tasks > 0
assert arm.selected_training_tokens > 0
serialized = json.dumps(payload["result"])
assert str(demo_run) not in serialized, "the submission leaks a local path"
assert demo_run.as_posix() not in serialized
@requires_torch
@pytest.mark.torch
@pytest.mark.parametrize("command", ["report", "export-benchmark"])
def test_shareable_writers_refuse_a_run_owned_by_training_without_mutation(
demo_run: Path, command: str
) -> None:
from miniverl.utils.locking import RunLock
before = {
path.relative_to(demo_run).as_posix(): path.read_bytes()
for path in demo_run.rglob("*")
if path.is_file()
}
with RunLock(demo_run.parent, demo_run.name):
result = _invoke(command, str(demo_run))
assert result.exit_code != 0
assert "locked" in _collapse(result.output).lower()
after = {
path.relative_to(demo_run).as_posix(): path.read_bytes()
for path in demo_run.rglob("*")
if path.is_file()
}
assert after == before
@requires_torch
@pytest.mark.torch
def test_demo_rerun_refuses_without_modifying_any_existing_artifact(demo_run: Path) -> None:
before = {
path.relative_to(demo_run).as_posix(): path.read_bytes()
for path in demo_run.rglob("*")
if path.is_file()
}
result = _invoke("demo", "--output", str(demo_run), "--fast", "--json")
assert result.exit_code != 0
assert "--overwrite" in _collapse(result.output)
assert "--resume" in _collapse(result.output)
after = {
path.relative_to(demo_run).as_posix(): path.read_bytes()
for path in demo_run.rglob("*")
if path.is_file()
}
assert after == before