Repository navigation
Expand file tree
/
Copy pathcli.py
More file actions
340 lines (306 loc) · 14.2 KB
/
Copy pathcli.py
File metadata and controls
340 lines (306 loc) · 14.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
"""Command-line entry points for the distribution workflows.
Kept thin on purpose: every non-trivial decision lives in a tested module.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from minibwa_dist import rerere_cache
from minibwa_dist.assemble import (
AssemblyError,
TrainingStop,
assemble,
previously_merged,
regressions,
set_version,
stamp_dev_version,
upstream_version,
)
from minibwa_dist.gates import run_gates
from minibwa_dist.gitutil import GitError
from minibwa_dist.manifest import ManifestError, load_manifest, parse_manifest
from minibwa_dist.reconcile import apply_changes, graduated, nomination_candidates, reconcile
from minibwa_dist.render import render_graveyard, render_release_notes, update_readme
from minibwa_dist.rerere_cache import OrphanOverPublishedCache
def _sync(args: argparse.Namespace) -> int:
repo = Path(args.repo)
manifest = load_manifest(Path(args.manifest))
restored = rerere_cache.restore(repo)
print(f"rerere: restored {restored} resolution(s)", file=sys.stderr)
try:
result = assemble(
repo, manifest, args.base, args.out, remote=args.remote, stop_at=args.stop_at
)
except TrainingStop:
# `--stop-at` left a conflict in the worktree on purpose (see
# assemble()'s docstring). That is the expected, successful outcome of
# training mode, not a build failure, so it must not print "FAILED" or
# exit nonzero. A genuine required-feature conflict can still fire while
# `--stop-at` is set -- it raises the base class, lands in the handler
# below, and still exits 1.
#
# The resolution is only worth recording if it reaches the fork: `save()`
# ran in the `finally` below at the moment of the conflict, so it has the
# pre-image and nothing else. Spell out the whole follow-up, naming the
# remote the caller already chose -- `origin` is upstream in a local
# checkout, and a resolution pushed there goes to the wrong repository.
print(
f"stopped at '{args.stop_at}' for rerere training -- resolve, commit, "
"then `git reset --hard HEAD~1`. Re-run this same sync to record the "
f"resolution on the local `{rerere_cache.CACHE_BRANCH}` branch, then "
f"`git push {args.remote} {rerere_cache.CACHE_BRANCH}` to publish it.",
file=sys.stderr,
)
return 0
except AssemblyError as exc:
print(f"ASSEMBLY FAILED: {exc}", file=sys.stderr)
return 1
finally:
# Save even on failure: a partial run may have recorded new pre-images,
# and throwing them away means paying for the same conflicts next time.
rerere_cache.save(repo)
version = stamp_dev_version(repo, args.out) if args.stamp else None
regressed = (
regressions(previously_merged(repo, args.regression_baseline), manifest, result.merged)
if args.regression_baseline
else ()
)
# Emit the JSON even when regressed: the caller redirects this to a file and
# reads it after the step fails, and a run that fails silently is worse than
# the drop it is reporting.
print(
json.dumps(
{
"base": result.base,
"head": result.head,
"version": version,
"merged": list(result.merged),
"dropped": [
{"name": d.name, "files": list(d.conflicting_files)} for d in result.dropped
],
"regressed": list(regressed),
}
)
)
if regressed:
print(
f"REGRESSION: {', '.join(regressed)} merged in the build at "
f"'{args.regression_baseline}' and no longer merge. Resolve the conflict with "
f"`--stop-at <name>` and publish the resolution to `{rerere_cache.CACHE_BRANCH}`, "
"or retire the feature from the manifest if the drop is intended.",
file=sys.stderr,
)
return 1
return 0
def _gates(args: argparse.Namespace) -> int:
"""Run the output gates and turn their verdicts into an exit code.
A subcommand rather than two hand-copied heredocs in two workflows: the
aggregation (`all(...)`) is the whole point of the step, and neither copy
was reachable by `ruff check minibwa_dist/` or `pytest minibwa_dist/tests`.
`--repo` is where each merged feature's declared test suites are run. It has
no default -- it is not always the working directory, and guessing wrong
points the suites at a tree that does not carry them -- but it is optional
rather than required, and its absence is reported instead of raising.
That last part is not politeness. The engine and the workflow file update on
different schedules: every sync checks out the freshly assembled `dist-next`
and runs THAT `minibwa_dist`, while the YAML executing comes from `dist` and
changes only when a build is shipped. A newly required argument would be
demanded by the engine a full sync before the workflow could supply it, and
the gates step would die on argparse before the ship that would fix it.
"""
manifest = load_manifest(Path(args.manifest))
merged: tuple[str, ...] | None = None
if args.assembly:
merged = tuple(json.loads(Path(args.assembly).read_text())["merged"])
results = run_gates(
Path(args.candidate),
Path(args.stock),
Path(args.fixtures),
manifest,
Path(args.workdir),
merged=merged,
repo=Path(args.repo) if args.repo else None,
)
for result in results:
print(f"{'PASS' if result.passed else 'FAIL'} {result.name}: {result.detail}")
unrun = [f.name for f in manifest.features if f.tests and (merged is None or f.name in merged)]
if args.repo is None and unrun:
# Not a gate result: it did not run, so it neither passed nor failed.
# Said out loud all the same -- unrun coverage that looks like absent
# coverage is how a gate quietly stops meaning anything.
print(f"NOTE feature suites not run (--repo not given): {', '.join(unrun)}")
return 0 if all(result.passed for result in results) else 1
def _stamp(args: argparse.Namespace) -> int:
"""Set MB_VERSION to an exact version string, for the release workflow."""
previous = set_version(Path(args.repo), args.version)
print(f"MB_VERSION: {previous} -> {args.version}", file=sys.stderr)
return 0
def _reconcile(args: argparse.Namespace) -> int:
manifest_path = Path(args.manifest)
manifest = load_manifest(manifest_path)
pr_states = json.loads(Path(args.pr_states).read_text())
changes = reconcile(manifest, {int(k): v for k, v in pr_states.items()})
if changes and args.write:
new_text = apply_changes(manifest_path.read_text(), changes)
# Refuse to write a manifest this same engine cannot load back: CI's
# own load of the committed file is not the first opportunity to
# notice a malformed rewrite, this is.
parse_manifest(new_text)
manifest_path.write_text(new_text)
print(
json.dumps(
{
"changes": [{"feature": c.feature, "old": c.old, "new": c.new} for c in changes],
"graduated": graduated(manifest, changes),
"nominations": nomination_candidates(manifest),
}
)
)
return 0
def _notes(args: argparse.Namespace) -> int:
manifest = load_manifest(Path(args.manifest))
dropped = tuple(args.dropped) if args.dropped else ()
print(
render_release_notes(
manifest,
Path(args.upstream_notes).read_text(),
args.upstream_tag,
dropped,
upstream_ref=args.upstream_ref,
)
)
return 0
def _upstream_version(args: argparse.Namespace) -> int:
"""Print upstream's version at an untagged commit, for the release workflow."""
try:
print(upstream_version(Path(args.repo), args.rev, args.nearest_tag))
except RuntimeError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
return 0
def _docs(args: argparse.Namespace) -> int:
"""Regenerate README.md's distribution block and GRAVEYARD.md in place."""
manifest = load_manifest(Path(args.manifest))
cta_path = Path(args.cta) if args.cta else None
update_readme(Path(args.readme), manifest, cta_path=cta_path)
Path(args.graveyard).write_text(render_graveyard(manifest) + "\n")
print(f"rendered {args.readme} and {args.graveyard}", file=sys.stderr)
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="minibwa_dist")
parser.add_argument(
"--manifest", default="minibwa_dist/features.toml", help="path to the distribution manifest"
)
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("sync", help="assemble dist-next")
p.add_argument("--repo", default=".", help="path to the minibwa checkout to assemble in")
p.add_argument(
"--base", default="master", help="ref to assemble onto, normally upstream master"
)
p.add_argument("--out", default="dist-next", help="branch to (re)create with the assembly")
p.add_argument(
"--remote",
required=True,
help="remote namespace for feature branches: nh13 locally, origin in CI",
)
p.add_argument(
"--stamp",
action="store_true",
help="rewrite MB_VERSION so the build identifies itself as downstream",
)
p.add_argument(
"--regression-baseline",
default=None,
metavar="REF",
help="ref whose dist-manifest.json records the previous build; a feature that "
"merged there and does not now fails the run",
)
p.add_argument(
"--stop-at",
default=None,
help="training mode: leave this feature's conflict in the worktree",
)
p.set_defaults(func=_sync)
p = sub.add_parser("reconcile", help="compare recorded statuses to live PR states")
p.add_argument("--pr-states", required=True, help="JSON file: {pr: state}")
p.add_argument("--write", action="store_true", help="rewrite features.toml in place")
p.set_defaults(func=_reconcile)
p = sub.add_parser("notes", help="render release notes")
p.add_argument("--upstream-notes", required=True, help="file holding upstream's own notes")
p.add_argument(
"--upstream-tag",
required=True,
help="upstream tag being mirrored, e.g. v0.6; with --upstream-ref, the last tag before it",
)
p.add_argument(
"--upstream-ref",
default=None,
help="full sha of the untagged upstream commit this build is of",
)
p.add_argument(
"--dropped", nargs="*", help="feature names excluded from this build, space separated"
)
p.set_defaults(func=_notes)
p = sub.add_parser(
"upstream-version", help="print upstream's MB_VERSION at a commit, e.g. 0.7-r427"
)
p.add_argument("--repo", default=".", help="path to a checkout holding the commit")
p.add_argument("--rev", required=True, help="the untagged upstream commit")
p.add_argument(
"--nearest-tag",
default=None,
help="the most recent upstream tag before --rev; its maj.min must match",
)
p.set_defaults(func=_upstream_version)
p = sub.add_parser("docs", help="regenerate README.md block and GRAVEYARD.md")
p.add_argument("--readme", default="README.md", help="README to rewrite the distro block in")
p.add_argument("--graveyard", default="GRAVEYARD.md", help="graveyard file to overwrite")
p.add_argument(
"--cta",
default="minibwa_dist/cta.md",
help="benchmark CTA artifact (from the bench harness); prepended to the "
"distro block when present and its feature set still matches the manifest. "
"Pass '' to omit. A missing file renders nothing, so the default is safe.",
)
p.set_defaults(func=_docs)
p = sub.add_parser("gates", help="run the output gates; exit 1 if any fails")
p.add_argument("--candidate", required=True, help="the distribution binary under test")
p.add_argument("--stock", required=True, help="a binary built from unmodified upstream")
p.add_argument("--fixtures", required=True, help="directory holding upstream's chrM fixtures")
p.add_argument("--workdir", required=True, help="scratch directory for indexes and SAMs")
p.add_argument(
"--assembly",
default=None,
help="assembly JSON, so coverage names only the features this binary contains",
)
# No default: the two gate jobs lay their workspaces out differently -- one
# runs from the assembled checkout, the other from the tooling checkout with
# the candidate in a worktree -- so any default is wrong for one of them. A
# "." default aimed the arm64 gate at the tooling checkout, which carries no
# suites. Optional rather than required, though: see _gates().
p.add_argument(
"--repo",
default=None,
help="the assembled checkout; each merged feature's declared test suites are run in it",
)
p.set_defaults(func=_gates)
p = sub.add_parser("stamp", help="set MB_VERSION to an exact version string")
p.add_argument("--repo", default=".", help="path to the minibwa checkout to stamp")
p.add_argument("--version", required=True, help="version to write, without a leading 'v'")
p.set_defaults(func=_stamp)
args = parser.parse_args(argv)
try:
return int(args.func(args))
except (ManifestError, GitError, OrphanOverPublishedCache) as exc:
# A malformed manifest, a failed git call, and a rerere cache that is
# published but was never fetched locally are all ordinary operating
# conditions for an unattended workflow -- the last of those is what a
# fresh `git clone` looks like. Report them as one line naming the
# cause, not as a traceback that discards an assembly that actually
# succeeded (`_sync` saves the cache from its `finally`, so this can
# fire after `sync` has already done its real work).
print(f"ERROR: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())