xerj/.github/workflows/ci.yml at main · hpahm/xerj

Repository navigation

View runs

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

on:

push:

branches: [main]

pull_request:

branches: [main]

# Least-privilege by default for an OSS repo; jobs opt into more if needed.

permissions:

contents: read

# Cancel superseded runs on a PR branch (rapid pushes), but NEVER on `main`:

# every commit that lands on the default branch must run CI to completion, or a

# later merge cancels the earlier merge's run and the branch's status badge shows

# that cancellation as "failing" even though nothing broke.

concurrency:

group: ci-${{ github.workflow }}-${{ github.ref }}

cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:

FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"

CARGO_TERM_COLOR: always

# #456: every `uses:` below is pinned to a commit SHA (with the resolved

# tag/version as a trailing comment for humans), including `dtolnay/rust-

# toolchain`, whose git ref IS the rustc version it installs. Before this,

# `@stable` meant "whatever rustup resolves as stable on the day the job

# runs" -- the compiler CI validates against could move with no commit, no

# review, and nothing in the diff. Bumping any of these now takes an actual

# PR, the same as bumping a Cargo dependency. `dtolnay/rust-toolchain@nightly`

# (the fuzz job) is the one exception left floating: nightly drifting is the

# point there, so only its ACTION ref is pinned, not what "nightly" resolves

# to on the day the job runs.

jobs:

# Change scoping. Docs-only pull requests used to run all ~22 jobs, and with

# the org's concurrent-job limit that queued docs PRs behind Rust builds for

# about an hour (53 queued runs on 2026-10-15). `changes` classifies the

# diff with scripts/ci/change_scope.sh (tested by

# scripts/ci/test_change_scope.sh, run first below) and the heavy Rust jobs

# run only when `code == 'true'`. The light guards (cla-config,

# reference-coding, landing-constants-guard, workflow-timeouts,

# release-notes-gate) never depend on it.

# It FAILS OPEN: an empty diff, a failed diff, an all-zeros `before`, or any

# path not on the narrow docs allowlist all mean code=true (run everything).

# If `changes` itself fails, the gated jobs still run: their condition is

# `!cancelled() && (needs.changes.result != 'success' || ...)`, because a

# plain `needs:` would SKIP them, and a skipped required check reads green.

# The allowlist excludes every docs file a test reads by name; see the

# script header for the list and the tests that read them.

# Pushes to main always run everything. Build + Test also carries doc-claim

# tests that walk EVERY .md/.html/.txt in the tree (security_tooling_claims,

# docs_capability_lists), which no path allowlist can exclude. On a PR that

# risk is accepted; on main a docs merge that breaks one of them shows up red

# on its own merge commit instead of on the next unrelated code PR.

changes:

runs-on: ubuntu-latest

timeout-minutes: 5

outputs:

code: ${{ steps.scope.outputs.code }}

ux: ${{ steps.scope.outputs.ux }}

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

with:

fetch-depth: 0

- name: Classifier unit tests

run: bash scripts/ci/test_change_scope.sh

- name: Classify changed files

id: scope

env:

EVENT: ${{ github.event_name }}

PR_BASE: ${{ github.event.pull_request.base.sha }}

PUSH_BEFORE: ${{ github.event.before }}

HEAD_SHA: ${{ github.sha }}

run: |

fail_open() {

echo "::notice::change scope: $1 -> running everything"

printf 'code=true\nux=true\n' >> "$GITHUB_OUTPUT"

exit 0

}

if [ "$EVENT" = "pull_request" ]; then

range="${PR_BASE}...${HEAD_SHA}"

else

if [ -z "$PUSH_BEFORE" ] || [ "$PUSH_BEFORE" = "0000000000000000000000000000000000000000" ]; then

fail_open "no usable push base"

range="${PUSH_BEFORE}..${HEAD_SHA}"

if ! files="$(git diff --name-only "$range")"; then

fail_open "git diff $range failed"

echo "Changed files ($range):"

printf '%s\n' "$files" | sed 's/^/ /'

result="$(printf '%s\n' "$files" | bash scripts/ci/change_scope.sh)" || fail_open "classifier errored"

if [ "$EVENT" != "pull_request" ]; then

echo "Push event: classifier said [${result//$'\n'/ }], running everything (see job comment)"

fail_open "push to $GITHUB_REF"

echo "$result"

echo "$result" >> "$GITHUB_OUTPUT"

cla-config:

runs-on: ubuntu-latest

timeout-minutes: 15

permissions:

contents: read

pull-requests: read

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

# cla-bot reads .contributors straight off this repo; malformed JSON or a

# missing bot exemption breaks the gate silently (it just flags everyone).

# Regression test for the dependabot[bot] exemption (PR #194 comment).

- name: Validate .contributors (valid JSON, bot exemptions present)

run: |

jq -e 'type == "array" and all(.[]; type == "string")' .contributors

jq -e 'index("dependabot[bot]") != null' .contributors

# cla-bot matches commit AUTHORS only, so a maintainer re-land whose

# commits carry the fork contributor's Co-authored-by trailer goes green

# without that contributor ever signing (issue #269; live instance

# PR #248 re-landing #166). Check the trailers too.

- name: Co-author CLA check unit tests

run: python3 scripts/test_cla_coauthors.py

# The narrow field-report carve-out lives here: pass the PR's changed

# files to the checker, which SKIPS the trailer gate iff every changed

# path is a markdown field report under user-feedback/16-agent-field-reports/

# (and nothing else). Any code/CI/other-docs change re-arms the full gate.

# The binding `verification/cla-signed` status for a field-report-only PR

# is provided separately (.github/workflows/cla-field-report-exempt.yml),

# because cla-bot cannot path-scope itself. A rename also emits its source

# path (previous_filename), so renaming a code file into the field-report

# folder cannot hide its deletion from the carve-out.

- name: Co-authored-by trailers resolve to signed contributors

if: github.event_name == 'pull_request'

env:

GH_TOKEN: ${{ github.token }}

PR_NUMBER: ${{ github.event.pull_request.number }}

REPO: ${{ github.repository }}

run: |

FILES="$(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" --jq '.[] | .filename, (.previous_filename // empty)')"

python3 scripts/check_cla_coauthors.py --pr "$PR_NUMBER" --changed-files "$FILES"

reference-coding:

runs-on: ubuntu-latest

timeout-minutes: 25

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

# Issue #977: the xc.py/xc-index.sh/xc-corpus.sh wrapper scripts are now

# `xerj code` + `xerj corpus` + the xerj_code_search MCP tool, so the

# toolkit's contracts are gated where the code lives — scoped cargo

# tests, same profile as every other engine job.

- name: Rust toolchain

uses: dtolnay/rust-toolchain@stable

- name: Cached cargo build

uses: Swatinem/rust-cache@v2

with:

workspaces: engine

# The contracts the old script tests pinned, now in Rust:

# xerj-common::xccode - staleness refusal, RESTRICTED licence

# warning, exit triangle, hybrid shape,

# passage selection, state ledger

# xerj-autoindex::xc - corpus lifecycle, path gate, usage

# xerj-mcp - tool schema presence + isError mapping

# xerj-server - subcommand dispatch + help text

- name: Toolkit tests

working-directory: engine

run: |

# cargo test takes ONE name filter per invocation — the four

# filters are separate runs (the single-command form never parsed:

# "unexpected argument 'xc_' found", and the branch run that would

# have caught it was cancelled before this step). xc::, not xc_:

# the autoindex tests live at xc::tests::… and no test name ever

# contained the substring "xc_" — that filter matched zero tests

# from the day it was written.

for filter in xccode xc:: published_schema help_text; do

cargo test --profile ci-test -p xerj-common -p xerj-autoindex -p xerj-mcp -p xerj-server "$filter"

done

# The corpus manifests are the shareable artifact (issue #319): if they

# cannot rebuild a corpus, publishing them is worse than not shipping

# them. The same untrusted-input gate `xerj corpus add --from` enforces

# now runs as a Rust test (xccode::manifest::hub_manifests_pass_the_untrusted_input_gate).

- name: Vetted hub manifests validate

working-directory: engine

run: cargo test --profile ci-test -p xerj-common xccode::manifest

# The published MCP schema must match the served tools byte-for-byte;

# regenerating it is scripts/mcp-schema-check.sh --write.

# XERJ_BIN must be set explicitly: this job builds with --profile

# ci-test, so the binary the Toolkit-tests loop compiled lands in

# target/ci-test/, while the script's default is target/release/ —

# a binary this job never builds (the step as #977 shipped it failed

# "MCP server binary not found" every run; the same interaction is

# already solved this way in the smoke job below).

- name: Published MCP schema drift

env:

XERJ_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj

run: ./scripts/mcp-schema-check.sh

# #489: this script existed, worked, and had been failing locally for three

# releases — the version footer, headline and narrative on landing/index.html

# stayed pinned to rc.15 through rc.16 and rc.17 because nothing ran it. It is

# fast, needs no toolchain, and only touches static HTML/text, so it belongs

# beside the other docs/tooling gates rather than in the release job —

# catching drift at PR time is the point, not after the site has been stale

# for weeks. fetch-depth: 0 is required (not the default fetch-depth: 1):

# the guard resolves "the newest released tag" via `git tag --list`, which a

# shallow checkout does not fetch.

landing-constants-guard:

runs-on: ubuntu-latest

timeout-minutes: 15

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

with:

fetch-depth: 0

- name: Landing page constants agree with the newest release

run: bash .github/scripts/landing-constants-guard.sh

# #770. A job with no `timeout-minutes` runs against GitHub's 360-minute

# default, and 22 of this repo's 23 jobs had none. Three of its workflows

# serialise on a concurrency group that never auto-cancels

# (`ci-CI-refs/heads/main` here, `pages-deploy`, `release-metrics`), so one

# hung job holds that slot for hours and every later push queues behind it.

# Measured, not hypothetical: on 2026-08-25 the #751 hang ran `Build + Test`

# into the 360-minute default and was killed by it (run 32796557309,

# 01:21:28 -> 07:21:44; every other job in that run finished inside 18 min),

# and a second hang the same morning held the slot 11:28 -> 17:03 while seven

# consecutive main pushes were discarded having run ZERO jobs. #899 fixed that

# particular deadlock; the 6-hour exposure of every OTHER job is what #770 was

# still asking about, and these caps plus the guard below are what close it.

# The caps are sized in three tiers, from the measured duration of every

# successful-or-failed job execution across the last ~245 CI runs (3 873 job

# executions) and 27 release runs:

# 15 min — installs no toolchain, compiles nothing (max 2.3 min repo-wide)

# 30 min — installs a toolchain or a third-party

# binary, compiles no workspace member (max 3.8 min)

# 60 min — compiles workspace crates (max 48.3 min)

# Two jobs sit above their tier for measured reasons, each argued where it is

# set: `autoindex-fd-smoke` at 75 (its windows-latest leg has the heaviest

# tail here, max 49.6 min) and release.yml's cross-compile matrix at 75.

# The point is not to tune each job to its median — a cap that false-fails a

# slow-but-healthy run is worse than the hang it guards against, which is why

# #780 widened build-test's from 50 to 60. It is to put every job at least 4x

# below the 360-minute default so no single hang costs main an afternoon

# again. Job timeouts start when the job STARTS EXECUTING, not when it is

# queued, so runner-slot contention cannot spend any of the budget. That

# contention is real and larger than it looks -- over 2 835 job executions the

# wait from created to started is a 4-second median but a 417-second p90, a

# 1 661-second p99 and a 2 202-second max, with 27% of jobs waiting over a

# minute. It inflates a RUN's wall clock (which is what a reader mistakes for

# a hang) and never a job's timeout, so no cap here can false-fail because the

# runners were busy.

workflow-timeouts:

runs-on: ubuntu-latest

timeout-minutes: 15

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Guard's own unit tests

run: python3 .github/scripts/workflow-timeout-guard.py --self-test

- name: Every workflow job declares a bounded timeout-minutes

run: python3 .github/scripts/workflow-timeout-guard.py

# #474: the rc.18 release PR (#460) shipped three note-vs-tree contradictions

# with the branch 16/16 green every time - a fix credited with more than it

# did (#361), ROADMAP.md contradicting the CHANGELOG in the same commit, and

# a note reverted by a later merge in the same release (#472 vs #457). The

# notes were cut at time T and described main at time T; main kept moving and

# nothing re-checked the notes at merge time. This job re-checks them on

# every push to a release/* PR: ROADMAP/CHANGELOG issue states against the

# live tracker, merge coverage since the previous tag, and freshness against

# the base. It checks out the PR HEAD, not the merge ref, because on the

# merge ref "behind base" is unobservable by construction.

release-notes-gate:

runs-on: ubuntu-latest

timeout-minutes: 15

if: github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/')

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

with:

ref: ${{ github.event.pull_request.head.sha }}

fetch-depth: 0

- name: Notes vs tree (#474)

env:

GH_TOKEN: ${{ github.token }}

BASE_REF: origin/${{ github.base_ref }}

run: |

git fetch origin "${{ github.base_ref }}"

bash .github/scripts/release-notes-gate.sh

lint:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

with:

components: rustfmt, clippy

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

- name: rustfmt (check)

working-directory: engine

run: cargo fmt --all --check

- name: clippy (deny warnings)

working-directory: engine

run: cargo clippy --workspace --all-targets -- -D warnings

# #352: #327 (tree-sitter 0.25) and #317 (0.26) were each green alone, but

# tree-sitter is a `links` crate so only one version may exist. The merged

# tree stopped resolving, taking out build/test/clippy/release at once.

# Each PR's checks ran on its own merge ref (head + base-at-check-time) and

# neither base had the sibling yet; GitHub does not re-run a green PR when the

# base moves, so the conflicting pair was never in one check's tree. This job

# runs on refs/pull/N/merge too (default checkout on pull_request), so it does

# NOT close that stale-base gap by itself. Its value is a fast dedicated signal

# (seconds, no compile) plus `--locked`, which catches Cargo.lock drift no

# other job checks. The tree-sitter recurrence is prevented structurally by the

# hoist to [workspace.dependencies], where two version bumps now collide on one

# line. (#749)

workspace-resolves:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 30

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: cargo metadata (resolves the merge; fails on a links conflict or lock drift)

working-directory: engine

run: cargo metadata --locked --format-version 1 >/dev/null

# XERJ's security pitch is "no JVM, so none of the Log4Shell class of supply

# chain CVEs" — which is only worth anything if somebody is actually watching

# the 660-odd crates we do depend on. This job is that somebody. It was

# documented as existing long before it did (#207); a claimed gate that no

# runner executes is worse than no gate, because readers price it in.

# Failure policy: RUSTSEC *vulnerabilities* fail the build. Informational

# warnings (unmaintained / unsound / yanked) are printed and do not, because

# most of ours sit in transitive crates we cannot upgrade unilaterally and a

# gate that is red for reasons nobody can act on gets ignored, then disabled.

# They are still in the log on every run, which is the point.

security-audit:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 30

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

# Cache the binary, not the advisory database — the database is the part

# that must be fetched fresh on every run. Bump the `-vN` suffix to force

# a cargo-audit upgrade (a very old binary can fail to read a newer

# advisory-db schema).

- name: Cache cargo-audit

uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0

with:

path: ~/.cargo/bin/cargo-audit

key: ci-cargo-audit-bin-v1

- name: Install cargo-audit

run: command -v cargo-audit >/dev/null || cargo install cargo-audit --locked

- name: cargo audit

working-directory: engine

run: cargo audit

# Real fuzzing, for the same reason: `user-feedback/09-security/` promised

# "fuzz testing (cargo-fuzz) on all input parsing paths" while the tree held

# no fuzz target at all (#207). The harnesses live in engine/fuzz and cover

# the parsers an unauthenticated request reaches: the ES query DSL, the

# Lucene query_string grammar, date math and date formats, SQL, and Painless.

# On a pull request this is mostly a *regression* run — every checked-in seed

# is replayed through every target, and the leftover budget explores. For a

# real campaign, run the same script with a bigger budget:

# FUZZ_SECONDS=3600 bash .github/scripts/fuzz-smoke.sh

fuzz:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

# libFuzzer + the sanitizer runtimes are nightly-only.

- name: Install Rust (nightly)

uses: dtolnay/rust-toolchain@7c8d7d138f5c09cef361f8214cf96882cd029cdb # nightly (pinned action ref)

with:

# Temporarily pinned: rustc 1.101.0-nightly (f9ceb4a4d 2026-10-31)

# fails to compile xerj-engine with E0391 ("cycle detected when

# computing when `xerj_fts::search::Query` needs async drop"), a

# compiler regression; nightly-2026-10-30 (rustc 92b5b8c87

# 2026-10-29) built and ran every target. Drop this line to float

# again once a newer nightly compiles the engine.

toolchain: nightly-2026-10-30

components: rustfmt

- name: Cache

uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0

with:

path: |

~/.cargo/registry/cache

~/.cargo/registry/index

~/.cargo/git/db

~/.cargo/bin/cargo-fuzz

engine/fuzz/target

key: ci-fuzz-${{ hashFiles('engine/Cargo.lock', 'engine/fuzz/Cargo.toml') }}

restore-keys: ci-fuzz-

- name: Install cargo-fuzz

run: command -v cargo-fuzz >/dev/null || cargo install cargo-fuzz --locked

# The fuzz crate is its own workspace, so the `lint` job's

# `cargo fmt --all` never sees it. No `--all` here: it would follow the

# path dependencies back into the engine and re-check what `lint` covers.

- name: rustfmt the fuzz crate

working-directory: engine

run: cargo fmt --manifest-path fuzz/Cargo.toml --check

- name: Build + run every fuzz target

env:

# Must match the pinned toolchain above; the script defaults to the

# runner's floating `nightly`, which is the broken 2026-10-31 build.

FUZZ_TOOLCHAIN: nightly-2026-10-30

run: bash .github/scripts/fuzz-smoke.sh

# A crashing input is the entire product of this job; losing it would

# mean the failure is unreproducible.

- name: Upload crashing inputs

if: failure()

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

with:

path: engine/fuzz/artifacts/

if-no-files-found: ignore

build-test:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

# #751's second-order damage was the whole job hanging against GitHub's 6h

# default while main's `cancel-in-progress: false` never auto-cancels — main

# then sat ~4h with no CI verdict. #751's root cause is fixed (see the step-6

# comment below); the step-6 `timeout-minutes: 12` stays as the precise, fast

# bound on any recurrence. This job-level cap is defense-in-depth: it bounds ANY hang in the job

# (setup, cache, a future test race the step timeout would not cover) to

# 60 min instead of the 6h default. It is deliberately generous because

# step 5 (build + threads=2 tests) can legitimately take ~36 min on a loaded

# 2-4 vCPU runner — a slow compile, not a hang (measured: step 5 built for

# ~36 min then step 6 ran and passed) — so a tighter cap would false-fail

# slow-but-healthy runs. #770 measured the legitimate worst case at ~43-46 min

# against the old 50-min cap (setup + ~36-min step 5 + ~2.3-min step 6 +

# cache-save) — too thin a margin on a loaded runner — so this was bumped to

# 60 min. Neither cap is what fixed #751 — the engine deadlock was; they

# bound the blast radius of the next one. By Oct 2026 cold builds took

# ~46-55 min, so healthy runs kept hitting 60 and were cancelled; bumped

# to 90 min.

timeout-minutes: 90

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# Single writer for the Linux [profile.ci-test] dependency cache.

# This job compiles the whole workspace, so its dependency set is a

# superset of what api-checks / conformance / usecase-smoke need;

# they restore this same key read-only. rust-cache's default key is

# per-job, which meant five near-identical ~700 MB copies competing

# for a 10 GB repo budget and evicting each other.

shared-key: ci-test-linux

# One cargo invocation, not two. `cargo build --release --workspace` used

# to run first, but `panic = "abort"` in [profile.release] makes cargo

# rebuild the whole dependency graph with `panic = "unwind"` for the test

# targets, so that step handed the tests almost nothing: on run

# 31571998506 it compiled 39 crates in 9m17s and the test step then

# compiled 453. The fat-LTO link of every workspace member is still

# gated -- it moved to the `release-build` job, which runs concurrently

# instead of in front of this one.

# [profile.ci-test] keeps opt-level 3 and release's assertion settings and

# only drops whole-program LTO / codegen-unit merging. Fat-LTO linking of

# ~99 separate test binaries was 76m05s of this job's 78-minute test step;

# the tests themselves ran in 1m52s.

- name: Unit + integration tests

working-directory: engine

run: cargo test --profile ci-test --workspace -- --test-threads=2

# Issue #372. `--test-threads=2` above is a deliberate configuration, and

# it is not the one a contributor gets: plain `cargo test` runs one thread

# per core. Two xerj-engine lib tests failed on a clean checkout at that

# default and this workflow could not see either of them, because two

# threads serialise enough of the suite to hide the interleaving. The

# general rule the third such defect in one cycle earns: CI must run at

# least one configuration that matches what a contributor actually types.

# Honest about its reach, because "default parallelism" is not a fixed

# number: `ubuntu-latest` is a 2-4 vCPU runner, so the default here is

# 2-4 threads -- barely above the `--test-threads=2` that hid #372, and

# far from the 20 that surfaced it (11 of 16 clean-main runs red on a

# 20-core box). This step would most likely NOT have caught #372 on this

# runner. It is cheap insurance and the place a wider, better-sized gate

# (#384) can land -- not a claim that the class is now covered.

# #384: widened progressively — `-p xerj-engine --lib` -> `--workspace --lib`

# -> the full `--workspace` (all targets, INCLUDING the integration tests

# under `crates/*/tests/*.rs`) — now that every scope is *measured* green at

# default parallelism (6 clean `--workspace --lib` runs, then 4 clean full

# `--workspace` runs, ci-test profile, 20 cores). This was blocked until

# #385: xerj-autoindex used to be red because `incremental_reconcile_http_tests`

# could reach the global `SNAPSHOT_FAILPOINT` without holding the

# `SNAPSHOT_TEST_LOCK` and steal another test's injected failure; the

# thread-local failpoint fix (merged) removed that defect. The integration

# targets bind ports and share fixture dirs, so they are the likeliest to

# carry this class of defect (#384's own note) — but they passed every run

# here, and a hosted runner has FEWER cores than this 20-core box, so its

# default parallelism is strictly lower and less contended than what these

# runs already survived. A gate is worth nothing if it is merged red, so it

# goes in at the width it passes at.

# Reuses the binaries the step above already built: the added cost is test

# runtime, not another compile.

# #751: this step used to hang intermittently on the 2-4 vCPU hosted runner

# (normal ~2.3 min, observed hanging 18-28 min). It was NOT a port or a

# shared fixture — it was a tokio blocking-pool self-deadlock in the engine:

# `Index::search` parks a blocking-pool thread inside `block_in_place` for

# the whole search, and the aggregation-corpus segment hydration then queued

# its decode onto that same pool. Root-caused and fixed in the engine

# (`Index::stored_values_for_async` now decodes on the engine's own rayon

# maintenance pool); pinned by

# `xerj-engine/tests/agg_corpus_hydration_deadlock.rs`.

# The 12-minute cap stays as defense in depth: main's

# `cancel-in-progress: false` never auto-cancels, so ANY future hang here

# would otherwise leave the run stuck for hours with no verdict. A hang

# fails the step in 12 min (5x normal) so the run completes and

# `gh run rerun --failed` can recover.

- name: Workspace tests (all targets) at default parallelism (contributor's `cargo test`)

timeout-minutes: 12

working-directory: engine

run: cargo test --profile ci-test --workspace

# Fat LTO + codegen-units=1 + panic=abort over every workspace member is a

# failure mode of its own (LTO ICEs, symbol collisions, panic-strategy

# mismatches) and is not covered by `cargo clippy --all-targets` in `lint`

# nor by the `-p xerj-server` builds elsewhere -- xerj-wasm, for one, is in

# no other job's dependency closure. So it stays gated, just not in series

# with the tests.

release-build:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

- name: Build (release)

working-directory: engine

run: cargo build --release --workspace

onnx-feature:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

with:

components: clippy

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

- name: Compile the server ONNX feature path

working-directory: engine

run: cargo check -p xerj-server --no-default-features --features onnx-experimental

- name: Test the ONNX embedder

working-directory: engine

< 10BC8 div>

run: cargo test -p xerj-ai --features onnx-experimental --lib -- --test-threads=2

- name: Test engine embedding identity with ONNX

working-directory: engine

run: cargo test -p xerj-engine --features onnx-experimental --lib index::embedding_identity_tests -- --test-threads=2

- name: Clippy the ONNX implementation

working-directory: engine

run: cargo clippy -p xerj-ai -p xerj-engine --all-targets --features onnx-experimental -- -D warnings

api-checks:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# Read-only consumer of build-test's ci-test dependency cache.

# save-if:false is what makes sharing safe: these jobs run

# concurrently, and a cache key is immutable once written, so if

# they all saved it one would win and the rest would log

# "Unable to reserve cache" -- the collision this PR set out to fix.

shared-key: ci-test-linux

save-if: false

- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0

with:

node-version: '20'

# ci-test, not release: this job boots the binary and checks liveness,

# it does not measure performance. Fat LTO cost 6m21s here to run 3s of

# checks. Same opt-level and features, only LTO scope differs.

- name: Build xerj (ci-test profile)

working-directory: engine

run: cargo build --profile ci-test -p xerj-server

# Boots the binary, seeds a real corpus, then gates on:

# smoke suite (61/61), API liveness (no 5xx across the read surface),

# and prints the hot-path benchmark. Deterministic — no model needed.

- name: API smoke + liveness + benchmark

env:

XERJ_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj

run: bash demo/playbooks/ci-check.sh

conformance:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# Read-only consumer of build-test's ci-test dependency cache.

# save-if:false is what makes sharing safe: these jobs run

# concurrently, and a cache key is immutable once written, so if

# they all saved it one would win and the rest would log

# "Unable to reserve cache" -- the collision this PR set out to fix.

shared-key: ci-test-linux

save-if: false

# ci-test: conformance asserts wire behaviour, not throughput.

- name: Build server + conformance runner (ci-test profile)

working-directory: engine

run: cargo build --profile ci-test -p xerj-server -p es-yaml-runner

# rc.3 regression: the shipped xerj.default.toml failed to parse

# (`unknown field hnsw_offload_threshold`) so the documented

# `--config` invocation was a dead boot. Boot the real binary with

# the real shipped file and require it to answer HTTP before the

# conformance server takes port 9200. Runs from a temp dir because

# the shipped config uses a relative `data_dir = "./data"`, and

# accepts any HTTP status (auth is enabled in the shipped config,

# so an unauthenticated probe legitimately gets 401).

- name: Shipped default config boots (rc.3 regression)

run: |

set -euo pipefail

SMOKE="$(mktemp -d)"

cd "$SMOKE"

"$GITHUB_WORKSPACE/engine/target/ci-test/xerj" \

--config "$GITHUB_WORKSPACE/engine/xerj.default.toml" > server.log 2>&1 &

SMOKE_PID=$!

up=""

for _ in $(seq 1 80); do

code="$(curl -s -m1 -o /dev/null -w '%{http_code}' localhost:9200/ || true)"

if [ "$code" != "000" ] && [ -n "$code" ]; then up="$code"; break; fi

if ! kill -0 "$SMOKE_PID" 2>/dev/null; then break; fi

sleep 0.25

done

if [ -z "$up" ]; then

echo "xerj failed to boot with the shipped xerj.default.toml"

cat server.log

exit 1

echo "shipped-config boot OK (HTTP $up)"

kill "$SMOKE_PID" 2>/dev/null || true

wait "$SMOKE_PID" 2>/dev/null || true

- name: Boot xerj

working-directory: engine

run: |

set -euo pipefail

DATA="$(mktemp -d)"

echo "XERJ_DATA=$DATA" >> "$GITHUB_ENV"

./target/ci-test/xerj --insecure --data-dir "$DATA" > "$DATA/server.log" 2>&1 &

echo "XERJ_PID=$!" >> "$GITHUB_ENV"

for _ in $(seq 1 80); do

if curl -fs -m1 localhost:9200/_cluster/health >/dev/null 2>&1; then

echo "xerj is up"; exit 0

sleep 0.25

done

echo "xerj failed to become healthy"; cat "$DATA/server.log"; exit 1

# The runner exits non-zero if any case fails. Skipped cases do not count

# as failures, so the gate is 0 failures — not an exact pass total, which

# grows as cases are added (1365 passed / 3 skipped at the last measured

# run, per #189; this comment previously said 1326 and had drifted).

- name: Run ES-compat YAML suite (0 failures required)

working-directory: engine

run: cargo run --release -p es-yaml-runner -- --dir tests/es-compat-yaml/yaml

- name: Server log on failure

if: failure()

run: cat "$XERJ_DATA/server.log" || true

- name: Stop xerj

if: always()

run: kill "${XERJ_PID:-}" 2>/dev/null || true

# THE MAP's clustering pipeline is pure, dependency-free JS, so its two

# load-bearing claims — "≤ 13 groups at any scale" and "byte-identical

# across runs" — are checkable offline in a few hundred milliseconds. Until

# this job existed the whole pipeline was covered only by a manual Puppeteer

# script that needs a live server and a real browser, i.e. by nothing.

ux-tests:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true' || needs.changes.outputs.ux == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 15

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

# Node 22: the browser suite below drives Chrome over DevTools with the

# global WebSocket, so xerj-ux keeps having no package.json to install.

- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0

with:

node-version: '22'

- name: Console unit tests (THE MAP pipeline, safe-dom, reader, guest contract)

run: node --test xerj-ux/test/*.test.mjs

# The reader shows other people's email and a guest's API key lives in

# sessionStorage: hostile documents must render inert in a REAL browser,

# under the Content-Security-Policy the server ships, and a guest tab

# must never call a console/admin endpoint. The runner image carries

# Chrome; XERJ_REQUIRE_BROWSER turns "no browser found" from a skip into

# a failure, so this can never go quietly green by not running.

- name: Console browser security suite (headless Chrome)

env:

XERJ_REQUIRE_BROWSER: '1'

run: node --test xerj-ux/test/browser/*.test.mjs

# The harnesses under demo/usecases/ are the only executable proof that the

# shipped use cases still work — the brain HTTP surface, the MCP tool surface

# an agent actually sees, and autoindex's zero-config discovery. Every one of

# them was manual until this job, so a regression in any was caught by

# nothing. The installer lint rides along: landing/get{,.ps1} had no

# verification of any kind and needs no toolchain, so it gates first and

# cheaply. The /get counter test rides along for the same reason — it is the

# only gate on the promise that the install counter stores no IP and never

# breaks an install, and it needs nothing but Node.

usecase-smoke:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

timeout-minutes: 60

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Installer script lint (landing/get, landing/get.ps1)

run: bash .github/scripts/installer-lint.sh

- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0

with:

node-version: '20'

- name: Install counter behaviour + privacy (functions/get.js)

run: node .github/scripts/functions-get-test.mjs

- name: Lead request validation (functions/lead.js)

run: node .github/scripts/functions-lead-test.mjs

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# Read-only consumer of build-test's ci-test dependency cache.

# save-if:false is what makes sharing safe: these jobs run

# concurrently, and a cache key is immutable once written, so if

# they all saved it one would win and the rest would log

# "Unable to reserve cache" -- the collision this PR set out to fix.

shared-key: ci-test-linux

save-if: false

- name: Build xerj + xerj-mcp (release)

working-directory: engine

run: cargo build --profile ci-test -p xerj-server -p xerj-mcp

- name: Local definition CLI and MCP parity

run: python3 scripts/mcp-def-smoke.py engine/target/ci-test/xerj

- name: MCP query compatibility and default response budget

run: python3 scripts/mcp-search-contract-smoke.py engine/target/ci-test/xerj

- name: MCP authentication discovery diagnostics

run: python3 scripts/auth-discovery-smoke.py engine/target/ci-test/xerj

# Drives the demo/usecases scripts themselves, so the thing CI gates is

# the same thing a reader runs. No browser, no model, no network.

- name: Use-case smoke (brain transcript, MCP tools, autoindex discovery)

env:

XERJ_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj

XERJ_MCP_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj-mcp

run: bash .github/scripts/usecase-smoke.sh

# Share links end to end, against throwaway nodes this script boots on

# 127.0.0.1:9510-9512: what a guest key can and cannot reach (the live

# security test, with and without a trusted proxy), the `xerj share` CLI,

# the guest page in the runner's headless Chrome (claim, search, read, an

# XSS probe document, sign-out), the open-node refusal, and the

# `xerj brain` -> `xerj share <folder>` path. No npm install: Chrome is

# driven over the DevTools pipe. NOT covered, because it needs the public

# internet and a third party: `--tunnel` opening a real cloudflared quick

# tunnel (its hostname parser and fallbacks are unit tests).

- name: Share links (guest reach, CLI, headless-Chrome guest flow)

env:

XERJ_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj

XERJ_SHARE_REQUIRE_BROWSER: '1'

run: bash .github/scripts/share-links-smoke.sh

# The published agent-facing schema vs what the server actually serves.

# It drifted once already — six tools published against ten served, every

# xerj_brain_* tool missing — and nothing failed, because nothing

# compared them. This drives `xerj mcp` (the subcommand a user who ran

# the installer has, not a dev-only binary), issues a real tools/list

# over stdio, and diffs the response against

# landing/docs/agents/schemas/mcp-tools.json. Needs no running node:

# tools/list is answered before any proxying.

# XERJ_BIN must be set explicitly: this job builds with --profile ci-test

# (#331), so the binary lands in target/ci-test/, while the script's

# default is target/release/. Without this the gate looks for a binary CI

# never built. Each PR was green alone; the interaction only appears once

# both are on main.

- name: MCP published-schema drift gate (xerj mcp ↔ mcp-tools.json)

env:

XERJ_BIN: ${{ github.workspace }}/engine/target/ci-test/xerj

run: bash scripts/mcp-schema-check.sh

# Per-OS runtime gate. The earlier autoindex "Too many open files" crash was

# macOS-only and slipped through because nothing in CI actually RAN the binary

# on macOS/Windows. This job boots xerj and autoindexes a many-dataset corpus

# on each OS under a constrained descriptor budget, failing on any EMFILE.

# macOS runners default to `ulimit -n 256` — the exact condition it regressed

# under.

autoindex-fd-smoke:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

strategy:

fail-fast: false

matrix:

os: [ubuntu-latest, macos-latest, windows-latest]

runs-on: ${{ matrix.os }}

# Tier 3 plus 15 minutes, and the exception is measured rather than felt:

# the windows-latest leg has the widest spread of any job here — over 204

# runs, median 18.8 min, p95 37.6, max 49.6 (ubuntu 15.6, macOS 22.2). At

# 60 that is 1.21x its worst observed run, thinner than `Build + Test`'s

# 1.24x and on a much heavier tail, so a slow-but-healthy Windows runner

# could false-fail. 75 is 1.51x, and still 4.8x below the 360-min default

# that is what actually matters for slot starvation.

timeout-minutes: 75

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# ci-test, not release: this job boots the binary and checks liveness,

# it does not measure performance. Fat LTO cost 6m21s here to run 3s of

# checks. Same opt-level and features, only LTO scope differs.

- name: Build xerj (ci-test profile)

working-directory: engine

run: cargo build --profile ci-test -p xerj-server

- name: Autoindex a many-dataset corpus (must not exhaust file descriptors)

shell: bash

env:

XERJ_BIN: engine/target/ci-test/xerj

run: bash .github/scripts/autoindex-fd-smoke.sh

# Issue #874's verification fixture, kept runnable: 15 repos x ~30 datasets

# (~450 tiny indices), boot, settle 120 s, measure idle CPU / wakeups / RSS

# via /proc, gate coarsely. The unit-level guards (merge-evaluation counter,

# idle-flush, memtable recount tests in xerj-engine) pin the mechanisms; this

# job pins the PROCESS-LEVEL budget they exist to protect: an idle node must

# cost < 0.5 % of one core, O(1) wakeups/s, <= 0.2 MB per idle index, and

# boot without replaying WAL on a cleanly-flushed corpus — independent of N.

# ubuntu only on purpose: the sampler walks /proc. Thresholds are coarse

# (~2x the measured healthy values, see benchmarks/idle-budget/README.md):

# a 2-4 vCPU shared runner cannot support ±1 % claims, and the regressions

# this catches (per-index timers, per-index polling, per-index retention)

# arrive as multiples, not percents.

idle-budget:

needs: [changes]

if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}

runs-on: ubuntu-latest

# One tier up from the no-compile tier: compiles one crate (xerj-server)

# on the shared ci-test cache, then boots and measures. The fixture itself

# is ~8 min measured (two 120 s settle windows + ~450 PUT/_bulk rounds);

# 30 min leaves 3x+ headroom for a cold cache and a slow runner.

timeout-minutes: 30

steps:

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

- name: Install Rust

uses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1

- name: Cache

uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

with:

workspaces: engine

# Read-only consumer of build-test's ci-test dependency cache; that

# job owns the key (same arrangement as api-checks / conformance).

View remainder of file in raw view

FilesExpand file tree

ci.yml

Latest commit

History

ci.yml

File metadata and controls