offshoot/Makefile at main · sricola/offshoot

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

8000 193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

.PHONY: test test-torture fuzz build test-s3 bench bench-cow bench-s3 check-python-version test-python-sdk test-ts-sdk test-sdks test-python-langgraph \

check-sdk-versions dry-run-python-sdk dry-run-ts-sdk dry-run-sdks test-pytest-plugin \

ci-local ci-local-host ci-local-linux ci-local-s3 ci-local-minio ci-local-sdks lint lint-python \

check-plugin bench-isolation bench-branchbench bench-ab example-pass-k

# Override with `make PYTHON=python3.14 ...` when the platform's unversioned

# python3 is older than the SDKs' declared Python 3.10 minimum.

PYTHON ?= python3

check-python-version:

@$(PYTHON) -c 'import sys; sys.exit("offshoot SDK tooling requires Python 3.10+ (got " + sys.version.split()[0] + "); pass PYTHON=python3.10 or newer") if sys.version_info < (3, 10) else None'

test:

go test ./... -count=1

# lint fails if any file is unformatted (gofmt -l prints offenders; the

# `(! read)` trick makes any output a nonzero exit) or go vet finds a

# problem. gofmt sees the Go files git tracks or would add — not ignored

# scratch directories such as .superpowers/, which `gofmt -l .` walked and

# which once failed the gate on a file that could never ship; staticcheck runs best-effort last (`go run` fetches it, so it

# needs network — the || clause keeps an offline run from failing the two

# gates that already passed). ci.yml runs the same gofmt/vet pair on every

# PR so an unformatted non-test file can't ship silently again (audit §7).

# STATICCHECK pins a master commit, not a release: v0.8.1 (the latest

# release as of 2026-10-10) cannot read the Go 1.27 export data the

# toolchain in go.mod produces ("export data version 5 is greater than

# maximum supported version 4"), and the || clause hid that for every run

# since the 1.27 upgrade. Move the pin to the next tagged release once one

# supports 1.27. lint-python is the same gate for the Python SDKs, scripts

# and examples (ruff.toml at the repo root is the rule set).

STATICCHECK ?= honnef.co/go/tools/cmd/staticcheck@v0.7.0-0.dev.0.20261009230814-452d5bb86b45

lint:

gofmt -l $$(git ls-files -co --exclude-standard '*.go') | tee /dev/stderr | (! read)

go vet ./...

go run $(STATICCHECK) ./... || echo "staticcheck failed or unavailable (non-blocking)"

# lint-python runs ruff (lint + format check) over every Python file the

# repo tracks, with the rules in ruff.toml. ci.yml's sdks job runs the same

# two commands. RUFF defaults to the pinned-version uvx form so a machine

# with uv needs nothing installed; override with RUFF=ruff for a global

# install (pip install ruff==$(RUFF_VERSION)).

RUFF_VERSION = 0.17.0

RUFF ?= uvx ruff@$(RUFF_VERSION)

lint-python:

$(RUFF) check .

$(RUFF) format --check .

test-torture:

go test ./internal/capture -tags=torture -run TestTorture -count=1 -timeout 30m -v

# fuzz runs every Go native fuzz target for 30s each (~3 min), sequentially:

# `-fuzz` takes one target per package run. The nightly fuzz job runs the

# same loop at 60s per target. A crasher lands in the package's

# testdata/fuzz/<Target>/ and reproduces with

# `go test ./internal/<pkg> -run '<Target>/<id>'`.

FUZZ_TARGETS := store:FuzzValidateName ltxio:FuzzDecodeSnapshot ltxio:FuzzApplySegments ops:FuzzReadSidecar daemon:FuzzDecodeRequest

FUZZTIME ?= 30s

fuzz:

@set -e; for spec in $(FUZZ_TARGETS); do \

pkg=$${spec%%:*}; target=$${spec##*:}; \

echo "== $$target (internal/$$pkg, $(FUZZTIME))"; \

go test ./internal/$$pkg -run '^$$' -fuzz "^$$target\$$" -fuzztime $(FUZZTIME) -fuzzminimizetime 5s; \

done

build:

go build -o bin/torture ./cmd/torture

test-s3:

go test ./internal/store -run TestS3RealProvider -count=1 -v

# bench runs internal/ops's Fork/Checkout/session.Open benchmarks against a

# local store (see internal/ops/fork_bench_test.go and docs/benchmarks.md).

# -short skips the size=4GB Fork case: a multi-minute, multi-GB-RAM run,

# meant to be measured explicitly and rarely, not on every `make bench`. Run

# it directly when you want that number:

# go test ./internal/ops -bench 8000 9;ForkAtHead/size=4GB' -benchmem -run '^$$' -benchtime=1x -timeout 20m

bench:

go test ./internal/ops -bench . -benchmem -run '^$$' -count=3 -short

# bench-cow runs the copy-on-write fork benchmarks (internal/ops/

# cow_bench_test.go) exactly as docs/benchmarks.md's "Copy-on-write fork

# cost" section was produced: a fixed-iteration latency sweep repeated

# -count=5 (report the median), then the added-bytes accounting at

# N=1/10/100 shared forks and N=1/10 materialized forks (each N is a

# separate -benchtime=Nx invocation because the per-fork byte metric is

# diffed across exactly b.N forks), then the divergence-cost measurement.

# Not -short: the 1GB latency case is deliberately included here and only

# here. Local store only — see docs/benchmarks.md for why the S3 object

# COUNTS are identical by construction.

bench-cow:

go test ./internal/ops -bench 'CoWSharedFork$$|CoWCheckoutAfterFork|CoWCpBaseline' -benchmem -run '^$$' -count=5 -benchtime=3x -timeout 30m

go test ./internal/ops -bench 'CoWSharedForkAddedBytes|CoWMaterializedForkAddedBytes' -run '^$$' -benchtime=1x

go test ./internal/ops -bench 'CoWSharedForkAddedBytes|CoWMaterializedForkAddedBytes' -run '^$$' -benchtime=10x

go test ./internal/ops -bench 'CoWSharedForkAddedBytes' -run '^$$' -benchtime=100x

go test ./internal/ops -bench 'CoWDivergenceAddedBytes' -run '^$$' -benchtime=1x -count=3 -timeout 30m

# bench-s3 runs the same benchmarks against a real S3-compatible backend

# (RustFS in Docker — the same digest ci.yml pins) instead of the local

# store, so docs/benchmarks.md's local-store numbers can be compared against

# the network-bound path. Requires Docker. Always tears the container down,

# even on failure. The benchmark bucket is created by the ops benchmarks'

# own setup via OFFSHOOT_S3_CREATE_BUCKET (see internal/store's

# EnsureBucketForTest) — no client image needed.

S3_BENCH_IMAGE = rustfs/rustfs@sha256:8cc9801755448b71a786705ce76692c77e14936cccd87cf2fc31842e58f4d1ff

bench-s3:

docker rm -f offshoot-bench-s3 >/dev/null 2>&1 || true

docker run -d --name offshoot-bench-s3 -p 9100:9000 \

-e RUSTFS_ACCESS_KEY=rustfsadmin -e RUSTFS_SECRET_KEY=rustfsadmin \

$(S3_BENCH_IMAGE)

for i in $$(seq 1 30); do \

curl -sSf http://127.0.0.1:9100/health >/dev/null 2>&1 && break; \

sleep 1; \

done

curl -sSf http://127.0.0.1:9100/health >/dev/null

OFFSHOOT_S3_TEST_BUCKET=offshoot-bench \

OFFSHOOT_S3_CREATE_BUCKET=1 \

OFFSHOOT_S3_ENDPOINT=http://127.0.0.1:9100 \

OFFSHOOT_S3_PATH_STYLE=1 \

AWS_ACCESS_KEY_ID=rustfsadmin \

AWS_SECRET_ACCESS_KEY=rustfsadmin \

go test ./internal/ops -bench . -benchmem -run '^$$' -count=1 -short; \

status=$$?; \

docker rm -f offshoot-bench-s3 >/dev/null 2>&1; \

exit $$status

# Named explicitly (not `discover -s tests`, which globs test_*.py) because

# tests/test_pytest_plugin.py — the offshoot.pytest_plugin fixture plugin's

# own suite — imports pytest at module level and must NOT be picked up here:

# this target proves the base SDK's plain-unittest suites pass with no

# pytest installed at all. See `make test-pytest-plugin` for that file.

test-python-sdk: check-python-version

cd sdk/python && $(PYTHON) -m unittest tests.test_client tests.test_langgraph -v

test-ts-sdk:

cd sdk/typescript && npm install --no-audit --no-fund && npm test

# test-sdks is deliberately not a dependency of the default `test` target:

# it needs python3 and node/npm on PATH, which the Go suite does not.

test-sdks: test-python-sdk test-ts-sdk

# The companion's runtime install deliberately depends only on LangGraph's

# checkpoint packages; its [test] extra supplies pytest + the full framework

# needed by these real-StateGraph integration tests. Keep this separate from

# test-sdks so that target continues proving the base Python SDK is stdlib-only.

test-python-langgraph: check-python-version

cd sdk/python-langgraph && OFFSHOOT_REQUIRE_LANGGRAPH=1 $(PYTHON) -m pytest tests -v

# test-pytest-plugin runs offshoot.pytest_plugin's own test suite

# (sdk/python/tests/test_pytest_plugin.py). Unlike test-python-sdk (plain

# unittest, deliberately pytest-free so `pip install offshoot-db` with no

# extra keeps working), this target legitimately needs pytest + pytest-xdist

# on PATH — it IS the test suite for the pytest fixture plugin (the

# `offshoot-db[pytest]` extra) and its xdist smoke scenario. Not a

# dependency of test-sdks/test: see CONTRIBUTING.md's dev setup for the

# `pip install "sdk/python[pytest]" pytest-xdist` this needs first. Builds

# the offshoot binary once and pins OFFSHOOT_BIN so both the daemon fixture

# and its nested pytester-driven runs reuse it instead of a repeat `go

# build`.

test-pytest-plugin: check-python-version

go build -o bin/offshoot-pytest-plugin-test ./cmd/offshoot

OFFSHOOT_BIN=$(CURDIR)/bin/offshoot-pytest-plugin-test \

$(PYTHON) -m pytest sdk/python/tests/test_pytest_plugin.py -v

# check-sdk-versions verifies sdk/VERSION (the single source of truth — see

# CONTRIBUTING.md's "Release process") agrees with the version literally

# spelled out in sdk/python/pyproject.toml and sdk/typescript/package.json.

check-sdk-versions: check-python-version

$(PYTHON) scripts/check_sdk_versions.py

# dry-run-python-sdk builds the real sdist+wheel `offshoot-db` would publish,

# runs twine's metadata check against them, then installs the wheel into a

# throwaway venv and import-tests it. This is exactly what

# .github/workflows/publish.yml's PyPI job runs when the repository variable

# PUBLISH_ENABLED is not "true" instead of uploading anywhere; ci.yml's sdks

# job runs it on every PR so a manifest mistake (bad classifier, missing

# readme, broken package-discovery glob) fails long before a release tag

# does. Needs `python3 -m pip install build twine` on PATH — see

# CONTRIBUTING.md's dev setup.

dry-run-python-sdk: check-sdk-versions

rm -rf sdk/python/dist

cd sdk/python && $(PYTHON) -m build

$(PYTHON) -m twine check sdk/python/dist/*

rm -rf sdk/python/.dry-run-venv

$(PYTHON) -m venv sdk/python/.dry-run-venv

sdk/python/.dry-run-venv/bin/pip install --quiet sdk/python/dist/*.whl

sdk/python/.dry-run-venv/bin/python3 -c "import offshoot; print('import offshoot: ok,', offshoot.__doc__.splitlines()[0])"

rm -rf sdk/python/.dry-run-venv

# dry-run-ts-sdk builds dist/ (from a clean slate — rm -rf first, so a stale

# dist/ left over from a different files-array/branch can never be what

# gets packed), packs the exact tarball `npm publish` would upload

# (respecting package.json's "files" whitelist), asserts the tarball's

# actual contents are EXACTLY the allowed set (not "at least" or "no worse

# than" — any addition or removal fails the build, so a loosened "files"

# array or a stray committed dist/ artifact fails CI, not a human's

# memory), then installs *that tarball* — not the source tree — into a

# throwaway project and import-tests the published entry point. This is

# exactly what .github/workflows/publish.yml's npm job runs when

# PUBLISH_ENABLED is not "true" instead of uploading anywhere; ci.yml's

# sdks job runs it on every PR. Needs `npm` and `tar` on PATH.

dry-run-ts-sdk:

rm -rf sdk/typescript/dist

cd sdk/typescript && npm install --no-audit --no-fund && npm run build

rm -rf sdk/typescript/dry-run-pack sdk/typescript/dry-run-install

mkdir -p sdk/typescript/dry-run-pack sdk/typescript/dry-run-install

cd sdk/typescript && npm pack --silent --pack-destination dry-run-pack

tar -tzf sdk/typescript/dry-run-pack/offshoot-db-client-*.tgz | sort > sdk/typescript/dry-run-pack/actual-tarball-contents.txt

printf '%s\n' package/README.md package/dist/client.d.ts package/dist/client.js package/dist/testkit.d.ts package/dist/testkit.js package/package.json | sort > sdk/typescript/dry-run-pack/expected-tarball-contents.txt

diff -u sdk/typescript/dry-run-pack/expected-tarball-contents.txt sdk/typescript/dry-run-pack/actual-tarball-contents.txt

@echo "tarball contents: exact match (README.md, dist/client.d.ts, dist/client.js, dist/testkit.d.ts, dist/testkit.js, package.json) — nothing more, nothing less"

cd sdk/typescript/dry-run-install && npm init -y --silent >/dev/null

cd sdk/typescript/dry-run-install && npm install --no-audit --no-fund --silent ../dry-run-pack/offshoot-db-client-*.tgz

cd sdk/typescript/dry-run-install && node -e "import('@offshoot-db/client').then((m) => { if (typeof m.connect !== 'function') throw new Error('connect export missing from published package'); console.log('import @offshoot-db/client: ok'); })"

cd sdk/typescript/dry-run-install && node -e "import('@offshoot-db/client/testkit').then((m) => { if (typeof m.startDaemon !== 'function' || typeof m.seedOnce !== 'function' || typeof m.forkPerTest !== 'function' || typeof m.dump !== 'function') throw new Error('testkit exports missing from published package'); console.log('import @offshoot-db/client/testkit: ok'); })"

rm -rf sdk/typescript/dry-run-pack sdk/typescript/dry-run-install

# dry-run-sdks runs both dry runs — the "does this build and install"

# tier that gates on nothing but PATH tools, run on every PR touching

# sdk/** (see ci.yml's sdks job) so PUBLISH_ENABLED flipping on is never the

# first time a manifest mistake is discovered.

dry-run-sdks: dry-run-python-sdk dry-run-ts-sdk

# ci-local mirrors .github/workflows/ci.yml's job matrix locally, so

# CI-equivalent signal lands in minutes instead of waiting on a runner. Each

# job is also runnable alone (`make ci-local-host`, etc.) for a tighter

# loop while iterating on one thing. `ci-local` runs all four in sequence

# and prints a pass/fail-per-job summary table; see scripts/ci-local.sh's

# header and CONTRIBUTING.md's "Local CI" section for exactly what this

# does and does NOT prove versus a real GitHub Actions run.

ci-local:

./scripts/ci-local.sh all

ci-local-host:

./scripts/ci-local.sh host

ci-local-linux:

./scripts/ci-local.sh linux

ci-local-s3:

./scripts/ci-local.sh s3

ci-local-minio: ci-local-s3 # old name

ci-local-sdks:

./scripts/ci-local.sh sdks

.PHONY: third-party-licenses

# third-party-licenses regenerates THIRD_PARTY_LICENSES.csv from the current

# module graph. Runnable locally (network access required, to fetch

# go-licenses itself and consult per-dependency license URLs) and used by

# release.yml's collect job so a release always ships an up-to-date bundle.

third-party-licenses:

go run github.com/google/go-licenses@v1.6.0 report ./... > THIRD_PARTY_LICENSES.csv

check-plugin:

./scripts/check-plugin.sh

# bench-isolation runs scripts/bench-isolation.py: a stdlib-only benchmark

# of the primitives an eval harness could use to get a fresh, isolated

# database per test (offshoot fork+open+close, offshoot fork alone,

# sqlite3.Connection.backup(), shutil.copyfile, and — when Docker and a

# local postgres:16 image are available — Postgres `CREATE DATABASE ...

# TEMPLATE`, a cold-container start, and the docker-exec/readiness overhead

# figures), at seed sizes 10 and 100 MB. See docs/benchmarks.md's "Per-test

# isolation primitives" section for the measured numbers this produces, and

# the script's own docstring for what each row is. Builds the offshoot

# binary fresh. Stdlib only -- the script reaches sdk/python via sys.path

# (like sdk/python/tests/test_client.py's DaemonFixture does), never a pip

# install, so it runs directly under $(PYTHON) with no venv needed. Measured

# ~26s with the postgres:16 image already cached locally; without Docker,

# the Postgres rows and overhead figures skip instantly (the script never

# attempts a network pull).

bench-isolation: check-python-version

go build -o bin/offshoot-bench ./cmd/offshoot

$(PYTHON) scripts/bench-isolation.py --sizes 10,100 --iters 20

# BranchBench's five agentic-workflow topologies (arXiv:2604.17180) re-run

# against a local offshoot store, full-scale parameters: ~2,300 forks in all.

# Every workflow runs against its own fresh store (built, seeded and removed

# per workflow), so this needs about 30 GB of free space in $TMPDIR -- the

# largest single workflow, mcts, peaked near 26.5 GiB before v0.2.12 (now

# 43 GiB logical but ~2 GiB physical on APFS, where checkouts, shadows and

# by-chain entries are clones; not measured on a filesystem that cannot

# clone) -- and about 2 minutes on an

# Apple M5. `go run ./cmd/branchbench -quick` is the seconds-scale smoke

# run of the same five topologies, and `go test ./cmd/branchbench` runs it.

# Prints one markdown table; docs/benchmarks.md pastes it verbatim.

bench-branchbench:

@echo "branchbench: needs ~30 GB free in TMPDIR and ~2 minutes; Ctrl-C removes the store. Smoke run: go run ./cmd/branchbench -quick"

go run ./cmd/branchbench

# bench-ab alternates branchbench built from BEFORE (default: the latest

# release tag) with one built from the working tree, four rounds per

# workflow (cmd/bench-ab's default), and prints per-metric medians with a

# regressed/ok verdict; only fork, checkout and checkpoint p50 gate that

# verdict. About 10 minutes with the default rounds and workflows.

# --match 'v[0-9]*' keeps a stray non-release tag from becoming the

# baseline (see nightly.yml's perf-ab job). The tool is built to

# bin/bench-ab and run from there because its exit codes (0 ok, 1 a gating

# regression, 2 a tool error) are the binary's: go run would fold them all

# into 1. make still folds a failing recipe into its own exit 2, so read

# the report (or run ./bin/bench-ab directly) to tell a regression from a

# tool error.

bench-ab:

go build -o bin/bench-ab ./cmd/bench-ab

./bin/bench-ab -before "$${BEFORE:-$$(git describe --tags --abbrev=0 --match 'v[0-9]*')}" -run-timeout 10m

# example-pass-k runs examples/eval-pass-k/run.py: a runnable pass^k eval

# loop over offshoot (see docs/recipes/eval-harnesses.md's "tau2-style

# environments and pass^k" section) -- seed once, fork per trial, grade

# each attempt with `offshoot diff` against a golden reference, throw the

# fork away. Stdlib only -- it reaches sdk/python via sys.path, exactly

# like scripts/bench-isolation.py does, so no pip install and no venv are

# needed here. Defaults (--k 4 --tasks 5) keep it well under 30s (~2s

# measured locally) so it's cheap enough to run on every PR.

example-pass-k: check-python-version

go build -o bin/offshoot-bench ./cmd/offshoot

OFFSHOOT_BIN=$(CURDIR)/bin/offshoot-bench \

$(PYTHON) examples/eval-pass-k/run.py --k 4 --tasks 5

FilesExpand file tree

Makefile

Latest commit

History

Makefile

File metadata and controls