8000 ruby-3.3, ruby-3.4: versioned Ruby lines + ruby-flavor guard (after #989) by bryan-minimal · Pull Request #992 · gominimal/pkgs · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions .github/scripts/ruby_flavor_guard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""ruby-flavor guard: no closure may hold two Ruby lines (gominimal/pkgs#992).

`ruby`, `ruby-3.3` and `ruby-3.4` all install the same paths: usr/bin/ruby,
usr/lib/ruby/**, libruby.so and usr/etc/gemrc. A session (or build sandbox)
hardlinks every closure member's files into one tree, first-writer-wins PER
FILE, so two Ruby lines in one closure blend into an interpreter whose stdlib,
libruby and gems come from different releases. Nothing fails loudly; it just
misbehaves. This is the same hazard the node-runtime guard
(node_runtime_guard.py) closes for npm, and this guard has the same shape:

1. flavor A package builds or runs on a versioned line (`ruby-3.3`,
`ruby-3.4`) instead of `ruby`. One flavor across the catalog
means every package that needs Ruby brings the SAME interpreter,
so a user who adds `ruby-lsp` or `bun` gets the `ruby` they
already have. The versioned lines stay in the catalog as explicit
USER opt-ins, like `node` (Current). A package that genuinely
needs one goes in VERSIONED_ALLOWED with the reason.
2. closure A package's or collection's closure holds more than one Ruby
line. Checked for the runtime closure (what a session gets) and
the build closure (build deps plus their runtime closures, what a
build sandbox gets). Collections (stacks) are checked too: a stack
that names `ruby-3.4` next to `ruby-lsp` puts two rubies in one
session.

Not covered: a user's own loadout (the packages list in their project's
minimal.toml) never reaches this catalog, so `ruby-3.4` + `ruby-lsp` chosen
there is invisible to pkgs CI. Rule 1 keeps every packaged tool on `ruby`, so
the collision needs the user to name a versioned line explicitly; catching it
at activation is a minimal-side change.

Input: `minimal dump -p --format json --arch <arch>` output, one file per arch.

Exit codes: 0 = PASS, 1 = violations found, 2 = UNVERIFIED (missing, empty, or
unparseable dump). 2 is deliberately non-zero: a guard that cannot see the
catalog must not report success.
"""

from __future__ import annotations

import argparse
import json
import os
import sys

DEFAULT = "ruby"
RUBIES = frozenset({"ruby", "ruby-3.3", "ruby-3.4"})
# Rule 1 exceptions: package name -> why it cannot use `ruby`. Empty today; a
# new entry needs a reason a reviewer can check (e.g. a gem that does not build
# on the default line yet).
VERSIONED_ALLOWED: dict[str, str] = {}


def dep_name(dep: dict) -> str | None:
"""The package a dep entry points at, whole package or subset."""
if dep.get("type") == "package":
return dep.get("name")
if dep.get("type") == "subset_of":
return dep.get("package")
return None


def load(path: str) -> list[dict]:
with open(path, encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict): # tolerate a {name: spec} shape
data = [dict(v, name=k) for k, v in data.items()]
if not isinstance(data, list) or not data:
raise ValueError(f"{path}: expected a non-empty list of packages")
return data


def runtime_closure(name: str, by_name: dict[str, dict], memo: dict) -> frozenset[str]:
"""`name` plus everything reachable through runtime_deps."""
if name in memo:
return memo[name]
memo[name] = frozenset({name}) # cycle guard; replaced below
seen = {name}
for dep in (by_name.get(name) or {}).get("runtime_deps") or []:
d = dep_name(dep)
if d:
seen |= runtime_closure(d, by_name, memo)
memo[name] = frozenset(seen)
return memo[name]


def check(pkg: dict, by_name: dict[str, dict], memo: dict) -> list[str]:
name = pkg.get("name", "?")
problems = []
if not pkg.get("is_collection") and name not in RUBIES and name not in VERSIONED_ALLOWED:
for field in ("build_deps", "runtime_deps"):
for dep in pkg.get(field) or []:
d = dep_name(dep)
if d in RUBIES and d != DEFAULT:
problems.append(
f"flavor: {field} uses `{d}`; use `{DEFAULT}` "
f"(versioned lines are user opt-ins, see VERSIONED_ALLOWED)"
)
runtime = runtime_closure(name, by_name, memo)
rubies = sorted(RUBIES & runtime)
if len(rubies) > 1:
problems.append(f"closure: runtime closure holds {len(rubies)} Ruby lines: {', '.join(rubies)}")
build: set[str] = set()
for dep in pkg.get("build_deps") or []:
d = dep_name(dep)
if d:
build |= runtime_closure(d, by_name, memo)
rubies = sorted(RUBIES & build)
if len(rubies) > 1:
problems.append(f"closure: build closure holds {len(rubies)} Ruby lines: {', '.join(rubies)}")
return problems


def run(paths: list[str]) -> int:
failures = 0
for path in paths:
label = os.path.basename(path)
try:
pkgs = load(path)
except (OSError, ValueError) as e:
print(f"UNVERIFIED {label}: {e}")
return 2
by_name = {p.get("name"): p for p in pkgs}
memo: dict = {}
for pkg in sorted(pkgs, key=lambda p: p.get("name", "")):
for problem in check(pkg, by_name, memo):
print(f"FAIL {label} {pkg.get('name')}: {problem}")
failures += 1
print(f"checked {len(pkgs)} packages in {label}")
if failures:
print(f"{failures} violation(s)")
return 1
print("PASS")
return 0


def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("dumps", nargs="+", help="minimal dump -p --format json output, one per arch")
return run(ap.parse_args().dumps)


if __name__ == "__main__":
sys.exit(main())
97 changes: 97 additions & 0 deletions .github/scripts/test_ruby_flavor_guard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Unit tests for ruby_flavor_guard.py. Run: python3 -m unittest test_ruby_flavor_guard"""

import json
import os
import sys
import tempfile
import unittest

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import ruby_flavor_guard as g # noqa: E402


def pkg(name, runtime=(), build=(), collection=False):
return {
"name": name,
"is_collection": collection,
"runtime_deps": [{"type": "package", "name": d} for d in runtime],
"build_deps": [{"type": "package", "name": d} for d in build],
}


BASE = [
pkg("glibc"),
pkg("ruby", runtime=["glibc"]),
pkg("ruby-3.3", runtime=["glibc"]),
pkg("ruby-3.4", runtime=["glibc"]),
pkg("ruby-lsp", runtime=["ruby"], build=["ruby"]),
]


def problems(catalog):
by_name = {p["name"]: p for p in catalog}
memo = {}
return {p["name"]: g.check(p, by_name, memo) for p in catalog if g.check(p, by_name, memo)}


class Guard(unittest.TestCase):
def test_catalog_on_one_flavor_passes(self):
self.assertEqual(problems(BASE), {})

def test_stack_with_two_rubies_fails(self):
cat = BASE + [pkg("rails-dev", runtime=["ruby-3.4", "ruby-lsp"], collection=True)]
got = problems(cat)
self.assertIn("rails-dev", got)
self.assertTrue(any("runtime closure holds 2 Ruby lines: ruby, ruby-3.4" in p for p in got["rails-dev"]))

def test_package_on_a_versioned_line_fails_flavor(self):
cat = BASE + [pkg("gem-tool", runtime=["ruby-3.4"])]
got = problems(cat)
self.assertTrue(any(p.startswith("flavor: runtime_deps uses `ruby-3.4`") for p in got["gem-tool"]))

def test_transitive_two_rubies_in_build_closure_fails(self):
cat = BASE + [
pkg("helper", runtime=["ruby-3.3"]),
pkg("app", build=["ruby-lsp", "helper"]),
]
got = problems(cat)
self.assertTrue(any("build closure holds 2 Ruby lines: ruby, ruby-3.3" in p for p in got["app"]))

def test_subset_dep_counts_as_that_line(self):
cat = BASE + [{
"name": "mixed",
"is_collection": True,
"runtime_deps": [
{"type": "subset_of", "package": "ruby-3.4", "outputs": ["libruby"]},
{"type": "package", "name": "ruby-lsp"},
],
"build_deps": [],
}]
self.assertIn("mixed", problems(cat))

def test_allowlisted_package_skips_flavor_only(self):
g.VERSIONED_ALLOWED["gem-tool"] = "test"
try:
got = problems(BASE + [pkg("gem-tool", runtime=["ruby-3.4"])])
self.assertNotIn("gem-tool", got)
finally:
del g.VERSIONED_ALLOWED["gem-tool"]

def test_unreadable_dump_is_unverified(self):
with tempfile.TemporaryDirectory() as d:
empty = os.path.join(d, "dump.json")
with open(empty, "w") as f:
json.dump([], f)
self.assertEqual(g.run([empty]), 2)

def test_run_exits_1_on_violation(self):
with tempfile.TemporaryDirectory() as d:
path = os.path.join(d, "dump.json")
with open(path, "w") as f:
json.dump(BASE + [pkg("x", runtime=["ruby-3.4", "ruby-lsp"], collection=True)], f)
self.assertEqual(g.run([path]), 1)


if __name__ == "__main__":
unittest.main()
55 changes: 55 additions & 0 deletions .github/workflows/ruby-flavor-guard.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
name: ruby-flavor-guard

# BLOCKING. No package or stack closure may hold two Ruby lines
# (gominimal/pkgs#992): `ruby`, `ruby-3.3` and `ruby-3.4` install the same
# paths, and closure members are hardlinked first-writer-wins per file, so two
# lines blend into one broken interpreter. Evaluates the whole catalog on both
# arches and rejects (1) packages on a versioned line instead of `ruby`,
# (2) runtime or build closures holding more than one line. Rationale and
# escape hatch live in .github/scripts/ruby_flavor_guard.py; the shape mirrors
# node-runtime-guard.yml.
#
# A dump failure fails too (exit 2 = UNVERIFIED): a guard that cannot see the
# catalog must not go green.

on:
workflow_dispatch: {}
pull_request:
branches: ["main"]
paths:
- "packages/**"
- "stacks/**"
- ".github/scripts/ruby_flavor_guard.py"
- ".github/scripts/test_ruby_flavor_guard.py"
- ".github/workflows/ruby-flavor-guard.yml"

permissions:
contents: read

concurrency:
group: ruby-flavor-guard-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
guard:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- name: Guard unit tests
run: python3 -m unittest discover -s .github/scripts -p 'test_ruby_flavor_guard.py' -v

# The runner's pinned, sha256-verified evaluator, as node-runtime-guard uses.
- uses: ./.github/actions/setup-mip

- name: Dump catalog (amd64 + arm64)
run: |
set -euo pipefail
mip --version || true
for arch in amd64 arm64; do
mip dump -p --format json --arch "$arch" > "dump-$arch.json"
echo "dumped $(jq 'length' "dump-$arch.json") packages ($arch)"
done

- name: ruby-flavor guard
run: python3 .github/scripts/ruby_flavor_guard.py dump-amd64.json dump-arm64.json
7 changes: 7 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -712,6 +712,13 @@ done

`pkgmgr import npm <name>` generates this layout (using a committed lockfile and `npm ci`); see `packages/vlt`.

#### Ruby pattern

`ruby`, `ruby-3.3` and `ruby-3.4` install the same paths (`usr/bin/ruby`, `usr/lib/ruby/**`, `libruby.so`, `usr/etc/gemrc`), so two of them in one closure blend into one broken interpreter. `.github/workflows/ruby-flavor-guard.yml` enforces two rules:

- **Use `ruby`, not a versioned line**, in `build_deps` and `runtime_deps`. `ruby-3.3` and `ruby-3.4` are there for users to opt into, like `node` (Current); packages don't use them. An exception goes in `VERSIONED_ALLOWED` in `.github/scripts/ruby_flavor_guard.py`, with the reason.
- **No closure may hold two Ruby lines**, runtime or build, and that includes stacks: a stack naming `ruby-3.4` next to `ruby-lsp` fails. A user's own loadout isn't in the catalog, so the guard can't see `ruby-3.4` + `ruby-lsp` chosen there.


### Reproducibility (required)

Expand Down
49 changes: 49 additions & 0 deletions packages/glibc-locale-c-utf8/build.ncl
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
let { Attrs, BuildSpec, Local, OutputData, Test, .. } = import "minimal.ncl" in
let base = import "../base/build.ncl" in
let glibc = import "../glibc/build.ncl" in

# The C.UTF-8 locale. Minimal sessions run with LANG=C.UTF-8, but the glibc
# package's locale-archive only carries en_US, so setlocale(LC_ALL, "") fails
# and anything that checks it breaks. The visible case is PostgreSQL's
# `initdb`, which refuses with "invalid locale settings; check LANG and LC_*".
# Shipped as a separate directory-form locale (the same layout Debian uses for
# C.utf8) so adding it does not rebuild glibc and everything above it. The
# compiled locale format is tied to the glibc version, so this rebuilds
# whenever glibc does.
#
# TODO: fold `localedef -i C -f UTF-8 C.UTF-8` into glibc's own build.sh the
# next time glibc is rebuilt for other reasons, then RETIRE this package in the
# same change: its output (usr/lib/locale/C.utf8/**) sits under glibc's own
# usr/lib/locale/** glob, so once glibc ships the locale the two would install
# the same paths. Drop the edges from ruby, postgres and libidn at that point.
{
name = "glibc-locale-c-utf8",
build_deps = [
{ file = "build.sh" } | Local,
base,
glibc,
],
runtime_deps = [
glibc,
],
cmd = "./build.sh",
outputs = {
locale = { glob = "usr/lib/locale/C.utf8/**" } | OutputData,
},
attrs =
{
upstream_version = glibc.attrs.upstream_version,
license_spdx = "(GPL-2.0-only AND LGPL-2.1-only)",
} | Attrs,
tests = {
# `locale` warns on stderr when LANG names a locale that cannot be loaded.
c_utf8_loads =
{
class = 'Standalone,
test_deps = [base],
cmds = [
["/bin/bash", "-c", "out=$(LANG=C.UTF-8 LC_ALL= locale 2>&1 >/dev/null); [ -z \"$out\" ] || { echo \"$out\" >&2; exit 1; }; LC_ALL=C.UTF-8 locale charmap | grep -qx UTF-8"],
],
} | Test,
},
} | BuildSpec
9 changes: 9 additions & 0 deletions packages/glibc-locale-c-utf8/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash
set -euo pipefail
# Compile glibc's C.UTF-8 locale in directory (not archive) form. glibc's
# setlocale() looks in /usr/lib/locale/locale-archive first and then in
# /usr/lib/locale/<name>/, so this adds the locale without rewriting the
# archive that the glibc package owns.
mkdir -p "$OUTPUT_DIR/usr/lib/locale"
localedef --no-archive -i C -f UTF-8 "$OUTPUT_DIR/usr/lib/locale/C.utf8"
[ -f "$OUTPUT_DIR/usr/lib/locale/C.utf8/LC_CTYPE" ] || { echo "localedef produced no LC_CTYPE" >&2; exit 1; }
Loading
Loading
0