8000 build(spec-harness): runnable executable-spec test harness by devill · Pull Request #67 · habit-hooks/habit-hooks · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,15 @@ coverage
*.tgz
*.log
.vscode/
.venv/
__pycache__/
*.pyc
.pytest_cache/

# Local habit-hooks dogfooding setup (the npm tool we run against this code,
# not part of the rebuild deliverable). Installed via `npm i habit-hooks`.
package.json
package-lock.json
habit-hooks.config.js
.habit-hooks-baseline.json
ruff.toml
49 changes: 49 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""Collect the executable specs (``docs/**/*.spec.md``) as pytest tests.

Each leaf spec case (per ``docs/executable_spec.md``) becomes one pytest item,
so ``uv run pytest`` reports the specs alongside the harness's own unit tests
with native pass/skip/fail. The engine lives in ``tests/harness.py`` (importable
via the ``pythonpath`` set in ``pyproject.toml``).
"""

from __future__ import annotations

import tempfile
from pathlib import Path

import pytest

from harness import SpecCase, SpecError, SpecFailure, execute, parse_spec

_REPO_ROOT = Path(__file__).parent


def pytest_collect_file(parent, file_path):
if file_path.name.endswith(".spec.md"):
return SpecFile.from_parent(parent, path=file_path)


class SpecFile(pytest.File):
def collect(self):
for case in parse_spec(self.path.read_text()):
yield SpecItem.from_parent(self, name=case.name, case=case)


class SpecItem(pytest.Item):
def __init__(self, *, case: SpecCase, **kwargs):
super().__init__(**kwargs)
self.case = case

def runtest(self):
if self.case.skip:
pytest.skip("🟡 not built yet")
with tempfile.TemporaryDirectory() as tmp:
execute(self.case, Path(tmp), _REPO_ROOT)

def repr_failure(self, excinfo):
if isinstance(excinfo.value, (SpecError, SpecFailure)):
return str(excinfo.value)
return super().repr_failure(excinfo)

def reportinfo(self):
return self.path, None, self.name
26 changes: 26 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
[project]
name = "habit-hooks"
version = "0.0.0"
description = "Turn best-practice coding advice into AI habits — structural code-smell coaching for AI agents"
requires-python = ">=3.11"

[tool.uv]
# habit-hooks itself ships as plugin/skill files, not a Python package; this
# pyproject only wires up the dev tooling (the spec test harness under tests/).
package = false

[dependency-groups]
dev = ["pytest>=8", "markdown-it-py>=3"]

[tool.pytest.ini_options]
testpaths = ["tests", "docs"]
pythonpath = ["tests"]

[tool.deptry]
# `harness` and `tools` are our own packages (tests/, on pytest's pythonpath).
known_first_party = ["harness", "tools"]

[tool.deptry.per_rule_ignores]
# pytest is a dev-only test framework; its use in pytest's own conftest glue is
# expected, not a misplaced production dependency.
DEP004 = ["pytest"]
13 changes: 13 additions & 0 deletions tests/harness/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""Engine for the executable specs (``docs/**/*.spec.md``).

Dev/test tooling for building habit-hooks, not shipped product code. The grammar
contract is ``docs/executable_spec.md``. pytest is the runner (see
``conftest.py``); this package only parses (`parse_spec`) and runs (`execute`).
"""

from .errors import SpecError, SpecFailure
from .parser import SpecCase, parse_spec
from .runner import execute
from .text import normalize

__all__ = ["SpecCase", "SpecError", "SpecFailure", "execute", "normalize", "parse_spec"]
9 changes: 9 additions & 0 deletions tests/harness/errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""Exceptions raised while parsing or running a spec."""


class SpecError(Exception):
"""A spec is malformed (a parse-time problem)."""


class SpecFailure(Exception):
"""A test assertion failed or a step errored at run time."""
12 changes: 12 additions & 0 deletions tests/harness/glyphs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
"""Marker codepoints. Any U+FE0F variation selector is ignored when matching."""

FILE = "\U0001F4C4" # 📄
ENV = "✏" # ✏️
STDIN = "⌨" # ⌨️
SCREEN = "\U0001F5A5" # 🖥️
STDERR = "\U0001F6A8" # 🚨
SKIP = "\U0001F7E1" # 🟡
PASS = "✅" # ✅
FAIL = "❌" # ❌

MARKERS = {FILE: "file", ENV: "env", STDIN: "stdin", SCREEN: "screen", STDERR: "stderr"}
61 changes: 61 additions & 0 deletions tests/harness/markdown.py
< 10000 td class="blob-num blob-num-addition empty-cell">
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
"""Markdown → the ordered Heading / Marker / Block elements the grammar uses."""

from __future__ import annotations

from dataclasses import dataclass

from markdown_it import MarkdownIt
from markdown_it.tree import SyntaxTreeNode

from .glyphs import MARKERS, SKIP


@dataclass
class Heading:
level: int
text: str
skip: bool


@dataclass
class Marker:
kind: str
arg: str


@dataclass
class Block:
info: str
content: str

@property
def is_command(self) -> bool:
# ```bash is reserved for commands and never consumed as a marker payload.
head = self.info.split()
return bool(head) and head[0] == "bash"


def _heading(node: SyntaxTreeNode) -> Heading:
text = node.children[0].content.rstrip()
skip = text.replace("️", "").rstrip().endswith(SKIP)
return Heading(int(node.tag[1]), text, skip)


def _markers(text: str) -> list[Marker]:
"""Every marker line in a paragraph (one paragraph may hold several)."""
lines = (line.lstrip() for line in text.split("\n"))
return [Marker(MARKERS[h[0]], h[1:].replace("️", "")) for h in lines if h and h[0] in MARKERS]


def read_elements(text: str) -> list[object]:
"""Walk the markdown tree, emitting the elements we care about in order."""
root = SyntaxTreeNode(MarkdownIt("commonmark").parse(text))
elements: list[object] = []
for node in root.children:
if node.type == "heading":
elements.append(_heading(node))
elif node.type == "fence":
elements.append(Block(node.info.strip(), node.content.rstrip("\n")))
elif node.type == "paragraph":
elements.extend(_markers(node.children[0].content))
return elements
155 changes: 155 additions & 0 deletions tests/harness/parser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
"""Parse markdown elements into runnable SpecCases (the leaf execution contexts)."""

from __future__ import annotations

import re
from dataclasses import dataclass, field

from .errors import SpecError
from .glyphs import FAIL, PASS
from .markdown import Block, Heading, Marker, read_elements
from .steps import Command, CopyFile, Screen, SetEnv, Stderr, Stdin, WriteFile


@dataclass
class SpecCase:
name: str
skip: bool
steps: list[object]


def _exit_code(arg: str) -> int | None:
fail = re.search(FAIL + r"\s*(\d+)", arg)
if fail:
return int(fail.group(1))
return 0 if PASS in arg else None


def _file_step(arg: str) -> tuple[object, bool | None]:
arg = arg.strip()
if "@" in arg:
dst, src = (p.strip() for p in arg.split("@", 1))
return CopyFile(dst or src, src), None
return WriteFile(arg), True


def build_marker(marker: Marker) -> tuple[object, bool | None]:
"""(step, fence): fence is None=no block, True=required, False=optional."""
kind, arg = marker.kind, marker.arg
if kind == "file":
return _file_step(arg)
if kind == "env":
return SetEnv(arg.strip()), True
if kind == "stdin":
return Stdin(), True
if kind == "screen":
return Screen(_exit_code(arg) or 0), False
return Stderr(_exit_code(arg)), False # kind == "stderr"


class StepBuilder:
"""Pairs marker elements with their payload fences into ordered steps."""

def __init__(self):
self.steps: list[object] = []
self.pending: tuple[object, bool] | None = None # (step, fence_required)

def build(self, elements: list[object]) -> list[object]:
for element in elements:
self._feed(element)
self._flush()
return self.steps

def _feed(self, element: object) -> None:
if isinstance(element, Marker):
self._on_marker(element)
elif isinstance(element, Block):
self._on_block(element)

def _on_marker(self, marker: Marker) -> None:
self._flush()
step, fence = build_marker(marker)
if fence is None:
self.steps.append(step)
else:
self.pending = (step, fence)

def _on_block(self, block: Block) -> None:
if block.is_command:
self._flush()
self.steps.append(Command(block.content))
elif self.pending is not None: # a bare fence with no owning marker is cosmetic
self.pending[0].block = block.content
self.steps.append(self.pending[0])
self.pending = None

def _flush(self) -> None:
if self.pending is None:
return
step, required = self.pending
if required:
raise SpecError(f"marker {step!r} has no code block")
self.steps.append(step)
self.pending = None


@dataclass
class _Node:
level: int
skip: bool
parent: "_Node | None"
name: str
elements: list[object] = field(default_factory=list)
has_child: bool = False


class TreeBuilder:
"""Builds the heading tree; each node owns the elements directly beneath it."""

def __init__(self):
self.nodes: list[_Node] = []
self.stack: list[_Node] = [] # current ancestry, by increasing heading level

def build(self, elements: list[object]) -> list[_Node]:
for element in elements:
self._feed(element)
return self.nodes

def _feed(self, element: object) -> None:
if isinstance(element, Heading):
self._open(element)
elif self.stack:
self.stack[-1].elements.append(element)

def _open(self, heading: Heading) -> None:
while self.stack and self.stack[-1].level >= heading.level:
self.stack.pop()
parent = self.stack[-1] if self.stack else None
node = _Node(heading.level, heading.skip, parent, heading.text)
if parent is not None:
parent.has_child = True
self.nodes.append(node)
self.stack.append(node)


def _ancestry(leaf: _Node) -> list[_Node]:
chain: list[_Node] = []
node: _Node | None = leaf
while node is not None:
chain.append(node)
node = node.parent
chain.reverse()
return chain


def _leaf_case(leaf: _Node) -> SpecCase:
"""A leaf's test case: its ancestors' preambles, in order, then its own steps."""
chain = _ancestry(leaf)
steps = [step for node in chain for step in StepBuilder().build(node.elements)]
return SpecCase(leaf.name, any(n.skip for n in chain), steps)


def parse_spec(text: str) -> list[SpecCase]:
"""The markdown's leaf contexts, each as a runnable test case."""
nodes = TreeBuilder().build(read_elements(text))
return [_leaf_case(node) for node in nodes if not node.has_child]
16 changes: 16 additions & 0 deletions tests/harness/runner.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""Run a parsed SpecCase to completion."""

from __future__ import annotations

from pathlib import Path

from .parser import SpecCase
from .steps import Context


def execute(test: SpecCase, workdir: Path, repo_root: Path) -> None:
"""Run a test's steps in `workdir`; raise on any assertion failure."""
context = Context(workdir, repo_root)
for step in test.steps:
step.apply(context)
context.check_default_exit()
Loading
Loading
0