8000 Add reproducible benchmarks and documentation branding by rafatahmed · Pull Request #35 · rafatahmed/Flahax · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Benchmark v1 (informational timing)

on:
workflow_dispatch:

permissions:
contents: read

jobs:
baseline:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.13"
- name: Install benchmark tooling
id: install
run: python -m pip install -e . -r benchmarks/requirements.txt
- name: Measurement semantics and correctness scorecards
run: |
python -m unittest tests.test_benchmark_baseline
python -m tools.benchmark_baseline --output build/benchmark-v1/before.json
- name: ASV measurements (no speed threshold)
if: ${{ !cancelled() && steps.install.outcome == 'success' }}
shell: bash
run: |
python -m asv machine --yes --machine github-linux
python -m asv check --python=same
python -m asv run --python=same --machine github-linux --set-commit-hash "$GITHUB_SHA" --record-samples --show-stderr 2>&1 | tee build/benchmark-v1/asv.log
- name: Record provenance and sample summary
if: always()
run: python -m tools.benchmark_baseline --metadata-only --compare-inputs build/benchmark-v1/before.json --asv-results build/asv/results --output build/benchmark-v1/after.json
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: benchmark-v1-${{ github.sha }}
path: |
build/benchmark-v1/
build/asv/results/
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ include CHANGELOG.md
include CONTRIBUTING.md
include INTEGRATION.md
include pyproject.toml
include asv.conf.json
recursive-include benchmarks *.py *.txt
recursive-include docs *.md *.jpg *.png *.pdf
recursive-include docs/site *.csv
exclude docs/sources/04_Fertilizer.pdf
Expand Down
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# FlahaX

<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/site/_static/brand/v2/flahax-banner-white.png">
<img src="docs/site/_static/brand/v2/flahax-banner-color.png" alt="FlahaX — water and nutrients become chemistry calculations and traceable plans, with units, model identifiers and sources." width="100%">
</picture>

**Plant-nutrient chemistry engine in Python: fertilizer formulation, aqueous
equilibrium chemistry and auditable delivery planning.**

Expand Down Expand Up @@ -95,6 +100,11 @@ plan; they never operate pumps, valves or other hardware.

## Verification

The [Benchmark v1 protocol](docs/site/evidence/benchmark-v1.md) separates
formulation fit, numerical chemistry, runtime/memory measurements, and physical
validation. Its reproducible reports include input hashes and failed cases;
timings are informational, and physical validation remains separate.

- 153 automated tests, including live replay of every catalogue product,
phase and acid case against pinned PHREEQC 3.8.6 (USGS).
- Hosted checks on Python 3.11, 3.12 and 3.13, plus clean wheel and source
Expand Down
14 changes: 14 additions & 0 deletions asv.conf.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"version": 1,
"project": "FlahaX",
"project_url": "https://github.com/rafatahmed/Flahax",
"repo": ".",
"branches": ["main"],
"environment_type": "virtualenv",
"pythons": ["3.13"],
"matrix": {"numpy": ["2.5.4"], "scipy": ["1.18.1"]},
"benchmark_dir": "benchmarks",
"env_dir": "build/asv/env",
"results_dir": "build/asv/results",
"html_dir": "build/asv/html"
}
1 change: 1 addition & 0 deletions benchmarks/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Development-only Benchmark v1; not part of the installed FlahaX package."""
139 changes: 139 additions & 0 deletions benchmarks/cases.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
"""Fixed Benchmark v1 inputs, independent scoring, and correctness gates."""
import hashlib
import json
import math
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
FIXTURES = ROOT / "tests/fixtures/phreeqc"
VERSION = "benchmark-v1"
TARGETS = {"N_NO3": 128, "P": 58, "K": 211, "Ca": 104, "Mg": 40, "S": 54}
WATERS = {"zero_nutrients": {}, "calcium_20_mg_l": {"Ca": 20}}
# Analytic numerical systems, not fertilizer identities. Each reference receives
# precisely these rows and RHS; no library filtering or regularization is hidden.
NNLS_CASES = {
"exact": ([[1., 0.], [0., 1.], [1., 1.]], [2., 3., 5.]),
"infeasible": ([[1.], [1.]], [1., 3.]),
"water_exceeds_target": ([[1., 0.], [0., 1.]], [-1., 2.]),
"duplicate_columns": ([[1., 1.], [2., 2.]], [3., 6.]),
"near_dependent": ([[1., 1.], [1., 1.000001], [0., .000001]], [2., 2.000001, .000001]),
"trace_scale": ([[1., 0.], [0., 1.]], [1e-8, 1.]),
"zero_target": ([[1., 0.], [0., 1.]], [0., 1.]),
}


def digest(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def read_fixture(relative):
path = FIXTURES / relative / "expected.json"
data = json.loads(path.read_text(encoding="utf-8"))
for name, expected in data["hashes"].items():
if digest(path.parent / name) != expected:
raise ValueError(f"Reference hash mismatch: {relative}/{name}")
return data


def objective(matrix, rhs, weights):
if len(weights) != len(matrix[0]) or any(not math.isfinite(x) or x < 0 for x in weights):
raise ValueError("Invalid nonnegative solution")
return float(sum((sum(a*x for a, x in zip(row, weights)) - b)**2
for row, b in zip(matrix, rhs)))


def evaluate_recipe(products, result, targets, water, *, absolute_mg_l=.01, relative=.01):
"""Reconstruct target nutrients from returned rounded doses, not solver rows.

Catalogue-assay evaluator: unassayed counterions are outside this target-fit
scorecard. They remain separately reported by the public workflow.
"""
by_id = {p["id"]: p for p in products}
delivered = dict(water)
seen = set()
for item in result["salts"]:
ident, dose = item["id"], item["gramsPerLitre"]
if ident in seen or ident not in by_id or not math.isfinite(dose) or dose < 0:
raise ValueError("Invalid, duplicate or unidentified returned dose")
seen.add(ident)
for ion, percent in by_id[ident]["elements"].items():
delivered[ion] = delivered.get(ion, 0.) + dose * percent * 10.
rows = []
for ion, target in targets.items():
actual = delivered.get(ion, 0.)
error = actual - target
rows.append({
"ion": ion, "target_mg_l": target, "delivered_mg_l": actual,
"error_mg_l": error,
"error_percent": 100 * error / target if target > 0 else None,
"tolerance_mg_l": max(absolute_mg_l, relative * abs(target)),
"passes": abs(error) <= max(absolute_mg_l, relative * abs(target)),
})
return {"rows": rows, "whole_recipe_passes": all(r["passes"] for r in rows),
"products_used": len(seen), "delivered_assay_mg_l": delivered,
"solver_feasible": result.get("feasible"),
"incidental_requires_review": result.get("requiresReview")}


def solve_fixture(data):
from flahax.aqueous_model import solve, nitric_target
row = data["selected_rows"][-1]
if data.get("nitric_acid_molal") is not None:
plan = nitric_target(data["runtime_totals"], data["initial_ph"], row["pH"])
return plan.target, plan.nitric_acid_molal
return solve(data["runtime_totals"], row["pH"], phases=data.get("phases", ())), None


def evaluate_chemistry(data, result, acid=None):
"""Quantity-specific comparisons using the existing fixture's tolerances."""
row, tol = data["selected_rows"][-1], data["tolerances"]
checks = []

def check(quantity, actual, expected, absolute, relative=0.):
limit = absolute + relative * abs(expected)
checks.append({"quantity": quantity, "actual": actual, "reference": expected,
"absolute_error": abs(actual-expected), "tolerance": limit,
"passes": math.isfinite(actual) and abs(actual-expected) <= limit})

phase_case = "phases" in data
for component, residual in result.residuals.items():
check("conservation:" + component, residual, 0., 1e-15 + 1e-9 * result.totals[component])
check("ionic_strength_mol_kgw", result.ionic_strength, row["mu"], 1e-9, .03)
# pH is prescribed for these captures, not an independently solved pH test.
check("prescribed_pH", result.ph, row["pH"], 1e-12)
for key, expected in row.items():
if key.startswith("m_"):
check(key, result.species[key[2:]], expected,
tol["molality_absolute" if phase_case else "runtime_molality_absolute"],
tol["molality_relative" if phase_case else "runtime_molality_relative"])
elif key.startswith("si_"):
check(key, result.saturation_indices[key[3:]], expected,
tol["si_absolute" if phase_case else "runtime_si_absolute"])
elif key.startswith("la_") and not phase_case and row.get("m_" + key[3:], 0.) > 1e-10:
check(key, math.log10(result.activities[key[3:]]), expected,
tol["runtime_log_activity_absolute"])
for phase in data.get("phases", []):
amount = result.precipitated.get(phase, 0.)
check("phase_mol_kgw:" + phase, amount, row[phase] / row["mass_H2O"],
tol["phase_absolute"], tol["phase_relative"])
si = result.saturation_indices.get(phase)
# Inapplicable phases may be absent when their components are absent.
if si is not None:
checks.append({"quantity": "phase_complementarity:" + phase,
"amount_mol_kgw": amount, "saturation_index": si,
"passes": amount >= 0 and si <= 1e-7
and (amount == 0 or abs(si) <= 1e-7)})
if acid is not None:
for name in ("nitric_acid_molal", "fix_h_nitric_acid_molal"):
check(name, acid, data[name], tol["acid_absolute"], tol["acid_relative"])
checks.append({"quantity": "iterations", "actual": result.iterations,
"passes": 0 <= result.iterations < 1500})
return {"passes": all(c["passes"] for c in checks),
"iterations": result.iterations, "checks": checks}


def require_chemistry(data, result, acid=None):
report = evaluate_chemistry(data, result, acid)
if not report["passes"]:
failed = [c["quantity"] for c in report["checks"] if not c["passes"]]
raise ValueError(f"Chemistry correctness gate failed: {failed}")
Loading
Loading
0