8000 test: qualify the #841 nightly candidates (#863) by adityamukho · Pull Request #876 · project-minigraf/minigraf · GitHub
[go: up one dir, main page]

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions .github/workflows/qualify-863.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
name: Qualify #863 nightly candidates (temporary)

# Temporary, #863 spec §4: three runs on separate ubuntu-latest runners measure
# each candidate's cost (fixture construction, warm-up and measurement inside
# its Criterion case, plus its acceptance test) and its run-to-run spread.
# Nothing is uploaded to Bencher. Removed by the PR's last commit, before merge.

on:
pull_request:
paths:
- .github/workflows/qualify-863.yml
- benches/helpers/shape.rs
- benches/minigraf_bench/nightly_candidates.rs
- tests/nightly_candidates_acceptance_test.rs

permissions:
contents: read

jobs:
qualify:
name: qualify / run ${{ matrix.run }}
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
run: [1, 2, 3]
env:
CARGO_TERM_COLOR: never
IDS: >-
window/nightly_candidates/skew_running_sum/100k
query/nightly_candidates/set_membership_k256/10k
query/nightly_candidates/sort_offset_desc/100k
query/nightly_candidates/sort_offset_asc/100k
recursion/nightly_candidates/tree_ancestors_k1_pending/1110
recursion/nightly_candidates/tree_ancestors_k16_committed/1110
TESTS: >-
n1_skew_running_sum_streams_by_partition
n2_set_membership_k256
n3_sort_heap_holds_offset_plus_limit
n4_small_tree_takes_ancestor_alternative
n5_committed_tree_uses_bound_value_probes
n6_held_cursor_quarantine_and_reuse
steps:
- uses: actions/checkout@v4

- name: Install Rust stable
uses: dtolnay/rust-toolchain@stable

- name: Record runner
run: |
mkdir -p qualify
{
echo "run=${{ matrix.run }}"
echo "sha=$(git rev-parse HEAD)"
echo "image=${ImageOS:-unknown} ${ImageVersion:-unknown}"
echo "nproc=$(nproc)"
grep -m1 'model name' /proc/cpuinfo
free -m | sed -n 2p
rustc --version
} | tee qualify/runner.txt

- name: Build (not counted per candidate)
run: |
cargo bench --bench minigraf_bench --no-run
cargo test --profile bench --features test-hooks \
--test nightly_candidates_acceptance_test --no-run

- name: Criterion cases
run: |
set -o pipefail
for id in $IDS; do
start=$(date +%s.%N)
cargo bench --bench minigraf_bench -- "^${id}\$" 2>&1 | tee -a qualify/criterion.txt
end=$(date +%s.%N)
echo "{\"kind\":\"criterion\",\"id\":\"$id\",\"seconds\":$(python3 -c "print(round($end - $start, 3))")}" >> qualify/timings.jsonl
done
mkdir -p qualify/estimates
# benchmark.json names the full ID; estimates.json has the times.
n=0
for d in $(find target/criterion -path '*nightly_candidates*' -type d -name new); do
n=$((n + 1))
cp "$d/benchmark.json" "qualify/estimates/$n-benchmark.json"
cp "$d/estimates.json" "qualify/estimates/$n-estimates.json"
done

- name: Acceptance tests
run: |
set -o pipefail
for t in $TESTS; do
start=$(date +%s.%N)
cargo test --profile bench --features test-hooks \
--test nightly_candidates_acceptance_test -- --ignored --nocapture \
--test-threads=1 --exact "$t" 2>&1 | tee -a qualify/acceptance.txt
end=$(date +%s.%N)
echo "{\"kind\":\"acceptance\",\"test\":\"$t\",\"seconds\":$(python3 -c "print(round($end - $start, 3))")}" >> qualify/timings.jsonl
done

- name: Upload evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: qualify-863-run-${{ matrix.run }}
path: qualify/
if-no-files-found: error
retention-days: 30
1 change: 1 addition & 0 deletions benches/helpers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -517,3 +517,4 @@ fn insert_val_facts(db: &Minigraf, n: usize) {
db.execute(&cmd).unwrap();
}
}
pub mod shape;
Loading
Loading
0