8000 Web UI and deploy scripts: the Eval page reads /runs and shows status, error kind, metrics and withheld counts; the Fly smoke test PUTs runs by ynishi · Pull Request #43 · ynishi/evalhub · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 47 additions & 13 deletions deploy/fly/smoke.sh
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
#!/usr/bin/env bash
# Publish a real Eval and Card, with attachments, to the hosted service
# and read them back through the API and the UI: the acceptance test of
# a deployment, run from outside.
# Publish a real Eval (a header and its runs) and a Card that judges those
# runs, with attachments, to the hosted service and read them back through
# the API and the UI: the acceptance test of a deployment, run from outside.
#
# bash deploy/fly/user.sh alice # once; writes the token file
# bash deploy/fly/smoke.sh
#
# The token is read from EVALHUB_TOKEN, else from ~/.config/evalhub/token
# (or EVALHUB_TOKEN_FILE), else prompted for without echo when stdin is a
# terminal; it reaches curl through a header file, never a command line.
# Bodies are the schema crate's fixtures with the attachment digests
# replaced by the bytes this script uploads. Needs: curl, jq, sha256sum.
# Bodies are the schema crate's fixtures (eval-run-set.json, run.json,
# card-run-results.json) with the attachment digests replaced by the bytes
# this script uploads and the Card's references pointed at this Eval.
# Needs: curl, jq, sha256sum.
set -euo pipefail
cd "$(dirname "$0")/../.."

Expand Down Expand Up @@ -85,24 +87,54 @@ calls_sha=$(upload "$work/calls.jsonl"); echo " calls.jsonl $calls_sha"
diff_sha=$(upload "$work/diff.patch"); echo " diff.patch $diff_sha"
samples_sha=$(upload "$work/samples.jsonl"); echo " samples.jsonl $samples_sha"

echo "== eval $ns/$eval_name"
echo "== eval $ns/$eval_name (header)"
# The 2.0 header carries no runs; they are written below, one PUT each.
jq '.attachments = [] | .relations = []' crates/evalhub-schema/fixtures/eval-run-set.json > "$work/eval.json"
st=$(req POST "/evals/$ns/$eval_name" "$(cat "$work/eval.json")"); echo " $st $(head -c 300 "$work/body")"
check "eval header POST is 201 or 200" bash -c "[ '$st' = 201 ] || [ '$st' = 200 ]"
eval_seq=$(jq -r '.seq // empty' "$work/body")
# The Card below cites this seq; without one it would cite a wrong version.
[ -n "$eval_seq" ] || { echo "smoke: no seq from the Eval POST" >&2; exit 1; }

echo "== runs of $ns/$eval_name"
# r1 is fixtures/run.json with its attachments' digests replaced by the
# uploaded bytes; r2 is the same run failed, so the Card below can name
# both runs its attrs.runs lists.
jq --arg s1 "$calls_sha" --argjson n1 "$(stat -c %s "$work/calls.jsonl")" \
--arg s2 "$diff_sha" --argjson n2 "$(stat -c %s "$work/diff.patch")" '
.attachments = [
{path:"calls/r1.jsonl", sha256:$s1, size:$n1, media_type:"application/x-ndjson"},
{path:"artifacts/r1/diff.patch", sha256:$s2, size:$n2, media_type:"text/x-diff"}]
| .relations = []' crates/evalhub-schema/fixtures/eval-run-set.json > "$work/eval.json"
st=$(req POST "/evals/$ns/$eval_name" "$(cat "$work/eval.json")"); echo " $st $(head -c 300 "$work/body")"
check "eval POST is 201 or 200" bash -c "[ '$st' = 201 ] || [ '$st' = 200 ]"
eval_seq=$(jq -r '.seq // empty' "$work/body")
{path:"artifacts/r1/diff.patch", sha256:$s2, size:$n2, media_type:"text/x-diff"}]' \
crates/evalhub-schema/fixtures/run.json > "$work/r1.json"
jq '.run_id = "r2" | .status = "error" | .error = {kind:"timeout", message:"no answer in 600 s"}
| del(.calls, .artifacts, .attachments) | .metrics = {"core/duration_ms": 600000.0}' \
crates/evalhub-schema/fixtures/run.json > "$work/r2.json"
for run in r1 r2; do
st=$(req PUT "/evals/$ns/$eval_name/runs/$run" "$(cat "$work/$run.json")"); echo " $run $st $(head -c 300 "$work/body")"
check "run $run PUT is 201 or 200" bash -c "[ '$st' = 201 ] || [ '$st' = 200 ]"
done
st=$(req GET "/evals/$ns/$eval_name/runs"); echo " runs GET $st $(head -c 300 "$work/body")"
check "runs GET is 200" [ "$st" = 200 ]
check "runs GET lists r1 as ok" jq -e '[.items[] | select(.run_id == "r1" and .status == "ok")] | length == 1' "$work/body"
check "runs GET lists r2 as error timeout" jq -e '[.items[] | select(.run_id == "r2" and .status == "error" and .error_kind == "timeout")] | length == 1' "$work/body"
st=$(req GET "/evals/$ns/$eval_name"); check "eval GET counts the runs" bash -c "[ '$st' = 200 ] && jq -e '.runs.count >= 2' '$work/body' >/dev/null"

echo "== card $ns/$card_name"
jq --arg s "$samples_sha" --argjson n "$(stat -c %s "$work/samples.jsonl")" --arg to "$ns/$eval_name@${eval_seq:-1}" '
jq --arg s "$samples_sha" --argjson n "$(stat -c %s "$work/samples.jsonl")" \
--arg to "$ns/$eval_name@$eval_seq" --arg eval "$ns/$eval_name" '
.attachments = [{path:"samples.jsonl", sha256:$s, size:$n, media_type:"application/x-ndjson"}]
| .relations[0].to = $to' crates/evalhub-schema/fixtures/card-complete.json > "$work/card.json"
| .relations[0].to = $to
| .run_results |= map(.eval = $eval)' crates/evalhub-schema/fixtures/card-run-results.json > "$work/card.json"
st=$(req POST "/cards/$ns/$card_name" "$(cat "$work/card.json")"); echo " $st $(head -c 300 "$work/body")"
check "card POST is 201 or 200" bash -c "[ '$st' = 201 ] || [ '$st' = 200 ]"

echo "== runs of $ns/$eval_name with the judgements of $ns/$card_name"
st=$(req GET "/evals/$ns/$eval_name/runs?cards=$ns/$card_name"); echo " $st $(head -c 300 "$work/body")"
check "runs GET with cards= is 200" [ "$st" = 200 ]
check "the card used both runs" jq -e --arg c "$ns/$card_name" '.cards[0].card == $c and .cards[0].runs_used == 2' "$work/body"
check "r1 is judged pass" jq -e --arg c "$ns/$card_name" '.items[] | select(.run_id == "r1") | .cards[$c].results | any(.metric == "core/pass" and .label == "pass")' "$work/body"
check "r2 is judged fail" jq -e --arg c "$ns/$card_name" '.items[] | select(.run_id == "r2") | .cards[$c].results | any(.metric == "core/pass" and .label == "fail")' "$work/body"

echo "== read back"
st=$(req GET "/cards/$ns/$card_name?expand=fingerprints,badges,changed"); echo " card GET $st"
check "card GET is 200" [ "$st" = 200 ]
Expand All @@ -123,8 +155,10 @@ echo "== make public, then read without a token"
st=$(req PATCH "/evals/$ns/$eval_name/settings" '{"visibility":"public"}'); check "eval public" [ "$st" = 200 ]
st=$(req PATCH "/cards/$ns/$card_name/settings" '{"visibility":"public"}'); check "card public" [ "$st" = 200 ]
anon=$(curl -s -o /dev/null -w '%{http_code}' "$api/cards/$ns/$card_name"); check "anonymous card GET is 200" [ "$anon" = 200 ]
anon=$(curl -s -o /dev/null -w '%{http_code}' "$api/evals/$ns/$eval_name/runs?cards=$ns/$card_name"); check "anonymous runs GET with cards= is 200" [ "$anon" = 200 ]
anon=$(curl -s -o /dev/null -w '%{redirect_url}' "$api/attachments/$samples_sha"); check "anonymous attachment GET redirects" [ -n "$anon" ]
ui=$(curl -s -o /dev/null -w '%{http_code}' "$base/cards/$ns/$card_name"); check "UI deep link is 200" [ "$ui" = 200 ]
ui=$(curl -s -o /dev/null -w '%{http_code}' "$base/evals/$ns/$eval_name"); check "UI Eval deep link is 200" [ "$ui" = 200 ]

echo
if [ "$fails" -eq 0 ]; then
Expand Down
110 changes: 108 additions & 2 deletions web/src/lib/api/client.ts
< 10BC8 td class="blob-num blob-num-addition empty-cell">
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,24 @@ export type Member = components['schemas']['MemberDto'];
export type Scope = components['schemas']['ScopeDto'];
/** Whether a record is public or private. */
export type Visibility = components['schemas']['VisibilityParam'];
/** What a read removed from a record body because the reader may not see it. */
export type Withheld = components['schemas']['Withheld'];
/** An Eval's runs, counted, as its `GET` envelope carries them. */
export type RunsSummary = components['schemas']['RunsSummaryDto'];
/** One page of an Eval's runs, joined with the requested Cards' judgements. */
export type RunPage = components['schemas']['RunPageDto'];
/** One row of that page. */
export type RunRow = components['schemas']['RunRowDto'];
/** A Card of the request, as the run page joined it. */
export type RunPageCard = components['schemas']['RunPageCardDto'];
/** One run as stored, or its tombstone. */
export type RunEnvelope = components['schemas']['RunEnvelope'];
/** What `PUT …/runs/{run_id}` did. */
export type RunWritten = components['schemas']['RunWrittenDto'];
/** What `POST …/runs:batch` did, per run. */
export type RunsWritten = components['schemas']['BatchResponse'];
/** Why a run is deleted. */
export type TombstoneReason = components['schemas']['TombstoneReasonParam'];

/** The two kinds of record, as they appear in a URL. */
export type Kind = 'cards' | 'evals';
Expand Down Expand Up @@ -217,6 +235,93 @@ export async function comparison(ns: string, name: string, groupBy?: string): Pr
return unwrap(await api.GET('/api/v1/evals/{ns}/{name}/cards', params), 'the comparison view');
}

/** The query string of `GET …/runs`. `cards` and `sort` repeat. */
export interface RunsQuery {
/** `{ns}/{name}` of each Card whose judgements to join, in column order. */
cards?: string[];
/** The filter, as the query grammar's JSON text. */
where?: string;
/** `{path}`, `{path}:asc` or `{path}:desc`, most significant first. */
sort?: string[];
/** Page size, 1–200. */
limit?: number;
/** The previous page's `next_cursor`. */
cursor?: string;
/** `archived`, `deleted` or both; honoured for members only. */
include?: string;
}

/** One page of an Eval's runs. Runs belong to the record, so `name` never
* carries an `@{seq}`. */
export async function listRuns(ns: string, name: string, query: RunsQuery = {}): Promise<RunPage> {
const params = { params: { path: { ns, name }, query } } as never;
return unwrap(await api.GET('/api/v1/evals/{ns}/{name}/runs', params), 'listing runs');
}

/** One run as stored; a deleted run comes back as its tombstone. */
export async function getRun(ns: string, name: string, runId: string): Promise<RunEnvelope> {
const params = { params: { path: { ns, name, run_id: runId } } } as never;
return unwrap(
await api.GET('/api/v1/evals/{ns}/{name}/runs/{run_id}', params),
'reading the run'
);
}

/** Write one run under the `run_id` it carries. */
export async function putRun(
ns: string,
name: string,
run: Record<string, unknown> & { run_id: string }
): Promise<RunWritten> {
const params = { params: { path: { ns, name, run_id: run.run_id } }, body: run } as never;
return unwrap(
await api.PUT('/api/v1/evals/{ns}/{name}/runs/{run_id}', params),
'writing the run'
);
}

/** Write several runs in one transaction: all or nothing. */
export async function putRuns(
ns: string,
name: string,
runs: Record<string, unknown>[]
): Promise<RunsWritten> {
const params = { params: { path: { ns, name } }, body: { runs } } as never;
return unwrap(await api.POST('/api/v1/evals/{ns}/{name}/runs:batch', params), 'writing the runs');
}

/** Archive or unarchive a run. */
export async function archiveRun(
ns: string,
name: string,
runId: string,
archived: boolean
): Promise<RunEnvelope> {
const params = { params: { path: { ns, name, run_id: runId } }, body: { archived } } as never;
return unwrap(
await api.PATCH('/api/v1/evals/{ns}/{name}/runs/{run_id}', params),
'archiving the run'
);
}

/** Delete a run. The id, the content hash and the metrics stay. */
export async function deleteRun(
ns: string,
name: string,
runId: string,
reason: TombstoneReason,
note?: string
): Promise<RunEnvelope> {
const params = {
params: { path: { ns, name, run_id: runId } },
body: { reason, note: note ?? null }
} as never;
return unwrap(
await api.DELETE('/api/v1/evals/{ns}/{name}/runs/{run_id}', params),
'deleting the run'
);
}

/** Make a record public or private. */
export async function setVisibility(
kind: Kind,
Expand Down Expand Up @@ -314,8 +419,9 @@ export async function listRegistry(
).items;
}

/** Fetch one of the served JSON Schemas (`card`, `eval`, `error`,
* `query`). The query builder walks these for its path vocabulary, which
/** Fetch one of the served JSON Schemas (`card`, `eval` (the 1.0 body,
* until 0.3.0), `eval-2` (the Eval header), `run`, `error`, `query`). The
* query builder walks these for its path vocabulary, which
* is why it stays right when the record types change. */
export async function getSchema(name: string): Promise<Record<string, unknown>> {
const response = await raw(`/schemas/${name}`, { method: 'GET' });
Expand Down
2 changes: 1 addition & 1 deletion web/src/lib/components/QueryBuilder.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
error = null,
onrun
}: {
schemaName: 'card' | 'eval';
schemaName: 'card' | 'eval-2';
error?: unknown;
onrun: (where: Filter | undefined) => void;
} = $props();
Expand Down
15 changes: 12 additions & 3 deletions web/src/lib/components/RecordList.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,20 @@
type Kind,
type ListItem,
type QueryHit,
type QueryRequest
type QueryRequest,
type Withheld as WithheldCounts
} from '$lib/api/client';
import Badges from '$lib/components/Badges.svelte';
import Errors from '$lib/components/Errors.svelte';
import QueryBuilder from '$lib/components/QueryBuilder.svelte';
import Withheld from '$lib/components/Withheld.svelte';

let { kind }: { kind: Kind } = $props();

const schemaName = $derived(kind === 'cards' ? 'card' : 'eval');
// `/schemas/eval` stays the 1.0 document (header and `runs[]` in one)
// for the compatibility release; the stored Eval is the 2.0 header,
// which is what a record query ranges over.
const schemaName = $derived(kind === 'cards' ? 'card' : 'eval-2');
const singular = $derived(kind === 'cards' ? 'Card' : 'Eval');

let ns = $state('');
Expand All @@ -34,6 +39,8 @@
badges: string[];
created_at: string;
visibility?: string;
/** What the query result's body lost to the caller's visibility. */
withheld?: WithheldCounts | null;
}

let rows = $state<Row[]>([]);
Expand Down Expand Up @@ -67,7 +74,8 @@
: undefined,
seq: hit.seq,
badges: hit.badges ?? [],
created_at: hit.created_at
created_at: hit.created_at,
withheld: hit.withheld
};
}

Expand Down Expand Up @@ -204,6 +212,7 @@
<td>
<a href="/{kind}/{row.ns}/{row.name}">{row.ns}/{row.name}</a>
{#if row.visibility === 'private'}<span class="tag">private</span>{/if}
<Withheld withheld={row.withheld} compact />
</td>
<td>{row.title ?? ''}</td>
<td class="num">{row.seq}</td>
Expand Down
39 changes: 39 additions & 0 deletions web/src/lib/components/Withheld.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<script lang="ts">
// What a read removed from a record body because the reader may not see
// it: relation edges to versions they cannot see, and a Card's
// `run_results[]` entries that judge runs of an Eval they cannot see.
// Only the counts are shown; the hub itself says no more than that
// (a withheld relation keeps its commitment, a withheld run result
// keeps nothing but the fact that there was one). Nothing renders when
// nothing was withheld.
import type { Withheld } from '$lib/api/client';

let { withheld, compact = false }: { withheld: Withheld | null | undefined; compact?: boolean } =
$props();

const relations = $derived(withheld?.relations.length ?? 0);
const runResults = $derived(withheld?.run_results ?? 0);
const parts = $derived(
[
relations > 0 ? `${relations} ${relations === 1 ? 'relation' : 'relations'}` : null,
runResults > 0 ? `${runResults} ${runResults === 1 ? 'run result' : 'run results'}` : null
].filter((p): p is string => p !== null)
);
const summary = $derived(`withheld: ${parts.join(', ')}`);
</script>

{#if parts.length > 0}
{#if compact}
<span
class="tag"
title="Removed from the body because they point at records this session may not see."
>{summary}</span
>
{:else}
<div class="notice" role="note">
<strong>Part of this record is not shown to you.</strong>
The hub removed {parts.join(' and ')} that point at records this session may not see, so the body
below is not the stored one and does not match its content hash.
</div>
{/if}
{/if}
2 changes: 2 additions & 0 deletions web/src/routes/cards/[ns]/[name]/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import Errors from '$lib/components/Errors.svelte';
import Facets from '$lib/components/Facets.svelte';
import Relations from '$lib/components/Relations.svelte';
import Withheld from '$lib/components/Withheld.svelte';
import { session } from '$lib/session.svelte';

const ns = $derived(page.params.ns ?? '');
Expand Down Expand Up @@ -97,6 +98,7 @@

<Errors {error} />
{#if notice}<p class="notice">{notice}</p>{/if}
<Withheld withheld={version?.withheld} />

{#if version?.tombstone}
<div class="notice error">
Expand Down
Loading
Loading
0