8000 fix(eval): render metrics that report a value but reach no verdict by i-yliu · Pull Request #528 · google/adk-web · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions src/app/components/chat/chat.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -724,17 +724,19 @@
<div style="display: flex; gap: 12px; align-items: center; flex-wrap: wrap;">
@for (metric of result.overallEvalMetricResults; track metric.metricName) {
<div class="metric-block"
[style.border]="metric.evalStatus == 1 ? '1px solid #2e7d32' : '1px solid var(--mat-sys-error)'"
[style.border]="'1px solid ' + getMetricColor(metric)"
style="position: relative; display: flex; flex-direction: column; gap: 2px; background: var(--mat-sys-surface-container-high); padding: 6px 12px; border-radius: 6px; flex-shrink: 0; cursor: pointer;">
<span style="color: var(--mat-sys-on-surface-variant); font-size: 11px; font-weight: 500;">{{ metric.metricName | formatMetricName }}</span>
<div style="display: flex; align-items: baseline; gap: 4px;">
<span [style.color]="metric.evalStatus == 1 ? '#2e7d32' : 'var(--mat-sys-error)'"
<span [style.color]="getMetricColor(metric)"
style="font-size: 16px; font-weight: 600;">
{{ metric.score != null ? (metric.score | number:'1.2-2') : '?' }}
</span>
<span style="color: var(--mat-sys-on-surface-variant); font-size: 14px; font-weight: 500;">
/ {{ metric.threshold | number:'1.2-2' }}
</span>
@if (metric.threshold != null) {
<span style="color: var(--mat-sys-on-surface-variant); font-size: 14px; font-weight: 500;">
/ {{ metric.threshold | number:'1.2-2' }}
</span>
}
</div>

<!-- Tooltip -->
Expand All @@ -743,14 +745,16 @@
<div class="tooltip-subtitle" style="font-size: 10px; color: var(--mat-sys-on-surface-variant); margin-bottom: 4px;">{{ metric.metricName }}</div>
<div class="tooltip-grid">
<div class="tooltip-item"><span class="tooltip-label">Actual:</span> <span class="tooltip-value"
[style.color]="metric.evalStatus == 1 ? '#2e7d32' : 'var(--mat-sys-error)'">{{ metric.score != null ?
[style.color]="getMetricColor(metric)">{{ metric.score != null ?
(metric.score | number:'1.2-2') : '?' }}</span></div>
<div class="tooltip-item"><span class="tooltip-label">Threshold:</span> <span class="tooltip-value">{{
metric.threshold | number:'1.2-2' }}</span></div>
<div class="tooltip-item"><span class="tooltip-label">Min:</span> <span class="tooltip-value">{{
getMetricMin(metric.metricName) }}</span></div>
<div class="tooltip-item"><span class="tooltip-label">Max:</span> <span class="tooltip-value">{{
getMetricMax(metric.metricName) }}</span></div>
@if (metricHasVerdict(metric)) {
<div class="tooltip-item"><span class="tooltip-label">Threshold:</span> <span class="tooltip-value">{{
metric.threshold | number:'1.2-2' }}</span></div>
<div class="tooltip-item"><span class="tooltip-label">Min:</span> <span class="tooltip-value">{{
getMetricMin(metric.metricName) }}</span></div>
<div class="tooltip-item"><span class="tooltip-label">Max:</span> <span class="tooltip-value">{{
getMetricMax(metric.metricName) }}</span></div>
}
</div>
@let desc = getMetricDescription(metric.metricName);
@if (desc) {
Expand Down
17 changes: 17 additions & 0 deletions src/app/components/chat/chat.component.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,23 @@ describe('ChatComponent', () => {
expect(component.rubricPassed({})).toBe(false);
});

// EvalStatus: 1 PASSED, 2 FAILED, 3 NOT_EVALUATED, 4 INFORMATIONAL.
it('metricHasVerdict is false only for an informational metric', () => {
expect(component.metricHasVerdict({evalStatus: 1})).toBe(true);
expect(component.metricHasVerdict({evalStatus: 2})).toBe(true);
// NOT_EVALUATED keeps the failure styling it carried before this change.
expect(component.metricHasVerdict({evalStatus: 3})).toBe(true);
expect(component.metricHasVerdict({evalStatus: 4})).toBe(false);
});

it('getMetricColor is neutral when the metric reached no verdict', () => {
expect(component.getMetricColor({evalStatus: 1})).toBe('#2e7d32');
expect(component.getMetricColor({evalStatus: 2}))
.toBe('var(--mat-sys-error)');
expect(component.getMetricColor({evalStatus: 4}))
.toBe('var(--mat-sys-on-surface-variant)');
});

it(
'should pre-fill user input from "q" query param only when app is selected',
fakeAsync(() => {
Expand Down
20 changes: 19 additions & 1 deletion src/app/components/chat/chat.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import { catchError, distinctUntilChanged, filter, first, map, shareReplay, star

import { URLUtil } from '../../../utils/url-util';
import { AgentRunRequest } from '../../core/models/AgentRunRequest';
import { EvalCase, EvaluationResult } from '../../core/models/Eval';
import { EvalCase, EvaluationResult, EvalStatus } from '../../core/models/Eval';
import { Session, SessionState } from '../../core/models/Session';
import { Event as AdkEvent, Part } from '../../core/models/types';
import { UiEvent } from '../../core/models/UiEvent';
Expand Down Expand Up @@ -512,6 +512,24 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy {
return info?.description || '';
}

/**
* Whether a metric reached a pass/fail verdict. An informational metric
* reports a value and never does, so it must not be styled as a failure
* merely for not having passed.
*/
metricHasVerdict(metric: any): boolean {
return metric?.evalStatus !== EvalStatus.INFORMATIONAL;
}

/** Green when passed, red when failed, neutral when there is no verdict. */
getMetricColor(metric: any): string {
if (!this.metricHasVerdict(metric)) {
return 'var(--mat-sys-on-surface-variant)';
}
return metric?.evalStatus === EvalStatus.PASSED ? '#2e7d32' :
'var(--mat-sys-error)';
}

/** Whether a single rubric result passed, per the backend verdict/score. */
rubricPassed(rubric: any): boolean {
if (rubric?.verdict !== undefined && rubric?.verdict !== null) {
Expand Down
7 changes: 5 additions & 2 deletions src/app/components/eval-tab/eval-tab.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -297,8 +297,11 @@ <h4 style="margin-bottom: 8px;">Past Runs</h4>
<div class="status-card__metrics">
@for (evalMetric of currentHistoryMetrics(); track evalMetric) {
<span class="status-card__metric">
<span class="status-card__metric-name" [matTooltip]="evalMetric.metricName">{{ evalMetric.metricName | formatMetricName }}</span>:
<span class="status-card__metric-value">{{ evalMetric.threshold | number:'1.2-2' }}</span>
<!-- This row lists the threshold each metric ran with.
A metric that has none is named on its own, rather
than trailing a colon with nothing after it. -->
<span class="status-card__metric-name" [matTooltip]="evalMetric.metricName">{{ evalMetric.metricName | formatMetricName }}</span>@if (evalMetric.threshold != null) {<span>:
<span class="status-card__metric-value">{{ evalMetric.threshold | number:'1.2-2' }}</span></span>}
</span>
}
</div>
Expand Down
51 changes: 51 additions & 0 deletions src/app/components/eval-tab/eval-tab.component.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -331,4 +331,55 @@ describe('EvalTabComponent', () => {
expect(evalService.getEvalSet).toHaveBeenCalledWith('my-app', 'my-set');
});
});

describe('getMetricsScore', () => {
// EvalStatus: 1 PASSED, 2 FAILED, 3 NOT_EVALUATED, 4 INFORMATIONAL.
it('counts only metrics that reach a verdict', () => {
const evalRes = {
evalMetricResults: [
{metricName: 'tool_trajectory_avg_score', evalStatus: 1},
{metricName: 'response_match_score', evalStatus: 2},
{metricName: 'safety_v1', evalStatus: 3},
],
};

expect((component as any).getMetricsScore(evalRes)).toBe('1/2');
});

it('excludes informational metrics from the ratio', () => {
// The efficiency metrics are reported on every run and never pass or
// fail. Counting them would show a fully passing case as 1/4.
const evalRes = {
evalMetricResults: [
{metricName: 'tool_trajectory_avg_score', evalStatus: 1},
{metricName: 'tool_call_count_v1', evalStatus: 4},
{metricName: 'inference_call_count_v1', evalStatus: 4},
{metricName: 'token_usage_v1', evalStatus: 4},
],
};

expect((component as any).getMetricsScore(evalRes)).toBe('1/1');
});

it('excludes informational metrics tallied per invocation', () => {
const evalRes = {
evalMetricResultPerInvocation: [
{
evalMetricResults: [
{metricName: 'tool_trajectory_avg_score', evalStatus: 1},
{metricName: 'token_usage_v1', evalStatus: 4},
],
},
{
evalMetricResults: [
{metricName: 'tool_trajectory_avg_score', evalStatus: 2},
{metricName: 'token_usage_v1', evalStatus: 4},
],
},
],
};

expect((component as any).getMetricsScore(evalRes)).toBe('1/2');
});
});
});
10 changes: 8 additions & 2 deletions src/app/components/eval-tab/eval-tab.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -579,10 +579,16 @@ export class EvalTabComponent implements OnInit, OnChanges {
let passed = 0;
let total = 0;

// Excludes NOT_EVALUATED so the ratio reads "passed / evaluated".
// Counts only metrics that reach a verdict, so the ratio reads
// "passed / judged". NOT_EVALUATED produced nothing to judge, and
// INFORMATIONAL reports a value without ever passing or failing; counting
// either one would make a fully passing case read as a partial one.
const tally = (results: any[]) => {
for (const r of results) {
if (r.evalStatus === EvalStatus.NOT_EVALUATED) continue;
if (r.evalStatus === EvalStatus.NOT_EVALUATED ||
r.evalStatus === EvalStatus.INFORMATIONAL) {
continue;
}
total += 1;
if (r.evalStatus === EvalStatus.PASSED) passed += 1;
}
Expand Down
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ <h2 mat-dialog-title class="dialog-title">Run evaluation</h2>
<div style="display: flex; flex-direction: column; align-items: flex-start;">
<span class="slider-label" style="margin-right: 0; font-size: 11px; color: var(--mat-sys-on-surface-variant);">Threshold</span>
<div style="display: flex; align-items: center;">
<mat-slider [min]="metric.metricValueInfo.interval.minValue" [max]="metric.metricValueInfo.interval.maxValue" step="0.1" thumbLabel class="threshold-slider">
<!-- The interval is optional on the wire. A metric that arrives without
one still needs a threshold, so the slider falls back to 0..1 rather
than binding undefined bounds. -->
<mat-slider [min]="metric.metricValueInfo?.interval?.minValue ?? 0" [max]="metric.metricValueInfo?.interval?.maxValue ?? 1" step="0.1" thumbLabel class="threshold-slider">
<input matSliderThumb [formControlName]="metric.metricName + '_threshold'" />
</mat-slider>
<span class="threshold-value">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,98 @@ describe('RunEvalConfigDialogComponent', () => {
});
});

describe('with metrics that need no threshold', () => {
// An informational metric reports a value and never passes or fails, so
// the backend sends it with requiresThreshold false and no value interval.
const INFORMATIONAL_METRIC = {
metricName: 'token_usage_v1',
description: 'Tokens consumed',
metricValueInfo: {},
requiresThreshold: false,
};

it('does not offer them for selection', async () => {
const {component} = await createComponent({
evalMetrics: [],
metricsInfo: [...METRICS_INFO, INFORMATIONAL_METRIC],
});

expect(component.metricsInfo.map((m) => m.metricName)).toEqual([
'tool_trajectory_avg_score',
'response_match_score',
]);
expect(component.evalForm.get('token_usage_v1_selected')).toBeNull();
expect(component.evalForm.get('token_usage_v1_threshold')).toBeNull();
});

it('never emits them, so the backend is not sent a threshold it rejects',
async () => {
const {component, dialogRef} = await createComponent({
evalMetrics: [],
metricsInfo: [...METRICS_INFO, INFORMATIONAL_METRIC],
});
component.evalForm.get('tool_trajectory_avg_score_selected')
?.setValue(true);

component.onStart();

const arg = dialogRef.close.calls.mostRecent().args[0] as any;
expect(arg.metrics.map((m: any) => m.metricName)).toEqual([
'tool_trajectory_avg_score',
]);
});
});

describe('with a metric whose interval the backend omitted', () => {
// `interval` is optional on the wire. A metric can still require a
// threshold without carrying one, and the dialog has to offer it.
const UNBOUNDED_METRIC = {
metricName: 'custom_unbounded_v1',
description: 'Requires a threshold but carries no value interval',
metricValueInfo: {},
requiresThreshold: true,
};

it('still offers it, and renders the threshold slider', async () => {
// Regression test: the slider read `metricValueInfo.interval.minValue`
// unconditionally, so this metric took the whole dialog down. The
// assertions below only hold if the metric survives the filter, which is
// what puts the slider on the page in the first place.
const {fixture, component} = await createComponent({
evalMetrics: [],
metricsInfo: [UNBOUNDED_METRIC],
});

expect(component.metricsInfo.map((m) => m.metricName)).toEqual([
'custom_unbounded_v1',
]);
expect(fixture.nativeElement.querySelector('mat-slider')).not.toBeNull();
});

it('bounds the slider at 0..1 and validates only that a value is set',
async () => {
const {component} = await createComponent({
evalMetrics: [],
metricsInfo: [UNBOUNDED_METRIC],
});

const threshold =
component.evalForm.get('custom_unbounded_v1_threshold');
expect(threshold).not.toBeNull();
// getDefaultThreshold falls back to the slider's upper bound.
expect(threshold!.value).toBe(1.0);
expect(threshold!.valid).toBeTrue();

// No interval means no bounds to enforce, so a value outside 0..1 is
// still accepted by the form; only `required` applies.
threshold!.setValue(5);
expect(threshold!.valid).toBeTrue();

threshold!.setValue(null);
expect(threshold!.valid).toBeFalse();
});
});

describe('without metricsInfo (fallback)', () => {
let component: RunEvalConfigDialogComponent;
let dialogRef: jasmine.SpyObj<MatDialogRef<RunEvalConfigDialogComponent>>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,20 @@ export class RunEvalConfigDialogComponent {
private fb: FormBuilder,
@Inject(MAT_DIALOG_DATA) public data: EvalConfigData) {
this.evalMetrics = this.data.evalMetrics || [];
this.metricsInfo = this.data.metricsInfo || [];
// This dialog asks the user to select metrics and set a threshold for
// each. A metric that needs no threshold has nothing to configure here, so
// it is not offered. Those metrics are always on and are reported without
// the user selecting them.
//
// Dropping them also keeps `collectMetrics` from emitting one: every metric
// it emits carries a threshold, and a metric that takes none rejects the
// request outright.
//
// Not showing them at all is interim. The end state the server's
// metrics-info filter is waiting on is a separate always-on, non-selectable
// section listing them; whoever removes that filter has to add it here.
this.metricsInfo = (this.data.metricsInfo || [])
.filter((metric) => metric.requiresThreshold !== false);

this.runForm = this.fb.group({
runMode: [DEFAULT_RUN_MODE],
Expand All @@ -132,12 +145,17 @@ export class RunEvalConfigDialogComponent {

this.evalForm.addControl(`${metric.metricName}_selected`, this.fb.control(isSelected));

const interval = metric.metricValueInfo.interval;
this.evalForm.addControl(`${metric.metricName}_threshold`, this.fb.control(threshold, [
Validators.required,
Validators.min(interval.minValue),
Validators.max(interval.maxValue)
]));
// A metric that needs a threshold should carry an interval to bound it,
// but the field is optional on the wire. Without one there is nothing to
// validate against, so `required` is the only constraint we can apply.
const interval = metric.metricValueInfo?.interval;
const validators = [Validators.required];
if (interval) {
validators.push(
Validators.min(interval.minValue), Validators.max(interval.maxValue));
}
this.evalForm.addControl(
`${metric.metricName}_threshold`, this.fb.control(threshold, validators));
});

// Fallback if metricsInfo is empty, add the hardcoded ones to avoid empty UI if backend fails
Expand Down Expand Up @@ -169,7 +187,10 @@ export class RunEvalConfigDialogComponent {
private getDefaultThreshold(metric: MetricsInfo): number {
if (metric.metricName === 'tool_trajectory_avg_score') return 1.0;
if (metric.metricName === 'response_match_score') return 0.7;
return metric.metricValueInfo.interval.maxValue;
// Default to the top of the metric's range, which for a score metric means
// "must be perfect". A metric whose interval the backend omitted falls back
// to 1.0, matching the bounds the slider defaults to.
return metric.metricValueInfo?.interval?.maxValue ?? 1.0;
}

onReset(): void {
Expand Down
11 changes: 10 additions & 1 deletion src/app/core/models/Eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ export const DEFAULT_EVAL_METRICS: EvalMetric[] = [
];

export declare interface MetricValueInfo {
interval: {
// Absent for metrics whose value is unbounded, such as the informational
// efficiency metrics (token and call counts).
interval?: {
minValue: number;
openAtMin: boolean;
maxValue: number;
Expand All @@ -49,6 +51,9 @@ export declare interface MetricsInfo {
metricName: string;
description: string;
metricValueInfo: MetricValueInfo;
// False for metrics that report a value without passing or failing. Such a
// metric has no threshold to configure and no interval to bound one by.
requiresThreshold?: boolean;
}

export declare interface Invocation {
Expand Down Expand Up @@ -91,6 +96,10 @@ export enum EvalStatus {
PASSED = 1,
FAILED = 2,
NOT_EVALUATED = 3,
// The metric reported a value but reached no verdict, so it is neither a
// pass nor a failure. Distinct from NOT_EVALUATED, which means the metric
// produced nothing at all.
INFORMATIONAL = 4,
}

export declare interface EvaluationResult {
Expand Down
Loading
0