Repository navigation
Expand file tree
/
Copy pathllm-judge.ts
More file actions
114 lines (101 loc) · 3.28 KB
/
Copy pathllm-judge.ts
File metadata and controls
114 lines (101 loc) · 3.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { LlmRole, type BaseLlmClient } from '@google/gemini-cli-core';
export interface JudgeOptions {
/**
* The number of parallel generations to run for majority voting.
* Defaults to 1. Use 3 or 5 for self-consistency.
*/
selfConsistencyRuns?: number;
/**
* The model to use for judging. Defaults to gemini-3-flash-base.
*/
model?: string;
}
export interface JudgeResult {
verdict: boolean;
reasoning: string[];
votes: { yes: number; no: number; other: number };
}
/**
* A reusable LLM-as-a-judge utility for behavioral evaluations.
*/
export class LLMJudge {
constructor(private readonly llmClient: BaseLlmClient) {}
/**
* Asks the LLM a Yes/No question and returns a boolean verdict.
* If selfConsistencyRuns > 1, it runs in parallel and returns the majority vote.
*/
async judgeYesNo(
question: string,
options: JudgeOptions = {},
): Promise<JudgeResult> {
const runs = options.selfConsistencyRuns ?? 1;
const model = options.model ?? 'gemini-3-flash-base';
const systemPrompt = `You are a strict, impartial expert judge. Read the provided evidence and question carefully. You MUST answer the question with ONLY "YES" or "NO". Do not provide any conversational filler or explanation before your answer.`;
const generateCall = async (): Promise<string> => {
try {
const response = await this.llmClient.generateContent({
modelConfigKey: { model },
contents: [{ role: 'user', parts: [{ text: question }] }],
systemInstruction: {
role: 'system',
parts: [{ text: systemPrompt }],
},
promptId: 'llm-judge-eval',
role: LlmRole.UTILITY_TOOL,
abortSignal: new AbortController().signal,
});
const text =
response.candidates?.[0]?.content?.parts?.[0]?.text
?.trim()
?.toUpperCase() || 'ERROR';
return text;
} catch (e: any) {
return `ERROR: ${e.message}`;
}
};
const promises = Array.from({ length: runs }).map(() => generateCall());
const rawResults = await Promise.all(promises);
let yes = 0;
let no = 0;
let other = 0;
for (const res of rawResults) {
// Remove any punctuation the model might have appended
const cleanRes = res.replace(/[^A-Z ]/g, '');
if (
cleanRes.includes('THE ANSWER IS YES') ||
cleanRes.includes('ANSWER IS YES') ||
cleanRes.endsWith('YES')
) {
yes++;
} else if (
cleanRes.includes('THE ANSWER IS NO') ||
cleanRes.includes('ANSWER IS NO') ||
cleanRes.endsWith('NO')
) {
no++;
} else if (cleanRes.trim() === 'YES') {
yes++;
} else if (cleanRes.trim() === 'NO') {
no++;
} else {
// Fallback: look for YES or NO as standalone words or at the end
const words = cleanRes.split(/\s+/);
if (words.includes('YES')) yes++;
else if (words.includes('NO')) no++;
else other++;
}
}
// Pass if YES > NO and YES > OTHER (plurality)
const pass = yes > no && yes > other;
return {
verdict: pass,
reasoning: rawResults,
votes: { yes, no, other },
};
}
}