Repository navigation
Expand file tree
/
Copy patheval_utils.js
More file actions
136 lines (122 loc) · 3.86 KB
/
Copy patheval_utils.js
File metadata and controls
136 lines (122 loc) · 3.86 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import fs from 'node:fs';
import path from 'node:path';
import { execSync } from 'node:child_process';
import os from 'node:os';
/**
* Finds all report.json files recursively in a directory.
*/
export function findReports(dir) {
const reports = [];
if (!fs.existsSync(dir)) return reports;
const files = fs.readdirSync(dir);
for (const file of files) {
const fullPath = path.join(dir, file);
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
reports.push(...findReports(fullPath));
} else if (file === 'report.json') {
reports.push(fullPath);
}
}
return reports;
}
/**
* Extracts the model name from the artifact path.
*/
export function getModelFromPath(reportPath) {
const parts = reportPath.split(path.sep);
// Look for the directory that follows the 'eval-logs-' pattern
const artifactDir = parts.find((p) => p.startsWith('eval-logs-'));
if (!artifactDir) return 'unknown';
const match = artifactDir.match(/^eval-logs-(.+)-(\d+)$/);
return match ? match[1] : 'unknown';
}
/**
* Escapes special characters in a string for use in a regular expression.
*/
export function escapeRegex(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
* Aggregates stats from a list of report.json files.
* @returns {Record<string, Record<string, {passed: number, total: number, file?: string}>>} statsByModel
*/
export function getStatsFromReports(reports) {
const statsByModel = {};
for (const reportPath of reports) {
try {
const model = getModelFromPath(reportPath);
if (!statsByModel[model]) {
statsByModel[model] = {};
}
const testStats = statsByModel[model];
const content = fs.readFileSync(reportPath, 'utf-8');
const json = JSON.parse(content);
for (const testResult of json.testResults) {
const filePath = testResult.name;
for (const assertion of testResult.assertionResults) {
const name = assertion.title;
if (!testStats[name]) {
testStats[name] = { passed: 0, total: 0, file: filePath };
}
testStats[name].total++;
if (assertion.status === 'passed') {
testStats[name].passed++;
}
}
}
} catch (error) {
console.error(`Error processing report at ${reportPath}:`, error.message);
}
}
return statsByModel;
}
/**
* Fetches historical nightly data using the GitHub CLI.
* @returns {Array<{runId: string, stats: Record<string, any>}>} history
*/
export function fetchNightlyHistory(lookbackCount) {
const history = [];
try {
const cmd = `gh run list --workflow evals-nightly.yml --branch main --limit ${
lookbackCount + 2
} --json databaseId,status`;
const runsJson = execSync(cmd, { encoding: 'utf-8' });
let runs = JSON.parse(runsJson);
// Filter for completed runs and take the top N
runs = runs.filter((r) => r.status === 'completed').slice(0, lookbackCount);
for (const run of runs) {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), `gemini-evals-hist-${run.databaseId}-`),
);
try {
execSync(
`gh run download ${run.databaseId} -p "eval-logs-*" -D "${tmpDir}"`,
{ stdio: 'ignore' },
);
const runReports = findReports(tmpDir);
if (runReports.length > 0) {
history.push({
runId: run.databaseId,
stats: getStatsFromReports(runReports),
});
}
} catch (error) {
console.error(
`Failed to process artifacts for run ${run.databaseId}:`,
error.message,
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}
} catch (error) {
console.error('Failed to fetch history:', error.message);
}
return history;
}