Repository navigation
Expand file tree
/
Copy pathmetadata.ts
More file actions
422 lines (367 loc) · 13.4 KB
/
Copy pathmetadata.ts
File metadata and controls
422 lines (367 loc) · 13.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
/**
* ZDR Metadata Tracker
*
* Always-on, privacy-first analytics for G0DM0D3.
* Records EVERYTHING about requests — except the actual content.
*
* What IS tracked:
* - Timestamps, endpoints, modes, tiers
* - Models used, winner model, scores, latencies
* - Pipeline flags (godmode, autotune, parseltongue, stm)
* - AutoTune detected context type + confidence
* - Per-model success/failure, response lengths, durations
* - Error types (categorized, never raw error messages)
*
* What is NEVER tracked:
* - Message content, system prompts, responses
* - API keys, auth tokens, IP addresses
* - Any PII whatsoever
*
* Storage: in-memory ring buffer with auto-publish to HuggingFace.
* When the buffer hits 80% capacity, it flushes to HF as JSONL.
* On graceful shutdown, remaining events are flushed.
* If HF publishing is not configured, falls back to FIFO eviction.
*/
import { randomUUID } from 'crypto'
import { registerMetadataStore, checkMetadataThreshold } from './hf-publisher'
// ── Types ────────────────────────────────────────────────────────────
export interface MetadataEvent {
id: string
timestamp: number
// Request shape
endpoint: string
mode: 'standard' | 'ultraplinian' | 'consortium'
tier?: string // fast | standard | full (ultraplinian only)
stream: boolean
// Pipeline config (what was enabled, not what it produced)
pipeline: {
godmode: boolean
autotune: boolean
parseltongue: boolean
stm_modules: string[]
strategy?: string
}
// AutoTune metadata (context detection only, no content)
autotune?: {
detected_context: string
confidence: number
}
// Model metadata
model?: string // single-model mode
models_queried?: number // ultraplinian
models_succeeded?: number
// Per-model results (no content, just numbers)
model_results?: Array<{
model: string
score: number
duration_ms: number
success: boolean
content_length: number
error_type?: string // categorized: 'timeout' | 'rate_limit' | 'auth' | 'model_error' | 'empty' | 'unknown'
}>
// Winner metadata
winner?: {
model: string
score: number
duration_ms: number
content_length: number
}
// Response metadata
total_duration_ms: number
response_length: number
// Liquid Response metadata (streaming upgrades)
liquid?: {
upgrades: number // how many leader changes happened
first_response_ms: number // time to first leader
}
}
// ── In-Memory Ring Buffer ────────────────────────────────────────────
let events: MetadataEvent[] = []
const MAX_EVENTS = 50000 // ~50k events before eviction
// Track how many events have been flushed so we can use index-based draining
// instead of copying the entire array on each snapshot.
let metadataFlushIndex = 0
// Register with HF publisher so it can snapshot/clear our buffer
registerMetadataStore({
snapshot: () => events.slice(metadataFlushIndex),
clear: (count: number) => {
metadataFlushIndex += count
// Compact the array when more than half has been drained to free memory
if (metadataFlushIndex > events.length / 2) {
events = events.slice(metadataFlushIndex)
metadataFlushIndex = 0
}
},
})
// ── Recording ────────────────────────────────────────────────────────
export function recordEvent(event: Omit<MetadataEvent, 'id' | 'timestamp'>): string {
const id = randomUUID()
const record: MetadataEvent = {
...event,
id,
timestamp: Date.now(),
}
events.push(record)
// Auto-flush to HF when approaching capacity (async, non-blocking)
checkMetadataThreshold(events.length, MAX_EVENTS)
// Evict oldest if over cap (fallback if HF publish is not configured or failed)
if (events.length > MAX_EVENTS) {
events = events.slice(events.length - MAX_EVENTS)
}
return id
}
// ── Error Categorization ─────────────────────────────────────────────
export function categorizeError(error?: string): string | undefined {
if (!error) return undefined
const lower = error.toLowerCase()
if (lower.includes('timeout') || lower.includes('abort')) return 'timeout'
if (lower.includes('rate') || lower.includes('429')) return 'rate_limit'
if (lower.includes('auth') || lower.includes('401') || lower.includes('403')) return 'auth'
if (lower.includes('empty')) return 'empty'
if (lower.includes('race ended')) return 'early_exit'
if (lower.includes('api error') || lower.includes('http')) return 'model_error'
return 'unknown'
}
// ── Query: Raw Events ────────────────────────────────────────────────
export function getEvents(opts?: {
limit?: number
offset?: number
since?: number // unix ms timestamp
mode?: 'standard' | 'ultraplinian'
}): { events: MetadataEvent[]; total: number } {
let filtered = events
if (opts?.since) {
filtered = filtered.filter(e => e.timestamp >= opts.since!)
}
if (opts?.mode) {
filtered = filtered.filter(e => e.mode === opts.mode)
}
const total = filtered.length
const offset = opts?.offset ?? 0
const limit = opts?.limit ?? 100
return {
events: filtered.slice(offset, offset + limit),
total,
}
}
// ── Query: Aggregated Stats ──────────────────────────────────────────
export interface MetadataStats {
total_requests: number
since: number | null // oldest event timestamp
uptime_ms: number
// Breakdown by mode
by_mode: Record<string, number>
by_tier: Record<string, number>
by_endpoint: Record<string, number>
// Model usage
models: {
total_queries: number // sum of all individual model calls
unique_models: number
by_model: Record<string, {
queries: number
wins: number
avg_score: number
avg_duration_ms: number
success_rate: number
}>
}
// Pipeline usage
pipeline: {
godmode_rate: number // % of requests with godmode enabled
autotune_rate: number
parseltongue_rate: number
stm_usage: Record<string, number> // module -> count
strategy_usage: Record<string, number>
}
// Context detection (from autotune)
contexts: Record<string, number>
// Latency
latency: {
avg_ms: number
p50_ms: number
p95_ms: number
p99_ms: number
}
// Response sizes
response_lengths: {
avg: number
p50: number
p95: number
}
// Streaming
streaming: {
stream_rate: number // % of requests that used streaming
avg_upgrades: number // avg leader changes per stream
avg_first_response_ms: number // avg time to first leader
}
// Errors
errors: {
total_model_errors: number
by_type: Record<string, number>
model_success_rate: number // across all individual model queries
}
}
export function getStats(): MetadataStats {
const total = events.length
if (total === 0) {
return {
total_requests: 0,
since: null,
uptime_ms: 0,
by_mode: {},
by_tier: {},
by_endpoint: {},
models: { total_queries: 0, unique_models: 0, by_model: {} },
pipeline: { godmode_rate: 0, autotune_rate: 0, parseltongue_rate: 0, stm_usage: {}, strategy_usage: {} },
contexts: {},
latency: { avg_ms: 0, p50_ms: 0, p95_ms: 0, p99_ms: 0 },
response_lengths: { avg: 0, p50: 0, p95: 0 },
streaming: { stream_rate: 0, avg_upgrades: 0, avg_first_response_ms: 0 },
errors: { total_model_errors: 0, by_type: {}, model_success_rate: 0 },
}
}
// Breakdowns
const byMode: Record<string, number> = {}
const byTier: Record<string, number> = {}
const byEndpoint: Record<string, number> = {}
const contexts: Record<string, number> = {}
const stmUsage: Record<string, number> = {}
const strategyUsage: Record<string, number> = {}
const errorTypes: Record<string, number> = {}
// Model tracking
const modelData: Record<string, { queries: number; wins: number; scores: number[]; durations: number[]; successes: number }> = {}
// Accumulators
let godmodeCount = 0
let autotuneCount = 0
let parseltongueCount = 0
let totalModelQueries = 0
let totalModelErrors = 0
let totalModelSuccesses = 0
const durations: number[] = []
const responseLengths: number[] = []
let streamCount = 0
let upgradeSum = 0
let firstResponseSum = 0
let streamWithLiquidCount = 0
for (const e of events) {
byMode[e.mode] = (byMode[e.mode] || 0) + 1
if (e.tier) byTier[e.tier] = (byTier[e.tier] || 0) + 1
byEndpoint[e.endpoint] = (byEndpoint[e.endpoint] || 0) + 1
if (e.pipeline.godmode) godmodeCount++
if (e.pipeline.autotune) autotuneCount++
if (e.pipeline.parseltongue) parseltongueCount++
if (e.pipeline.strategy) {
strategyUsage[e.pipeline.strategy] = (strategyUsage[e.pipeline.strategy] || 0) + 1
}
for (const mod of e.pipeline.stm_modules) {
stmUsage[mod] = (stmUsage[mod] || 0) + 1
}
if (e.autotune?.detected_context) {
const ctx = e.autotune.detected_context
contexts[ctx] = (contexts[ctx] || 0) + 1
}
durations.push(e.total_duration_ms)
responseLengths.push(e.response_length)
if (e.stream) streamCount++
if (e.liquid) {
upgradeSum += e.liquid.upgrades
firstResponseSum += e.liquid.first_response_ms
streamWithLiquidCount++
}
// Per-model results
if (e.model_results) {
for (const mr of e.model_results) {
totalModelQueries++
if (!modelData[mr.model]) {
modelData[mr.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
}
const md = modelData[mr.model]
md.queries++
if (mr.success) {
md.successes++
totalModelSuccesses++
md.scores.push(mr.score)
md.durations.push(mr.duration_ms)
} else {
totalModelErrors++
if (mr.error_type) {
errorTypes[mr.error_type] = (errorTypes[mr.error_type] || 0) + 1
}
}
}
}
// Winner tracking
if (e.winner) {
if (!modelData[e.winner.model]) {
modelData[e.winner.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
}
modelData[e.winner.model].wins++
}
}
// Build per-model stats
const byModel: Record<string, { queries: number; wins: number; avg_score: number; avg_duration_ms: number; success_rate: number }> = {}
for (const [model, data] of Object.entries(modelData)) {
byModel[model] = {
queries: data.queries,
wins: data.wins,
avg_score: data.scores.length > 0 ? Math.round(data.scores.reduce((a, b) => a + b, 0) / data.scores.length) : 0,
avg_duration_ms: data.durations.length > 0 ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length) : 0,
success_rate: data.queries > 0 ? Math.round((data.successes / data.queries) * 100) / 100 : 0,
}
}
return {
total_requests: total,
since: events[0].timestamp,
uptime_ms: Date.now() - events[0].timestamp,
by_mode: byMode,
by_tier: byTier,
by_endpoint: byEndpoint,
models: {
total_queries: totalModelQueries,
unique_models: Object.keys(modelData).length,
by_model: byModel,
},
pipeline: {
godmode_rate: Math.round((godmodeCount / total) * 100) / 100,
autotune_rate: Math.round((autotuneCount / total) * 100) / 100,
parseltongue_rate: Math.round((parseltongueCount / total) * 100) / 100,
stm_usage: stmUsage,
strategy_usage: strategyUsage,
},
contexts,
latency: (() => {
const sorted = durations.sort((a, b) => a - b)
return {
avg_ms: sorted.length > 0 ? Math.round(sorted.reduce((a, b) => a + b, 0) / sorted.length) : 0,
p50_ms: percentileFromSorted(sorted, 50),
p95_ms: percentileFromSorted(sorted, 95),
p99_ms: percentileFromSorted(sorted, 99),
}
})(),
response_lengths: (() => {
const sorted = responseLengths.sort((a, b) => a - b)
return {
avg: sorted.length > 0 ? Math.round(sorted.reduce((a, b) => a + b, 0) / sorted.length) : 0,
p50: percentileFromSorted(sorted, 50),
p95: percentileFromSorted(sorted, 95),
}
})(),
streaming: {
stream_rate: Math.round((streamCount / total) * 100) / 100,
avg_upgrades: streamWithLiquidCount > 0 ? Math.round((upgradeSum / streamWithLiquidCount) * 100) / 100 : 0,
avg_first_response_ms: streamWithLiquidCount > 0 ? Math.round(firstResponseSum / streamWithLiquidCount) : 0,
},
errors: {
total_model_errors: totalModelErrors,
by_type: errorTypes,
model_success_rate: totalModelQueries > 0 ? Math.round((totalModelSuccesses / totalModelQueries) * 100) / 100 : 0,
},
}
}
// ── Helpers ──────────────────────────────────────────────────────────
/** Compute a single percentile from an already-sorted array. */
function percentileFromSorted(sorted: number[], p: number): number {
if (sorted.length === 0) return 0
const idx = Math.ceil((p / 100) * sorted.length) - 1
return sorted[Math.max(0, idx)]
}