8000 evals: an empty Langfuse page no longer ends a trace fetch by Jiminyang1 · Pull Request #11 · Jiminyang1/minibot · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 68 additions & 27 deletions evals/promote.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

from __future__ import annotations

from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import UTC, date, datetime
import json
Expand All @@ -46,6 +47,10 @@
_ISO_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})(?:[T ](\d{1,2}):(\d{2}))?")
_MAX_CONTAINS_CHARS = 40
_NEVER = datetime.min.replace(tzinfo=UTC)
# The server scans for up to 45 s per page before it answers.
_PAGE_TIMEOUT_S = 60
# ``traces`` is an interactive listing: bound how long it can page.
_MAX_LISTING_PAGES = 10


@dataclass
Expand Down Expand Up @@ -196,50 +201,86 @@ def append_case(path: Path, case: dict[str, Any]) -> None:
# ── fetching from Langfuse ────────────────────────────────────────


def fetch_observations(langfuse: Any, trace_id: str) -> list[Any]:
"""Every observation of one trace, via the v2 observations API."""
observations: list[Any] = []
def observation_pages(
langfuse: Any, *, max_pages: int | None = None, **query: Any
) -> Iterator[list[Any]]:
"""Each page of a v2 observations query, until the server sends no cursor.

A page can be short or empty while more rows remain: the server stops a
scan when its time budget (about 45 s) runs out and still returns a
cursor. So only a missing cursor ends the query. A cursor the server
already sent also ends it, so a misbehaving server cannot loop forever.
The SDK's default timeout (5 s) is shorter than one scan, so every
request waits longer.
"""
cursor = None
while True:
seen: set[str] = set()
pages = 0
while max_pages is None or pages < max_pages:
page = langfuse.api.observations.get_many(
trace_id=trace_id,
fields="core,basic,io,metadata",
limit=1000,
**query,
< 10000 span class='blob-code-inner blob-code-marker ' data-code-marker=" "> cursor=cursor,
request_options={"timeout_in_seconds": _PAGE_TIMEOUT_S},
)
observations.extend(page.data)
pages += 1
yield list(page.data or [])
cursor = getattr(page.meta, "cursor", None)
if not cursor or not page.data:
return observations
if not cursor or cursor in seen:
return
seen.add(cursor)


def fetch_observations(langfuse: Any, trace_id: str) -> list[Any]:
"""Every observation of one trace, via the v2 observations API."""
pages = observation_pages(
langfuse, trace_id=trace_id, fields="core,basic,io,metadata", limit=1000
)
return _unique(obs for page in pages for obs in page)


def list_traces(langfuse: Any, *, limit: int = 20, failed: bool = False) -> list[dict[str, Any]]:
"""Recent MiniBot turns, newest first; with *failed*, only problem runs."""
rows: list[dict[str, Any]] = []
cursor = None
for _ in range(10): # at most 10 pages when filtering for failures
page = langfuse.api.observations.get_many(
name=TRACE_NAME,
type="AGENT",
fields="core,basic,io,trace_context",
limit=min(max(limit, 1) * (3 if failed else 1), 100),
cursor=cursor,
agents = observation_pages(
langfuse,
max_pages=_MAX_LISTING_PAGES,
name=TRACE_NAME,
type="AGENT",
fields="core,basic,io,trace_context",
limit=min(max(limit, 1) * (3 if failed else 1), 100),
)
seen: set[Any] = set()
for page in agents:
page = [o for o in page if _attr(o, "id") not in seen]
if not page:
continue
seen.update(_attr(o, "id") for o in page)
oldest = min(_start(o) for o in page)
tool_pages = observation_pages(
langfuse, type="TOOL", level="ERROR", from_start_time=oldest, fields="core", limit=1000
)
if not page.data:
break
oldest = min(_start(o) for o in page.data)
tool_errors = langfuse.api.observations.get_many(
type="TOOL", level="ERROR", from_start_time=oldest, fields="core", limit=1000
).data
rows.extend(summarize_traces(page.data, tool_errors))
tool_errors = _unique(obs for tools in tool_pages for obs in tools)
rows.extend(summarize_traces(page, tool_errors))
if failed:
rows = [row for row in rows if row["run_failed"] or row["failed_tools"]]
cursor = getattr(page.meta, "cursor", None)
if len(rows) >= limit or not cursor or not failed:
if len(rows) >= limit:
break
return rows[:limit]


def _unique(observations: Iterable[Any]) -> list[Any]:
"""Drop rows the server repeated across pages (it does not dedupe)."""
seen: set[Any] = set()
unique = []
for obs in observations:
key = _attr(obs, "id")
if key is not None and key in seen:
continue
seen.add(key)
unique.append(obs)
return unique


def summarize_traces(agents: list[Any], tool_errors: list[Any]) -> list[dict[str, Any]]:
"""One row per agent observation: id, time, source, input, failure flags."""
failed_tools: dict[str, int] = {}
Expand Down
104 changes: 103 additions & 1 deletion tests/test_evals_promote.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,13 @@
from minibot.config import Config
from minibot.evals import __main__ as cli
from minibot.evals import sandbox
from minibot.evals.promote import draft_case, format_traces, summarize_traces
from minibot.evals.promote import (
draft_case,
fetch_observations,
format_traces,
list_traces,
summarize_traces,
)
from minibot.evals.runner import load_cases
from minibot.evals.sandbox import run_case
from minibot.evals.scoring import score_deterministic
Expand Down Expand Up @@ -311,6 +317,102 @@ def test_summarize_flags_failed_runs_and_failed_tools(self) -> None:
self.assertIn("…", text)


def _page(data, cursor=None):
return SimpleNamespace(data=list(data), meta=SimpleNamespace(cursor=cursor))


class _PagedObservations:
"""A fake ``api.observations.get_many`` that serves scripted pages.

Pages are keyed by the query's ``type`` and served in order, one per
call. Since Langfuse's adaptive scan, an empty page can still carry a
cursor: it means "the time budget ran out, keep going", not "done".
"""

def __init__(self, pages: dict[str | None, list[SimpleNamespace]]) -> None:
self.pages = {key: list(value) for key, value in pages.items()}
self.calls: list[dict] = []

def get_many(self, **kwargs):
self.calls.append(kwargs)
if len(self.calls) > 50:
raise AssertionError("paging did not stop")
queue = self.pages.get(kwargs.get("type"))
return queue.pop(0) if queue else _page([])

def client(self):
return SimpleNamespace(api=SimpleNamespace(observations=self))


def _observation(obs_id, trace_id="t1", minutes=0, **extra):
return SimpleNamespace(
id=obs_id, trace_id=trace_id, start_time=T0 + timedelta(minutes=minutes), **extra
)


class ObservationPagingTests(unittest.TestCase):
def test_fetch_follows_the_cursor_past_an_empty_page(self) -> None:
wanted = _observation("o1")
fake = _PagedObservations({None: [_page([], "c1"), _page([wanted])]})

observations = fetch_observations(fake.client(), "t1")

self.assertEqual(observations, [wanted])
self.assertEqual(fake.calls[1]["cursor"], "c1")

def test_fetch_drops_rows_repeated_across_pages(self) -> None:
first, second = _observation("o1"), _observation("o2")
fake = _PagedObservations({None: [_page([first], "c1"), _page([first, second])]})

observations = fetch_observations(fake.client(), "t1")

self.assertEqual([o.id for o in observations], ["o1", "o2"])

def test_a_repeated_cursor_ends_paging(self) -> None:
stuck = [_page([], "same") for _ in range(10)]
fake = _PagedObservations({None: stuck})

self.assertEqual(fetch_observations(fake.client(), "t1"), [])
self.assertEqual(len(fake.calls), 2)

def test_every_request_waits_longer_than_the_server_scan(self) -> None:
fake = _PagedObservations({None: [_page([_observation("o1")])]})

fetch_observations(fake.client(), "t1")

# The server may scan for up to 45 s; the SDK's default is 5 s.
timeout = fake.calls[0]["request_options"]["timeout_in_seconds"]
self.assertGreater(timeout, 45)

def test_listing_reads_past_empty_agent_and_tool_pages(self) -> None:
agent = _observation("a1", trace_id="t1", level="DEFAULT", tags=["tui"], input="你好")
tool_error = _observation("x1", trace_id="t1")
fake = _PagedObservations(
{
"AGENT": [_page([], "c1"), _page([agent])],
"TOOL": [_page([], "d1"), _page([tool_error])],
}
)

rows = list_traces(fake.client(), limit=5)

self.assertEqual([r["trace_id"] for r in rows], ["t1"])
self.assertEqual(rows[0]["failed_tools"], 1)

def test_listing_fills_the_limit_from_short_pages(self) -> None:
def agent(n):
return _observation(f"a{n}", trace_id=f"t{n}", minutes=-n, level="DEFAULT",
tags=["cli"], input="q")

fake = _PagedObservations(
{"AGENT": [_page([agent(1)], "c1"), _page([agent(2)], "c2"), _page([agent(3)])]}
)

rows = list_traces(fake.client(), limit=2)

self.assertEqual([r["trace_id"] for r in rows], ["t1", "t2"])


class PromoteCommandTests(unittest.TestCase):
def test_promote_appends_to_the_given_cases_file_and_syncs(self) -> None:
rows, _ = _record(_ORIGINAL_INPUT, _ScriptedLLM(_recovery_script()))
Expand Down
Loading
0