Repository navigation
Expand file tree
/
Copy pathllm.py
More file actions
131 lines (98 loc) · 4.08 KB
/
Copy pathllm.py
File metadata and controls
131 lines (98 loc) · 4.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""LLM client abstraction for MiniBot.
Provides a provider-agnostic interface so the Agent Core
never depends on a specific SDK.
"""
from __future__ import annotations
import abc
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
if TYPE_CHECKING:
from .runtime.messages import ModelMessage
from .tools.definitions import ModelToolDefinition
# ── normalised response types ────────────────────────────────────
@dataclass(frozen=True)
class ToolCall:
"""A single tool invocation requested by the model."""
id: str
name: str
arguments: str # raw JSON string
@dataclass(frozen=True)
class TokenUsage:
"""Provider-reported token usage for one model call."""
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
# Portion of input_tokens served from the provider's prompt cache.
cached_input_tokens: int | None = None
@dataclass(frozen=True)
class LLMResponse:
"""Provider-agnostic model response."""
content: str | None = None
reasoning_content: str | None = None
tool_calls: list[ToolCall] = field(default_factory=list)
usage: TokenUsage | None = None
debug: dict[str, Any] | None = None
@dataclass(frozen=True)
class LLMStreamEvent:
"""One element of a streamed model response.
Contract: a stream yields zero or more ``text`` / ``reasoning`` deltas,
then exactly one ``response`` event carrying the complete ``LLMResponse``
as its final element. Deltas are advisory; the terminal response is the
single source of truth (usage and tool calls ride on it).
"""
kind: Literal["text", "reasoning", "response"]
text: str = ""
response: LLMResponse | None = None
@classmethod
def text_delta(cls, text: str) -> "LLMStreamEvent":
return cls(kind="text", text=text)
@classmethod
def reasoning_delta(cls, text: str) -> "LLMStreamEvent":
return cls(kind="reasoning", text=text)
@classmethod
def completed(cls, response: LLMResponse) -> "LLMStreamEvent":
return cls(kind="response", response=response)
# ── abstract client ──────────────────────────────────────────────
def is_retryable_llm_error(exc: Exception) -> bool:
"""Classify a provider exception as transient (worth retrying) or not.
Provider-agnostic: HTTP-style errors expose ``status_code`` (429 and 5xx
are transient, other 4xx are caller mistakes); connection/timeout errors
carry no status and are transient by nature.
"""
status = getattr(exc, "status_code", None)
if isinstance(status, int):
return status == 429 or status >= 500
transient_names = {
"APIConnectionError",
"APITimeoutError",
"ConnectionError",
"TimeoutError",
}
return any(
klass.__name__ in transient_names for klass in type(exc).__mro__
)
class LLMClient(abc.ABC):
"""Interface that every LLM provider must implement."""
@abc.abstractmethod
def chat(
self,
messages: list[ModelMessage],
tools: list[ModelToolDefinition] | None = None,
model: str | None = None,
) -> LLMResponse: ...
def chat_stream(
self,
messages: list[ModelMessage],
tools: list[ModelToolDefinition] | None = None,
model: str | None = None,
) -> Iterator[LLMStreamEvent]:
"""Stream a model response.
The default wraps ``chat`` in a single terminal event, so providers
(and test fakes) without native streaming keep working unchanged and
callers never need a streaming/non-streaming branch.
Retries belong to the caller: an implementation makes one attempt per
call and raises. ``chat`` is for callers with no retry of their own
(the summariser, the eval judge), so it may retry inside.
"""
yield LLMStreamEvent.completed(self.chat(messages, tools, model))