8000 [pull] main from can1357:main by pull[bot] · Pull Request #16 · ericrihm/oh-my-pi · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,7 @@ providers:
openaiWebsockets: auto
openrouterVariant: default
kimiApiFormat: auto
cacheRetention: auto
maxInFlightRequests:
anthropic: 2

Expand Down Expand Up @@ -738,6 +739,7 @@ searxng:
| `providers.openaiWebsockets` | enum | `auto` | `auto`, `off`, `on`. |
| `providers.openrouterVariant` | enum | `default` | `default`, `nitro`, `floor`, `online`, `exacto`. |
| `providers.kimiApiFormat` | enum | `auto` | `auto`, `openai`, `anthropic`. `auto` follows live model metadata. |
| `providers.cacheRetention` | enum | `auto` | `auto`, `short`, `long`, `none`. Prompt-cache retention forwarded to providers that support it. `auto` keeps provider defaults (Anthropic: 5m entries + idle keep-alive refreshes) and honors `PI_CACHE_RETENTION`; `short` forces 5m; `long` uses 1h TTLs where supported and disables keep-alive refreshes; `none` disables prompt caching and cache-affinity routing. |
| `provider.appendOnlyContext` | enum | `auto` | `auto`, `on`, `off`. |
| `exa.enabled` | boolean | `true` | Enable Exa integration. |
| `exa.enableSearch` | boolean | `true` | Exa search. |
Expand Down
4 changes: 4 additions & 0 deletions packages/ai/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## [Unreleased]

### Fixed

- Fixed thinking effort selections being ignored for local Qwen 3.8+ models on llama.cpp and vLLM: the Qwen chat-completions dialects only toggled `enable_thinking`, so the chat template always reasoned at its `xhigh` default no matter which level was selected. The encoder now routes the requested effort onto the template's `reasoning_effort` kwarg (`chat_template_kwargs` for both Qwen dialects, plus the top-level field newer llama.cpp builds map natively).

## [17.3.7] - 2026-08-17

### Changed
Expand Down
26 changes: 24 additions & 2 deletions packages/ai/src/providers/openai-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ export type OpenAICompletionsParams = Omit<ChatCompletionCreateParamsStreaming,
thinking?: { type: "enabled" | "disabled"; effort?: string; keep?: "all" };
enable_thinking?: boolean;
preserve_thinking?: boolean;
chat_template_kwargs?: { enable_thinking?: boolean; preserve_thinking?: boolean };
chat_template_kwargs?: { enable_thinking?: boolean; preserve_thinking?: boolean; reasoning_effort?: string };
reasoning?: { effort?: string } | { enabled: false };
reasoning_effort?: string | null;
service_tier?: ServiceTier;
Expand Down Expand Up @@ -1049,12 +1049,34 @@ export function applyChatCompletionsCompatPolicy(params: OpenAICompletionsParams
break;
case "qwen-enable-thinking-false":
params.enable_thinking = true;
// Qwen 3.8+ templates steer thinking depth via the
// `reasoning_effort` kwarg (low/medium/xhigh, template default
// xhigh) — without it every effort selection lands on xhigh.
// Twin emission mirrors `preserve_thinking` above: newer
// llama.cpp builds map the top-level OpenAI field into the
// template, older builds and Alibaba-style local servers read
// only the kwargs copy. The `qwen-chat-template` dialect (NIM,
// vLLM/SGLang) rides kwargs alone — NIM's request schema
// rejects unknown top-level fields (#2299).
if (policy.compat.qwenTemplateReasoningEffort && reasoning.wireEffort !== undefined) {
params.reasoning_effort = reasoning.wireEffort;
params.chat_template_kwargs = {
...params.chat_template_kwargs,
reasoning_effort: reasoning.wireEffort,
};
}
break;
case "qwen-template-false":
// Spread so the `preserve_thinking` kwarg hoisted above
// survives the merge — a bare `{ enable_thinking: true }`
// would clobber it.
params.chat_template_kwargs = { ...params.chat_template_kwargs, enable_thinking: true };
params.chat_template_kwargs = {
...params.chat_template_kwargs,
enable_thinking: true,
...(policy.compat.qwenTemplateReasoningEffort && reasoning.wireEffort !== undefined
? { reasoning_effort: reasoning.wireEffort }
: {}),
};
break;
case "openrouter-enabled-false":
if (reasoning.wireEffort !== undefined) {
Expand Down
30 changes: 28 additions & 2 deletions packages/ai/test/anthropic-cache-refresh.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from "bun:test";
import { streamSimple } from "@oh-my-pi/pi-ai";
import type { MessageCreateParams } from "@oh-my-pi/pi-ai/providers/anthropic-wire";
import type { Context, FetchImpl, Model, ProviderSessionState } from "@oh-my-pi/pi-ai/types";
import type { CacheControlEphemeral, MessageCreateParams } from "@oh-my-pi/pi-ai/providers/anthropic-wire";
import type { CacheRetention, Context, FetchImpl, Model, ProviderSessionState } from "@oh-my-pi/pi-ai/types";
import { buildModel } from "@oh-my-pi/pi-catalog/build";

const CACHE_REFRESH_DELAY_MS = 5 * 60_000 - 15_000;
Expand Down Expand Up @@ -161,6 +161,7 @@ function createFetch(modes: ResponseMode[], capture: FetchCapture): FetchImpl {

interface FinishRequestOptions {
anthropicCacheRefresh?: boolean;
cacheRetention?: CacheRetention;
model?: Model<"anthropic-messages">;
sessionId?: string;
}
Expand All @@ -175,6 +176,7 @@ async function finishRequest(
fetch,
apiKey: "test-anthropic-key",
anthropicCacheRefresh: options.anthropicCacheRefresh ?? true,
cacheRetention: options.cacheRetention,
providerSessionState,
sessionId: options.sessionId ?? "cache-refresh-test-session",
});
Expand Down Expand Up @@ -285,4 +287,28 @@ describe("Anthropic prompt-cache refresh", () => {
expect(capture.bodies[1]?.stream).toBe(true);
expect(capture.thinkingRefreshAborted).toBe(true);
});

it("skips keep-alive refreshes and emits 1h breakpoints when retention is long", async () => {
vi.useFakeTimers();
const capture: FetchCapture = { bodies: [], thinkingRefreshAborted: false };
const fetch = createFetch(["ordinary-write"], capture);
const states = createProviderSessionState();

await finishRequest(fetch, states, { cacheRetention: "long" });
vi.advanceTimersByTime(CACHE_REFRESH_DELAY_MS * 2);
await Promise.resolve();

// No zero-output replay was scheduled for the 1h entry.
expect(capture.bodies).toHaveLength(1);
const blocks = (capture.bodies[0]?.messages ?? []).flatMap(message =>
Array.isArray(message.content) ? message.content : [],
);
const breakpoints = blocks
.map(block => ("cache_control" in block ? (block.cache_control ?? undefined) : undefined))
.filter((cc): cc is CacheControlEphemeral => cc != null);
expect(breakpoints.length).toBeGreaterThan(0);
for (const cc of breakpoints) {
expect(cc.ttl).toBe("1h");
}
});
});
1 change: 1 addition & 0 deletions packages/ai/test/issue-967-vision-guard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ const compat: ResolvedOpenAICompat = {
allowsSyntheticReasoningContentForToolCalls: true,
replayReasoningContent: false,
qwenPreserveThinking: false,
qwenTemplateReasoningEffort: false,
requiresAssistantContentForToolCalls: false,
openRouterRouting: {},
vercelGatewayRouting: {},
Expand Down
66 changes: 66 additions & 0 deletions packages/ai/test/openai-compat-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,4 +199,70 @@ describe("OpenAI compat policy", () => {
expect(params.enable_thinking).toBe(true);
expect(params.reasoning_effort).toBeUndefined();
});

function localQwenModel(id: string, provider: string, baseUrl: string): Model<"openai-completions"> {
return buildModel({
id,
name: id,
api: "openai-completions",
provider,
baseUrl,
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262_144,
maxTokens: 32_768,
} satisfies ModelSpec<"openai-completions">);
}

it("routes local Qwen3.8 effort selections onto the chat template (llama.cpp qwen dialect)", () => {
// Regression: the qwen dialects used to emit only `enable_thinking: true`,
// so every effort selection ran at the template's xhigh default.
const model = localQwenModel("qwen3.8-27b", "llama.cpp", "http://127.0.0.1:8080/v1");
for (const effort of [Effort.Low, Effort.Medium, Effort.XHigh]) {
const params = chatParams();
applyChatCompletionsCompatPolicy(
params,
resolveOpenAICompatPolicy(model, { endpoint: "chat-completions", reasoning: effort }),
);
// Twin emission: top-level for newer llama.cpp builds, kwargs for
// older builds — and the preserve_thinking kwarg must survive.
expect(params.enable_thinking).toBe(true);
expect(params.reasoning_effort).toBe(effort);
expect(params.chat_template_kwargs).toEqual({ preserve_thinking: true, reasoning_effort: effort });
expect(params.preserve_thinking).toBe(true);
}
});

it("routes local Qwen3.8 effort selections via chat_template_kwargs only on vLLM", () => {
// vLLM's renderer reads chat_template_kwargs; NIM-style schemas reject
// unknown top-level fields, so nothing may ride top-level here.
const model = localQwenModel("qwen3.8-27b", "vllm", "http://127.0.0.1:8000/v1");
const params = chatParams();
applyChatCompletionsCompatPolicy(
params,
resolveOpenAICompatPolicy(model, { endpoint: "chat-completions", reasoning: Effort.Medium }),
);
expect(params.enable_thinking).toBeUndefined();
expect(params.reasoning_effort).toBeUndefined();
expect(params.chat_template_kwargs).toEqual({
preserve_thinking: true,
enable_thinking: true,
reasoning_effort: Effort.Medium,
});
});

it("keeps pre-3.8 local Qwen on the bare enable_thinking toggle", () => {
// Qwen 3.6 templates have no reasoning_effort kwarg; leaking one would
// inject an undefined template variable for zero benefit.
const model = localQwenModel("qwen-3.6-27b", "llama.cpp", "http://127.0.0.1:8080/v1");
const params = chatParams();
applyChatCompletionsCompatPolicy(
params,
resolveOpenAICompatPolicy(model, { endpoint: "chat-completions", reasoning: Effort.High }),
);
expect(params.enable_thinking).toBe(true);
expect(params.reasoning_effort).toBeUndefined();
expect(params.chat_template_kwargs).toEqual({ preserve_thinking: true });
});
});
1 change: 1 addition & 0 deletions packages/ai/test/openai-completions-compat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ describe("openai-completions compatibility", () => {
allowsSyntheticReasoningContentForToolCalls: true,
replayReasoningContent: false,
qwenPreserveThinking: false,
qwenTemplateReasoningEffort: false,
requiresAssistantContentForToolCalls: false,
openRouterRouting: {},
vercelGatewayRouting: {},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ const compat: ResolvedOpenAICompat = {
allowsSyntheticReasoningContentForToolCalls: true,
replayReasoningContent: false,
qwenPreserveThinking: false,
qwenTemplateReasoningEffort: false,
requiresAssistantContentForToolCalls: false,
openRouterRouting: {},
vercelGatewayRouting: {},
Expand Down
4 changes: 4 additions & 0 deletions packages/catalog/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## [Unreleased]

### Fixed

- Fixed local Qwen 3.8+ models (llama.cpp, vLLM, loopback custom providers) exposing the generic `minimal..high` thinking ladder instead of the chat template's real `low`/`medium`/`xhigh` `reasoning_effort` tiers. The derived metadata now marks thinking as mandatory (the official 3.8 template raises on `enable_thinking: false`), vLLM-served Qwen routes through the `chat_template_kwargs` dialect (top-level `enable_thinking` is ignored by vLLM), and vLLM discovery lights up the reasoning dial for Qwen 3.8+ ids its `/v1/models` endpoint reports as non-reasoning.

## [17.3.6] - 2026-08-17

### Changed
Expand Down
16 changes: 15 additions & 1 deletion packages/catalog/src/compat/openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
isKimiModelId,
isMimoModelIdOrName,
isOpenAISamplingRestrictedModelId,
isQwen38PlusTemplateEffortModelId,
isQwenModelId,
} from "../identity/family";
import type {
Expand Down Expand Up @@ -464,7 +465,7 @@ export function buildOpenAICompat(spec: ModelSpec<"openai-completions">): Resolv
? "zai"
: isOpenRouter
? "openrouter"
: isQwen && isNvidiaNim
: isQwen && (isNvidiaNim || provider === "vllm")
? "qwen-chat-template"
: isQwen && isFireworks
? "openai"
Expand Down Expand Up @@ -587,6 +588,18 @@ export function buildOpenAICompat(spec: ModelSpec<"openai-completions">): Resolv
// parameter, so the flag stays a no-op outside the Qwen path.
qwenPreserveThinking:
(thinkingFormat === "qwen" || thinkingFormat === "qwen-chat-template") && isLocalOpenAICompatBackend,
// Qwen 3.8+ templates steer thinking depth via the `reasoning_effort`
// template kwarg (low/medium/xhigh, default xhigh); without routing the
// requested effort there, the enable_thinking toggle alone leaves the
// model at xhigh no matter what the user selects.
// Local-only like `qwenPreserveThinking`: first-party Qwen APIs
// (Dashscope, Qwen Portal) drive effort through their own OpenAI-style
// dialect, and local Ollama keeps its native effort vocabulary.
qwenTemplateReasoningEffort:
(thinkingFormat === "qwen" || thinkingFormat === "qwen-chat-template") &&
isLocalOpenAICompatBackend &&
provider !== "ollama" &&
isQwen38PlusTemplateEffortModelId(spec.id),
requiresAssistantContentForToolCalls: isKimiModel || isDirectDeepseekReasoning,
cacheControlFormat: isOpenRouter && spec.id.startsWith("anthropic/") ? "anthropic" : undefined,
supportsPromptCacheBreakpoints,
Expand Down Expand Up @@ -761,6 +774,7 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol
// Responses-only; the Qwen `preserve_thinking` template knob lives on
// the chat-completions wire shape, never on Responses.
qwenPreserveThinking: false,
qwenTemplateReasoningEffort: false,
requiresThinkingAsText: false,
requiresMistralToolIds: false,
requiresToolResultName: false,
Expand Down
19 changes: 19 additions & 0 deletions packages/catalog/src/identity/family.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,25 @@ export const isQwenModelId = memo((modelId: string): boolean => {
return modelId.toLowerCase().includes("qwen");
});

/**
* Open-weight Qwen 3.8+ releases (`qwen3.8-27b`, `qwen3.8-2.4t-a95b`, GGUF
* names like `Qwen3.8-27B-UD-Q6_K_XL`) whose chat template steers thinking
* depth through a `reasoning_effort` template kwarg (`low`/`medium`/`xhigh`,
* template default `xhigh`; thinking itself cannot be disabled). Compared
* component-wise so `qwen3.10` sorts after `qwen3.8`. API-only `-max` SKUs are
* excluded — Dashscope drives them through OpenAI-style `reasoning_effort`
* with curated compat. The trailing guard rejects parameter-count lookalikes
* (`qwen-3.8b`) without breaking `qwen3.8-27b`.
*/
export const isQwen38PlusTemplateEffortModelId = memo((modelId: string): boolean => {
const match = /qwen[-_ ]?(\d+)\.(\d+)(?![\dbB])/i.exec(modelId);
if (!match) return false;
const major = Number.parseInt(match[1], 10);
const minor = Number.parseInt(match[2], 10);
if (major < 3 || (major === 3 && minor < 8)) return false;
return !/^-max(?:$|[-.:])/i.test(modelId.slice(match.index + match[0].length));
});

/** Gemma open-weights family (`gemma-3-27b-it`, `google/gemma-4-E2B-it`, `gemma2-9b`). */
export const isGemmaModelId = memo((modelId: string): boolean => {
return /(^|\/)gemma[-.]?\d/i.test(modelId);
Expand Down
24 changes: 22 additions & 2 deletions packages/catalog/src/model-thinking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ const LOW_HIGH_MAX_REASONING_EFFORTS: readonly Effort[] = [Effort.Low, Effort.Hi
const HIGH_MAX_REASONING_EFFORTS: readonly Effort[] = [Effort.High, Effort.Max];
/** OpenRouter's DeepSeek route accepts only `high`. */
const HIGH_ONLY_REASONING_EFFORTS: readonly Effort[] = [Effort.High];
/**
* Qwen 3.8+ open-weight chat template: prompt-steered `reasoning_effort`
* kwarg with exactly three wire tiers (template default is `xhigh`).
*/
const QWEN38_TEMPLATE_REASONING_EFFORTS: readonly Effort[] = [Effort.Low, Effort.Medium, Effort.XHigh];
/**
* Five wire tiers with a `low` floor: GPT-5.6+, Anthropic adaptive models
* with the real xhigh tier (Opus 4.7+, Sonnet 5+, Fable/Mythos 5), and the
Expand Down Expand Up @@ -179,7 +184,9 @@ function fillThinkingWireDefaults<TApi extends Api>(
thinking.supportsDisplay === undefined &&
(spec.api === "anthropic-messages" || spec.api === "bedrock-converse-stream") &&
supportsAdaptiveThinkingDisplay(spec.id);
const needsRequiresEffort = thinking.requiresEffort === undefined && impliesMandatoryReasoning(parsed, spec.id);
const needsRequiresEffort =
thinking.requiresEffort === undefined &&
(impliesMandatoryReasoning(parsed, spec.id) || isQwenTemplateReasoningEffortCompat(compat));
const needsDefaultLevel =
thinking.defaultLevel === undefined && (isKimiK3ModelId(spec.id) || isGlm53ReasoningEffortModelId(spec.id));
if (!effortsChanged && !shouldReplaceEffortMap && !needsDisplay && !needsRequiresEffort && !needsDefaultLevel) {
Expand Down Expand Up @@ -232,7 +239,7 @@ export function deriveThinking<TApi extends Api>(spec: ModelSpec<TApi>, compat:
) {
config.supportsDisplay = true;
}
if (impliesMandatoryReasoning(parsed, spec.id)) {
if (impliesMandatoryReasoning(parsed, spec.id) || isQwenTemplateReasoningEffortCompat(compat)) {
config.requiresEffort = true;
}
return config;
Expand Down Expand Up @@ -376,6 +383,13 @@ function getModelDefinedEfforts<TApi extends Api>(
if (spec.provider === "ollama") {
return OLLAMA_REASONING_EFFORTS;
}
// Qwen 3.8+ served through a local llama.cpp-style backend: the chat
// template's prompt-steered `reasoning_effort` kwarg accepts exactly
// low/medium/xhigh (and thinking cannot be turned off — the official 3.8
// template raises on `enable_thinking: false`, hence requiresEffort).
if (isOpenAICompatReasoningApi(spec.api) && isQwenTemplateReasoningEffortCompat(compat)) {
return QWEN38_TEMPLATE_REASONING_EFFORTS;
}
if (
(isOpenAICompatReasoningApi(spec.api) || (spec.api === "ollama-chat" && spec.provider === "ollama-cloud")) &&
isDeepseekReasoningModel(spec)
Expand Down Expand Up @@ -483,6 +497,12 @@ function isOpenRouterThinkingFormat(compat: CompatOf<Api>): boolean {
function isZaiThinkingFormat(compat: CompatOf<Api>): boolean {
return compat !== undefined && "thinkingFormat" in compat && compat.thinkingFormat === "zai";
}
/** Resolved-compat gate for the Qwen 3.8+ local template `reasoning_effort` dialect. */
function isQwenTemplateReasoningEffortCompat(compat: CompatOf<Api>): boolean {
return (
compat !== undefined && "qwenTemplateReasoningEffort" in compat && compat.qwenTemplateReasoningEffort === true
);
}

function inferDetectedEffortMap<TApi extends Api>(
spec: ModelSpec<TApi>,
Expand Down
Loading
Loading
0