Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ describe("CustomToolRegistry", () => {
const result = await registry.loadFromDirectory(TEST_FIXTURES_DIR)

expect(result.loaded).toContain("cached")
}, 30000)
}, 60000)
})

describe.sequential("loadFromDirectories", () => {
Expand Down
15 changes: 6 additions & 9 deletions src/api/providers/__tests__/anthropic-vertex.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1200,8 +1200,9 @@ describe("VertexHandler", () => {
)
})

it("should include tools even when toolProtocol is set to xml (user preference now ignored)", async () => {
// XML protocol deprecation: user preference is now ignored when model supports native tools
it("should NOT include tools when toolProtocol is set to xml (user preference is now respected)", async () => {
// User preference is now respected: when XML protocol is selected, tools are NOT sent to the API
// (tools are handled via XML in the system prompt instead)
handler = new AnthropicVertexHandler({
apiModelId: "claude-3-5-sonnet-v2@20241022",
vertexProjectId: "test-project",
Expand Down Expand Up @@ -1242,14 +1243,10 @@ describe("VertexHandler", () => {
// Just consume
}

// Native is forced when supportsNativeTools===true, so tools should still be included
// User preference for XML is now respected, so tools should NOT be included
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tools: expect.arrayContaining([
expect.objectContaining({
name: "get_weather",
}),
]),
expect.not.objectContaining({
tools: expect.anything(),
}),
undefined,
)
Expand Down
15 changes: 6 additions & 9 deletions src/api/providers/__tests__/anthropic.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -451,8 +451,9 @@ describe("AnthropicHandler", () => {
)
})

it("should include tools even when toolProtocol is set to xml (user preference now ignored)", async () => {
// XML protocol deprecation: user preference is now ignored when model supports native tools
it("should NOT include tools when toolProtocol is set to xml (user preference is now respected)", async () => {
// User preference is now respected: when XML protocol is selected, tools are NOT sent to the API
// (tools are handled via XML in the system prompt instead)
const xmlHandler = new AnthropicHandler({
...mockOptions,
toolProtocol: "xml",
Expand All @@ -468,14 +469,10 @@ describe("AnthropicHandler", () => {
// Just consume
}

// Native is forced when supportsNativeTools===true, so tools should still be included
// User preference for XML is now respected, so tools should NOT be included
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tools: expect.arrayContaining([
expect.objectContaining({
name: "get_weather",
}),
]),
expect.not.objectContaining({
tools: expect.anything(),
}),
expect.anything(),
)
Expand Down
110 changes: 79 additions & 31 deletions src/utils/__tests__/resolveToolProtocol.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,19 @@ import type { Anthropic } from "@anthropic-ai/sdk"

describe("resolveToolProtocol", () => {
/**
* XML Protocol Deprecation:
*
* XML tool protocol has been fully deprecated. All models now use Native
* tool calling. User preferences and model defaults are ignored.
* Tool Protocol Resolution:
*
* Precedence:
* 1. Locked Protocol (for resumed tasks that used XML)
* 2. Native (always, for all new tasks)
* 1. Locked Protocol (for resumed tasks - highest priority)
* 2. User/Profile Preference (providerSettings.toolProtocol) - allows users to force XML
* for models that don't handle native tool calling well (e.g., Qwen3, Kimi2)
* 3. Native (default for all new tasks without explicit preference)
*/

describe("Locked Protocol (Precedence Level 0 - Highest Priority)", () => {
it("should return lockedProtocol when provided", () => {
it("should return lockedProtocol when provided, ignoring user preference", () => {
const settings: ProviderSettings = {
toolProtocol: "xml", // Ignored
toolProtocol: "xml", // User preference - overridden by locked protocol
apiProvider: "openai-native",
}
// lockedProtocol overrides everything
Expand All @@ -29,44 +28,66 @@ describe("resolveToolProtocol", () => {

it("should return XML lockedProtocol for resumed tasks that used XML", () => {
const settings: ProviderSettings = {
toolProtocol: "native", // Ignored
toolProtocol: "native", // User preference - overridden by locked protocol
apiProvider: "anthropic",
}
// lockedProtocol forces XML for backward compatibility
const result = resolveToolProtocol(settings, undefined, "xml")
expect(result).toBe(TOOL_PROTOCOL.XML)
})

it("should fall through to Native when lockedProtocol is undefined", () => {
it("should fall through to user preference when lockedProtocol is undefined", () => {
const settings: ProviderSettings = {
toolProtocol: "xml", // Ignored
toolProtocol: "xml", // User preference takes effect
apiProvider: "anthropic",
}
// undefined lockedProtocol should return native
// undefined lockedProtocol should use user preference
const result = resolveToolProtocol(settings, undefined, undefined)
expect(result).toBe(TOOL_PROTOCOL.NATIVE)
expect(result).toBe(TOOL_PROTOCOL.XML)
})
})

describe("Native Protocol Always Used For New Tasks", () => {
it("should always use native for new tasks", () => {
describe("User/Profile Preference (Precedence Level 1)", () => {
it("should respect user preference for XML protocol", () => {
const settings: ProviderSettings = {
apiProvider: "anthropic",
toolProtocol: "xml",
apiProvider: "openai",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.XML)
})

it("should respect user preference for native protocol", () => {
const settings: ProviderSettings = {
toolProtocol: "native",
apiProvider: "openai",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE)
})

it("should use native even when user preference is XML (user prefs ignored)", () => {
it("should allow XML for OpenAI-compatible models when user prefers XML", () => {
// This is the key scenario for models like Qwen3, Kimi2 that
// don't handle native tool calling well
const settings: ProviderSettings = {
toolProtocol: "xml", // User wants XML - ignored
apiProvider: "openai-native",
toolProtocol: "xml",
apiProvider: "openai",
}
const result = resolveToolProtocol(settings, openAiModelInfoSaneDefaults)
expect(result).toBe(TOOL_PROTOCOL.XML)
})
})

describe("Default Protocol (Precedence Level 2 - No User Preference)", () => {
it("should default to native for new tasks without user preference", () => {
const settings: ProviderSettings = {
apiProvider: "anthropic",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE)
})

it("should use native for OpenAI compatible provider", () => {
it("should default to native for OpenAI compatible provider without preference", () => {
const settings: ProviderSettings = {
apiProvider: "openai",
}
Expand All @@ -79,26 +100,34 @@ describe("resolveToolProtocol", () => {
it("should handle missing provider name gracefully", () => {
const settings: ProviderSettings = {}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Always native now
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Default to native
})

it("should handle undefined model info gracefully", () => {
const settings: ProviderSettings = {
apiProvider: "openai-native",
}
const result = resolveToolProtocol(settings, undefined)
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Always native now
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Default to native
})

it("should handle empty settings", () => {
const settings: ProviderSettings = {}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Always native now
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Default to native
})

it("should respect user XML preference even with empty provider", () => {
const settings: ProviderSettings = {
toolProtocol: "xml",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.XML)
})
})

describe("Real-world Scenarios", () => {
it("should use Native for OpenAI models", () => {
it("should default to Native for OpenAI models", () => {
const settings: ProviderSettings = {
apiProvider: "openai-native",
}
Expand All @@ -112,7 +141,7 @@ describe("resolveToolProtocol", () => {
expect(result).toBe(TOOL_PROTOCOL.NATIVE)
})

it("should use Native for Claude models", () => {
it("should default to Native for Claude models", () => {
const settings: ProviderSettings = {
apiProvider: "anthropic",
}
Expand All @@ -134,21 +163,40 @@ describe("resolveToolProtocol", () => {
const result = resolveToolProtocol(settings, undefined, "xml")
expect(result).toBe(TOOL_PROTOCOL.XML)
})

it("should allow user to force XML for OpenAI-compatible models (Qwen3, Kimi2 use case)", () => {
// This is the primary use case from issue #10459
// Users with Qwen3 or Kimi2 models need XML protocol
const settings: ProviderSettings = {
toolProtocol: "xml",
apiProvider: "openai",
}
const result = resolveToolProtocol(settings, openAiModelInfoSaneDefaults)
expect(result).toBe(TOOL_PROTOCOL.XML)
})
})

describe("Backward Compatibility - User Preferences Ignored", () => {
it("should ignore user preference for XML", () => {
describe("Backward Compatibility - User Preferences Now Respected", () => {
it("should respect user preference for XML when set", () => {
const settings: ProviderSettings = {
toolProtocol: "xml", // User explicitly wants XML - ignored
toolProtocol: "xml",
apiProvider: "openai-native",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE) // Native is always used
expect(result).toBe(TOOL_PROTOCOL.XML)
})

it("should respect user preference for native when set", () => {
const settings: ProviderSettings = {
toolProtocol: "native",
apiProvider: "anthropic",
}
const result = resolveToolProtocol(settings)
expect(result).toBe(TOOL_PROTOCOL.NATIVE)
})

it("should return native regardless of user preference", () => {
it("should default to native when no preference set", () => {
const settings: ProviderSettings = {
toolProtocol: "native", // User preference - ignored but happens to match
apiProvider: "anthropic",
}
const result = resolveToolProtocol(settings)
Expand Down
23 changes: 13 additions & 10 deletions src/utils/resolveToolProtocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,19 @@ type ApiMessageForDetection = Anthropic.MessageParam & {
/**
* Resolve the effective tool protocol.
*
* **Deprecation Note (XML Protocol):**
* XML tool protocol has been deprecated. All models now use Native tool calling.
* User/profile preferences (`providerSettings.toolProtocol`) and model defaults
* (`modelInfo.defaultToolProtocol`) are ignored.
*
* Precedence:
* 1. Locked Protocol (task-level lock for resumed tasks - highest priority)
* 2. Native (always, for all new tasks)
* 2. User/Profile Preference (providerSettings.toolProtocol) - allows forcing XML for models
* that don't handle native tool calling well (e.g., some OpenAI-compatible models like Qwen3, Kimi2)
* 3. Native (default for all new tasks without explicit preference)
*
* @param _providerSettings - The provider settings (toolProtocol field is ignored)
* @param providerSettings - The provider settings (toolProtocol field used for user preference)
* @param _modelInfo - Unused, kept for API compatibility
* @param lockedProtocol - Optional task-locked protocol that takes absolute precedence
* @returns The resolved tool protocol (either "xml" or "native")
*/
export function resolveToolProtocol(
_providerSettings: ProviderSettings,
providerSettings: ProviderSettings,
_modelInfo?: unknown,
lockedProtocol?: ToolProtocol,
): ToolProtocol {
Expand All @@ -39,8 +36,14 @@ export function resolveToolProtocol(
return lockedProtocol
}

// 2. Always return Native protocol for new tasks
// All models now support native tools; XML is deprecated
// 2. User/Profile Preference - allow users to explicitly force XML protocol
// This is useful for models that don't handle native tool calling well
// (e.g., some OpenAI-compatible models like Qwen3, Kimi2)
if (providerSettings?.toolProtocol) {
return providerSettings.toolProtocol
}

// 3. Default to Native protocol for new tasks
return TOOL_PROTOCOL.NATIVE
}

Expand Down
27 changes: 27 additions & 0 deletions webview-ui/src/components/settings/ApiOptions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,33 @@ const ApiOptions = ({
}
onChange={(value) => setApiConfigurationField("consecutiveMistakeLimit", value)}
/>
<div>
<label className="block font-medium mb-1">{t("settings:toolProtocol.label")}</label>
<Select
value={apiConfiguration?.toolProtocol || "default"}
onValueChange={(value) =>
setApiConfigurationField(
"toolProtocol",
value === "default" ? undefined : (value as "xml" | "native"),
)
}>
<SelectTrigger className="w-full">
<SelectValue placeholder={t("settings:common.select")} />
</SelectTrigger>
<SelectContent>
<SelectItem value="default">
{t("settings:toolProtocol.currentDefault", {
protocol: t("settings:toolProtocol.native"),
})}
</SelectItem>
<SelectItem value="native">{t("settings:toolProtocol.native")}</SelectItem>
<SelectItem value="xml">{t("settings:toolProtocol.xml")}</SelectItem>
</SelectContent>
</Select>
<div className="text-sm text-vscode-descriptionForeground mt-1">
{t("settings:toolProtocol.description")}
</div>
</div>
{selectedProvider === "openrouter" &&
openRouterModelProviders &&
Object.keys(openRouterModelProviders).length > 0 && (
Expand Down
Loading