From df014e73b8b47baea240fbb08cbb109000f2d757 Mon Sep 17 00:00:00 2001 From: mattt Date: Sat, 17 Oct 2026 17:18:25 -0700 Subject: [PATCH 1/5] Send tool-calling mode as the tool choice of remote providers --- .../Models/AnthropicLanguageModel.swift | 10 + .../Models/GeminiLanguageModel.swift | 20 + .../Models/OpenAILanguageModel.swift | 8 + .../Models/OpenResponsesLanguageModel.swift | 4 + .../Shared/ToolCallingModeRequests.swift | 11 + .../ToolCallingModeTests.swift | 407 ++++++++++++++++++ 6 files changed, 460 insertions(+) create mode 100644 Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift create mode 100644 Tests/AnyLanguageModelTests/ToolCallingModeTests.swift diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 3f370420..88fe86aa 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -773,6 +773,16 @@ private func createMessageParams( } if let tools, !tools.isEmpty { params["tools"] = try JSONValue(tools) + // Sent only with tools. A custom `toolChoice` or `extraBody` overrides it below. + if let toolCallingMode = options.toolCallingMode { + let type = + switch toolCallingMode.kind { + case .allowed: "auto" + case .required: "any" + case .disallowed: "none" + } + params["tool_choice"] = .object(["type": .string(type)]) + } } if let responseSchema { // Structured outputs: https://platform.claude.com/docs/en/build-with-claude/structured-outputs diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 7e69a923..531130d5 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -580,6 +580,20 @@ private func createGenerateContentParams( break } } + + // The function-calling mode applies to function declarations, not to server tools, + // so it's sent only with the session's tools. + if let toolCallingMode = options.toolCallingMode, tools.contains(where: \.isFunctionDeclarations) { + let mode = + switch toolCallingMode.kind { + case .allowed: "AUTO" + case .required: "ANY" + case .disallowed: "NONE" + } + var toolConfig = params["toolConfig"]?.objectValue ?? [:] + toolConfig["functionCallingConfig"] = .object(["mode": .string(mode)]) + params["toolConfig"] = .object(toolConfig) + } } var generationConfig: [String: JSONValue] = [:] @@ -889,6 +903,12 @@ private enum GeminiTool: Sendable { } } + /// Whether this tool holds the session's function declarations, not a server tool. + var isFunctionDeclarations: Bool { + if case .functionDeclarations = self { return true } + return false + } + var toolConfigValue: JSONValue? { switch self { case .googleMaps(let latitude, let longitude): diff --git a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift index b6f17391..536fc958 100644 --- a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift @@ -986,6 +986,10 @@ private enum ChatCompletions { if let tools { body["tools"] = .array(tools.map { $0.jsonValue(for: .chatCompletions) }) + // Sent only with tools. `extraBody` can override it. + if let toolCallingMode = options.toolCallingMode { + body["tool_choice"] = .string(toolCallingMode.kind.openAIToolChoice) + } } if type != String.self { @@ -1246,6 +1250,10 @@ private enum Responses { if let tools { body["tools"] = .array(tools.map { $0.jsonValue(for: .responses) }) + // Sent only with tools. `extraBody` can override it. + if let toolCallingMode = options.toolCallingMode { + body["tool_choice"] = .string(toolCallingMode.kind.openAIToolChoice) + } } if type != String.self { diff --git a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift index 807bbb5e..f18a0e55 100644 --- a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift @@ -808,6 +808,10 @@ private enum OpenResponsesAPI { if let tools { body["tools"] = .array(tools.map { $0.jsonValue }) + // Sent only with tools. A custom `toolChoice` or `extraBody` overrides it below. + if let toolCallingMode = options.toolCallingMode { + body["tool_choice"] = .string(toolCallingMode.kind.openAIToolChoice) + } } if type != String.self { diff --git a/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift b/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift new file mode 100644 index 00000000..d1725872 --- /dev/null +++ b/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift @@ -0,0 +1,11 @@ +extension GenerationOptions.ToolCallingMode.Kind { + /// The `tool_choice` value for this kind + /// in the OpenAI Chat Completions, OpenAI Responses, and Open Responses APIs. + var openAIToolChoice: String { + switch self { + case .allowed: "auto" + case .required: "required" + case .disallowed: "none" + } + } +} diff --git a/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift new file mode 100644 index 00000000..07909811 --- /dev/null +++ b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift @@ -0,0 +1,407 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if canImport(Darwin) && !canImport(AsyncHTTPClient) + @Suite("Tool-calling mode in provider requests", .serialized) + struct ToolCallingModeRequestTests { + typealias Kind = GenerationOptions.ToolCallingMode.Kind + + enum Provider: String, CaseIterable, Sendable, CustomTestStringConvertible { + case chat, responses, openResponses, anthropic, gemini, ollama + + var testDescription: String { rawValue } + + func makeModel() -> any LanguageModel { + let http = ToolCallingModeURLProtocol.makeSession() + let model: any LanguageModel + switch self { + case .chat, .responses: + model = OpenAILanguageModel( + apiKey: "test", + model: "test", + apiVariant: self == .chat ? .chatCompletions : .responses, + session: http + ) + case .openResponses: + model = OpenResponsesLanguageModel( + baseURL: URL(string: "https://example.com/v1")!, + apiKey: "test", + model: "test", + session: http + ) + case .anthropic: + model = AnthropicLanguageModel(apiKey: "test", model: "test", session: http) + case .gemini: + model = GeminiLanguageModel(apiKey: "test", model: "test", session: http) + case .ollama: + model = OllamaLanguageModel(model: "test", session: http) + } + return model + } + + func makeSession(tools: [any Tool] = [WeatherTool()]) -> LanguageModelSession { + LanguageModelSession(model: makeModel(), tools: tools) + } + + /// A response with text, or with a call to `getWeather` for Paris. + func response(tool: Bool = false) -> String { + switch self { + case .chat: + let message = + tool + ? #"{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"getWeather","arguments":"{\"city\":\"Paris\"}"}}]}"# + : #"{"role":"assistant","content":"Hello"}"# + return #"{"id":"test","choices":[{"message":\#(message)}]}"# + case .responses, .openResponses: + let output = + tool + ? #"{"type":"function_call","call_id":"call_1","name":"getWeather","arguments":"{\"city\":\"Paris\"}"}"# + : #"{"type":"message","content":[{"type":"output_text","text":"Hello"}]}"# + return #"{"id":"test","output":[\#(output)]}"# + case .anthropic: + let content = + tool + ? #"{"type":"tool_use","id":"call_1","name":"getWeather","input":{"city":"Paris"}}"# + : #"{"type":"text","text":"Hello"}"# + return #"{"id":"test","type":"message","role":"assistant","model":"test","content":[\#(content)]}"# + case .gemini: + let part = + tool + ? #"{"functionCall":{"name":"getWeather","args":{"city":"Paris"}}}"# + : #"{"text":"Hello"}"# + return #"{"candidates":[{"content":{"role":"model","parts":[\#(part)]}}]}"# + case .ollama: + let message = + tool + ? #"{"role":"assistant","content":"","tool_calls":[{"function":{"name":"getWeather","arguments":{"city":"Paris"}}}]}"# + : #"{"role":"assistant","content":"Hello"}"# + let created = "2026-09-11T00:00:00.000Z" + return #"{"model":"test","created_at":"\#(created)","message":\#(message),"done":true}"# + } + } + + /// The tool choice in a request body, in the provider's format. + func toolChoice(in body: [String: JSONValue]) -> JSONValue? { + switch self { + case .gemini: body["toolConfig"]?.objectValue?["functionCallingConfig"] + default: body["tool_choice"] + } + } + + /// The tool choice a request should carry for a tool-calling mode. + func expectedToolChoice(_ kind: Kind?) -> JSONValue? { + guard let kind else { return nil } + switch self { + case .chat, .responses, .openResponses: + let values: [Kind: String] = [.allowed: "auto", .required: "required", .disallowed: "none"] + return .string(values[kind]!) + case .anthropic: + let values: [Kind: String] = [.allowed: "auto", .required: "any", .disallowed: "none"] + return .object(["type": .string(values[kind]!)]) + case .gemini: + let values: [Kind: String] = [.allowed: "AUTO", .required: "ANY", .disallowed: "NONE"] + return .object(["mode": .string(values[kind]!)]) + case .ollama: + // Ollama has no tool-choice setting. + return nil + } + } + } + + static let modes: [GenerationOptions.ToolCallingMode?] = [nil, .allowed, .disallowed, .required] + + private func requestBodies() throws -> [[String: JSONValue]] { + try ToolCallingModeURLProtocol.recordedBodies.map { + try JSONDecoder().decode([String: JSONValue].self, from: $0) + } + } + + @Test(arguments: Provider.allCases, modes) + func requestCarriesMode(_ provider: Provider, mode: GenerationOptions.ToolCallingMode?) async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(provider.response()) + let options = GenerationOptions(toolCallingMode: mode) + let response = try await provider.makeSession().respond(to: "Weather?", options: options) + #expect(response.content == "Hello") + + let body = try #require(requestBodies().first) + #expect(provider.toolChoice(in: body) == provider.expectedToolChoice(mode?.kind)) + if mode == nil { + // Without a mode, the request has no tool-choice fields. + #expect(body["tool_choice"] == nil) + #expect(body["toolConfig"] == nil) + } + #expect(body["tools"] != nil) + } + + @Test(arguments: Provider.allCases, modes) + func requestWithoutToolsOmitsMode(_ provider: Provider, mode: GenerationOptions.ToolCallingMode?) async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(provider.response()) + let options = GenerationOptions(toolCallingMode: mode) + _ = try await provider.makeSession(tools: []).respond(to: "Hi", options: options) + + let body = try #require(requestBodies().first) + #expect(body["tools"] == nil) + #expect(body["tool_choice"] == nil) + #expect(body["toolConfig"] == nil) + } + + @Test(arguments: [Provider.chat, .responses, .openResponses, .gemini]) + func followUpRequestKeepsRequiredMode(_ provider: Provider) async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(provider.response(tool: true)) + ToolCallingModeURLProtocol.enqueue(provider.response()) + let options = GenerationOptions(toolCallingMode: .required) + let response = try await provider.makeSession().respond(to: "Weather?", options: options) + #expect(response.content == "Hello") + + let bodies = try requestBodies() + #expect(bodies.count == 2) + for body in bodies { + #expect(provider.toolChoice(in: body) == provider.expectedToolChoice(.required)) + } + } + + @Test func anthropicFollowUpRequestKeepsRequiredMode() async throws { + ToolCallingModeURLProtocol.reset() + let message = #"{"id":"test","type":"message","role":"assistant","model":"test","content":[]}"# + let start = #"{"type":"message_start","message":\#(message)}"# + ToolCallingModeURLProtocol.enqueue( + eventStream: [ + start, + #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_1","name":"getWeather","input":{}}}"#, + #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Paris\"}"}}"#, + #"{"type":"content_block_stop","index":0}"#, + #"{"type":"message_stop"}"#, + ] + ) + ToolCallingModeURLProtocol.enqueue( + eventStream: [ + start, + #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}"#, + #"{"type":"message_stop"}"#, + ] + ) + let options = GenerationOptions(toolCallingMode: .required) + let response = try await Provider.anthropic.makeSession() + .streamResponse(to: "Weather?", options: options) + .collect() + #expect(response.content == "Hello") + + let bodies = try requestBodies() + #expect(bodies.count == 2) + for body in bodies { + #expect(body["tool_choice"] == .object(["type": .string("any")])) + } + } + + @Test func requiredModeEndsAtToolRoundLimit() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.chat.response(tool: true)) + ToolCallingModeURLProtocol.enqueue(Provider.chat.response(tool: true)) + var options = GenerationOptions(toolCallingMode: .required) + options.maximumToolRounds = 1 + + await #expect(throws: LanguageModelSession.ToolRoundLimitExceeded.self) { + try await Provider.chat.makeSession().respond(to: "Weather?", options: options) + } + let bodies = try requestBodies() + #expect(bodies.count == 2) + for body in bodies { + #expect(body["tool_choice"] == .string("required")) + } + } + + @Test func requiredModeEndsWhenDynamicInstructionsDropTools() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.chat.response(tool: true)) + ToolCallingModeURLProtocol.enqueue(Provider.chat.response()) + let session = LanguageModelSession( + model: Provider.chat.makeModel(), + dynamicInstructions: WeatherUntilCalled(called: Locked(false)) + ) + let options = GenerationOptions(toolCallingMode: .required) + let response = try await session.respond(to: "Weather?", options: options) + #expect(response.content == "Hello") + + let bodies = try requestBodies() + #expect(bodies.count == 2) + #expect(bodies.first?["tool_choice"] == .string("required")) + // The follow-up request has no tools, so it has no tool choice. + #expect(bodies.last?["tools"] == nil) + #expect(bodies.last?["tool_choice"] == nil) + } + + @Test func anthropicCustomToolChoiceOverridesMode() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.anthropic.response()) + var options = GenerationOptions(toolCallingMode: .disallowed) + options[custom: AnthropicLanguageModel.self] = .init(toolChoice: .tool(name: "getWeather")) + _ = try await Provider.anthropic.makeSession().respond(to: "Weather?", options: options) + + let body = try #require(requestBodies().first) + #expect(body["tool_choice"] == .object(["type": .string("tool"), "name": .string("getWeather")])) + } + + @Test func openResponsesCustomToolChoiceOverridesMode() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.openResponses.response()) + var options = GenerationOptions(toolCallingMode: .required) + options[custom: OpenResponsesLanguageModel.self] = .init(toolChoice: .auto) + _ = try await Provider.openResponses.makeSession().respond(to: "Weather?", options: options) + + let body = try #require(requestBodies().first) + #expect(body["tool_choice"] == .string("auto")) + } + + @Test(arguments: [Provider.chat, .responses, .openResponses, .anthropic]) + func extraBodyOverridesMode(_ provider: Provider) async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(provider.response()) + var options = GenerationOptions(toolCallingMode: .required) + let extraBody: [String: JSONValue] = ["tool_choice": .string("none")] + switch provider { + case .chat, .responses: + options[custom: OpenAILanguageModel.self] = .init(extraBody: extraBody) + case .openResponses: + options[custom: OpenResponsesLanguageModel.self] = .init(extraBody: extraBody) + default: + options[custom: AnthropicLanguageModel.self] = .init(extraBody: extraBody) + } + _ = try await provider.makeSession().respond(to: "Weather?", options: options) + + let body = try #require(requestBodies().first) + #expect(body["tool_choice"] == .string("none")) + } + + @Test func geminiModeJoinsRetrievalConfig() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.gemini.response()) + var options = GenerationOptions(toolCallingMode: .required) + options[custom: GeminiLanguageModel.self] = .init( + serverTools: [.googleMaps(latitude: 37.33, longitude: -122.01)] + ) + _ = try await Provider.gemini.makeSession().respond(to: "Weather?", options: options) + + let body = try #require(requestBodies().first) + let toolConfig = try #require(body["toolConfig"]?.objectValue) + #expect(toolConfig["functionCallingConfig"] == .object(["mode": .string("ANY")])) + #expect(toolConfig["retrievalConfig"] != nil) + } + + @Test func geminiServerToolsAloneOmitMode() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.gemini.response()) + var options = GenerationOptions(toolCallingMode: .required) + options[custom: GeminiLanguageModel.self] = .init(serverTools: [.googleSearch]) + _ = try await Provider.gemini.makeSession(tools: []).respond(to: "Weather?", options: options) + + let body = try #require(requestBodies().first) + #expect(body["tools"] != nil) + #expect(body["toolConfig"] == nil) + } + } + + /// Offers `getWeather` until it's called, + /// like the dynamic profile in the Foundation Models `ToolCallingMode` example. + private struct WeatherUntilCalled: DynamicInstructions { + let called: Locked + + var body: some DynamicInstructions { + Instructions("Answer questions about the weather.") + if !called.withLock({ $0 }) { + [OneShotWeatherTool(called: called)] as [any Tool] + } + } + } + + private struct OneShotWeatherTool: Tool { + let name = "getWeather" + let description = "Retrieve the latest weather information for a city" + let called: Locked + + func call(arguments: WeatherTool.Arguments) async throws -> String { + called.withLock { $0 = true } + return try await WeatherTool().call(arguments: arguments) + } + } + + /// Answers requests from a queue of canned responses and records each request body. + /// Its state is separate from other suites' fixtures, so suites can run concurrently. + private final class ToolCallingModeURLProtocol: URLProtocol { + private struct State: Sendable { + var pending: [Data] = [] + var recordedBodies: [Data] = [] + } + + private static let state = Locked(State()) + + static func reset() { + state.withLock { $0 = State() } + } + + static func enqueue(_ json: String) { + state.withLock { $0.pending.append(Data(json.utf8)) } + } + + static func enqueue(eventStream events: [String]) { + let body = events.map { "data: \($0)\n\n" }.joined() + state.withLock { $0.pending.append(Data(body.utf8)) } + } + + static var recordedBodies: [Data] { + state.withLock { $0.recordedBodies } + } + + static func makeSession() -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [ToolCallingModeURLProtocol.self] + return URLSession(configuration: configuration) + } + + override class func canInit(with request: URLRequest) -> Bool { true } + + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + + override func startLoading() { + // URLSession moves `httpBody` to `httpBodyStream` before the protocol sees the request. + let body = request.httpBody ?? request.httpBodyStream.map(Self.readAll) ?? Data() + let pending = Self.state.withLock { state -> Data? in + state.recordedBodies.append(body) + return state.pending.isEmpty ? nil : state.pending.removeFirst() + } + guard let pending, let url = request.url else { + client?.urlProtocol(self, didFailWithError: URLError(.resourceUnavailable)) + return + } + let streaming = request.value(forHTTPHeaderField: "Accept") == "text/event-stream" + let response = HTTPURLResponse( + url: url, + statusCode: 200, + httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": streaming ? "text/event-stream" : "application/json"] + )! + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: pending) + client?.urlProtocolDidFinishLoading(self) + } + + override func stopLoading() {} + + private static func readAll(_ stream: InputStream) -> Data { + stream.open() + defer { stream.close() } + var data = Data() + var buffer = [UInt8](repeating: 0, count: 4096) + while true { + let read = stream.read(&buffer, maxLength: buffer.count) + if read <= 0 { break } + data.append(buffer, count: read) + } + return data + } + } +#endif From fddd6f33240ec5baeda5db868cf32b4bf13708b3 Mon Sep 17 00:00:00 2001 From: mattt Date: Sat, 17 Oct 2026 17:19:40 -0700 Subject: [PATCH 2/5] Offer no tools for disallowed mode in models without a tool choice --- .../Models/CoreMLLanguageModel.swift | 10 +++++++--- .../Models/LlamaLanguageModel.swift | 12 ++++++++---- .../Models/MLXLanguageModel.swift | 12 ++++++++---- .../Models/OllamaLanguageModel.swift | 12 ++++++++---- .../Shared/ToolCallingModeRequests.swift | 14 ++++++++++++++ .../ToolCallingModeTests.swift | 18 +++++++++++++++++- 6 files changed, 62 insertions(+), 16 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift index 24733d54..5c17b946 100644 --- a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift @@ -141,7 +141,7 @@ if let chatTemplateHandler = chatTemplateHandler { // Use chat template handler with optional tools let messages = chatTemplateHandler(requestContext.instructions, prompt) - let toolSpecs: [ToolSpec]? = toolsHandler?(requestContext.tools) + let toolSpecs: [ToolSpec]? = toolsHandler?(options.toolsOffered(from: requestContext.tools)) tokens = try tokenizer.applyChatTemplate(messages: messages, tools: toolSpecs) } else { // Fall back to direct tokenizer encoding @@ -242,7 +242,9 @@ if let chatTemplateHandler = chatTemplateHandler { // Use chat template handler with optional tools let messages = chatTemplateHandler(requestContext.instructions, prompt) - let toolSpecs: [ToolSpec]? = toolsHandler?(requestContext.tools) + let toolSpecs: [ToolSpec]? = toolsHandler?( + options.toolsOffered(from: requestContext.tools) + ) tokens = try tokenizer.applyChatTemplate(messages: messages, tools: toolSpecs) } else { // Fall back to direct tokenizer encoding @@ -413,6 +415,7 @@ let promptTokens = try structuredPromptTokens( requestContext: requestContext, + tools: options.toolsOffered(from: requestContext.tools), prompt: prompt, schema: schema, includeSchemaInPrompt: includeSchemaInPrompt @@ -450,6 +453,7 @@ private func structuredPromptTokens( requestContext: LanguageModelSession.RequestContext, + tools: [any Tool], prompt: Prompt, schema: GenerationSchema, includeSchemaInPrompt: Bool @@ -462,7 +466,7 @@ messages.insert(["role": "system", "content": schemaPrompt], at: 0) } } - let toolSpecs: [ToolSpec]? = toolsHandler?(requestContext.tools) + let toolSpecs: [ToolSpec]? = toolsHandler?(tools) return try tokenizer.applyChatTemplate(messages: messages, tools: toolSpecs) } diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index d03386d4..9d1abd62 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -861,8 +861,10 @@ import Foundation if mmprojPath == nil { try validateNoImageSegments(in: requestContext.transcript) } + // llama.cpp has no tool-choice setting. + let tools = options.toolsOffered(from: requestContext.tools) let toolContext = try makeToolPromptContext( - tools: requestContext.tools, + tools: tools, pendingEntries: pendingEntries ) var promptImages: [Data] = [] @@ -932,7 +934,7 @@ import Foundation let resolution = try await resolveToolCalls( parsedCalls, - tools: requestContext.tools, + tools: tools, session: session ) switch resolution { @@ -1108,8 +1110,10 @@ import Foundation if self.mmprojPath == nil { try self.validateNoImageSegments(in: requestContext.transcript) } + // llama.cpp has no tool-choice setting. + let tools = options.toolsOffered(from: requestContext.tools) let toolContext = try self.makeToolPromptContext( - tools: requestContext.tools, + tools: tools, pendingEntries: pendingEntries ) var promptImages: [Data] = [] @@ -1194,7 +1198,7 @@ import Foundation let resolution = try await self.resolveToolCalls( parsedCalls, - tools: requestContext.tools, + tools: tools, session: session ) switch resolution { diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 51be9b0c..9ba15848 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -1133,7 +1133,9 @@ import Foundation // Loop until no more tool calls while true { let requestContext = session.resolvedRequestContext() - let toolSpecs = mlxToolSpecs(for: requestContext.tools) + // MLX has no tool-choice setting. + let tools = options.toolsOffered(from: requestContext.tools) + let toolSpecs = mlxToolSpecs(for: tools) let chat = convertTranscriptToMLXChat( requestContext: requestContext, @@ -1199,7 +1201,7 @@ import Foundation let resolution = try await resolveToolCalls( collectedToolCalls, - tools: requestContext.tools, + tools: tools, session: session ) switch resolution { @@ -1374,7 +1376,9 @@ import Foundation // Loop until the model stops without pending tool calls (mirrors `respond()`). toolLoop: while true { let requestContext = session.resolvedRequestContext() - let toolSpecs = mlxToolSpecs(for: requestContext.tools) + // MLX has no tool-choice setting. + let tools = options.toolsOffered(from: requestContext.tools) + let toolSpecs = mlxToolSpecs(for: tools) let chat = convertTranscriptToMLXChat( requestContext: requestContext, @@ -1440,7 +1444,7 @@ import Foundation let resolution = try await resolveToolCalls( collectedToolCalls, - tools: requestContext.tools, + tools: tools, session: session ) switch resolution { diff --git a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift index 36c3ddfb..dc194f51 100644 --- a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift @@ -129,7 +129,9 @@ public struct OllamaLanguageModel: LanguageModel { } let requestContext = session.resolvedRequestContext() - let ollamaTools = try requestContext.tools.map(convertToolToOllamaFormat) + // Ollama has no tool-choice setting. + let tools = options.toolsOffered(from: requestContext.tools) + let ollamaTools = try tools.map(convertToolToOllamaFormat) var messages = try requestContext.transcript.toOllamaMessages() if messages.isEmpty { messages.append(.init(role: .user, content: prompt.description)) @@ -160,7 +162,7 @@ public struct OllamaLanguageModel: LanguageModel { if let toolCalls = chatResponse.message.toolCalls, !toolCalls.isEmpty { let resolution = try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + tools: tools, session: session ) switch resolution { @@ -258,7 +260,9 @@ public struct OllamaLanguageModel: LanguageModel { while true { try Task.checkCancellation() let requestContext = session.resolvedRequestContext() - let tools = try requestContext.tools.map(convertToolToOllamaFormat) + // Ollama has no tool-choice setting. + let offeredTools = options.toolsOffered(from: requestContext.tools) + let tools = try offeredTools.map(convertToolToOllamaFormat) var messages = try requestContext.transcript.toOllamaMessages() if messages.isEmpty { messages.append(.init(role: .user, content: prompt.description)) @@ -296,7 +300,7 @@ public struct OllamaLanguageModel: LanguageModel { try toolRounds.record(completed: state.entries) switch try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + tools: offeredTools, session: session ) { case .stop(let calls): diff --git a/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift b/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift index d1725872..0ee94668 100644 --- a/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift +++ b/Sources/AnyLanguageModel/Shared/ToolCallingModeRequests.swift @@ -9,3 +9,17 @@ extension GenerationOptions.ToolCallingMode.Kind { } } } + +extension GenerationOptions { + /// The tools to offer the model in one request, + /// for a model that has no tool-choice setting. + /// + /// Such a model honors ``ToolCallingMode/disallowed`` by being offered no tools, + /// so no tool runs. + /// It can't be made to call a tool, + /// so it's offered the tools for ``ToolCallingMode/required`` + /// as it is for ``ToolCallingMode/allowed``. + func toolsOffered(from tools: [any Tool]) -> [any Tool] { + toolCallingMode?.kind == .disallowed ? [] : tools + } +} diff --git a/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift index 07909811..b29cfe91 100644 --- a/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift +++ b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift @@ -3,6 +3,20 @@ import Testing @testable import AnyLanguageModel +@Suite("Tool-calling mode for models without a tool-choice setting") +struct ToolsOfferedTests { + @Test(arguments: [nil, .allowed, .required] as [GenerationOptions.ToolCallingMode?]) + func toolsAreOffered(mode: GenerationOptions.ToolCallingMode?) { + let options = GenerationOptions(toolCallingMode: mode) + #expect(options.toolsOffered(from: [WeatherTool()]).map(\.name) == ["getWeather"]) + } + + @Test func disallowedOffersNoTools() { + let options = GenerationOptions(toolCallingMode: .disallowed) + #expect(options.toolsOffered(from: [WeatherTool()]).isEmpty) + } +} + #if canImport(Darwin) && !canImport(AsyncHTTPClient) @Suite("Tool-calling mode in provider requests", .serialized) struct ToolCallingModeRequestTests { @@ -133,7 +147,9 @@ import Testing #expect(body["tool_choice"] == nil) #expect(body["toolConfig"] == nil) } - #expect(body["tools"] != nil) + // Ollama honors `disallowed` by leaving the tools out. + let offersTools = !(provider == .ollama && mode == .disallowed) + #expect((body["tools"] != nil) == offersTools) } @Test(arguments: Provider.allCases, modes) From e29c0d785b61fae3dcf5f91328d4cb73dc7fb3b6 Mon Sep 17 00:00:00 2001 From: mattt Date: Sat, 17 Oct 2026 17:20:25 -0700 Subject: [PATCH 3/5] Document how each provider handles tool-calling mode --- README.md | 49 +++++++++++++++++-- .../AnyLanguageModel/GenerationOptions.swift | 27 ++++++++-- .../Models/AnthropicLanguageModel.swift | 1 + .../Models/OpenResponsesLanguageModel.swift | 2 + 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ad1e2259..079ea34b 100644 --- a/README.md +++ b/README.md @@ -537,12 +537,53 @@ let context = ContextOptions( ) ``` -Only the models that run through Foundation Models use `toolCallingMode`. -`SystemLanguageModel`, `PrivateCloudComputeLanguageModel`, and `FoundationLanguageModel` -pass it to Foundation Models +A `nil` tool-calling mode, the default, leaves each model's behavior unchanged. +Each provider handles the three modes as follows: + +| Provider | `allowed` | `disallowed` | `required` | +| --- | --- | --- | --- | +| `SystemLanguageModel`, `PrivateCloudComputeLanguageModel`, `FoundationLanguageModel` | Passed to Foundation Models | Passed to Foundation Models | Passed to Foundation Models | +| `OpenAILanguageModel` (both APIs), `OpenResponsesLanguageModel` | `"tool_choice": "auto"` | `"tool_choice": "none"` | `"tool_choice": "required"` | +| `AnthropicLanguageModel` | `"tool_choice": {"type": "auto"}` | `"tool_choice": {"type": "none"}` | `"tool_choice": {"type": "any"}` | +| `GeminiLanguageModel` | `functionCallingConfig` mode `AUTO` | `functionCallingConfig` mode `NONE` | `functionCallingConfig` mode `ANY` | +| `OllamaLanguageModel`, `MLXLanguageModel`, `LlamaLanguageModel`, `CoreMLLanguageModel` | Tools offered as usual | No tools offered | Not supported: tools offered as usual | + +The models that run through Foundation Models pass the mode on in apps built with Swift 6.4 or later that run on OS 27 or later, and not on tvOS. -Other providers ignore it. +Otherwise, they ignore it. + +The remote providers send the mode only with a request that has tools, +because a tool choice applies to the tools in the request. +For Gemini, server tools such as Google Search don't count: +the mode applies to the session's tools. +A provider's own tool-choice option takes precedence over `toolCallingMode`: +`toolChoice` in the custom options of `AnthropicLanguageModel` +and `OpenResponsesLanguageModel`, +and a `tool_choice` key in `extraBody` +for `OpenAILanguageModel`, `OpenResponsesLanguageModel`, and `AnthropicLanguageModel`. + +Ollama, MLX, llama.cpp, and Core ML have no tool-choice setting. +They honor `disallowed` by leaving the tools out of the prompt, +so no tool runs. +They can't make a model call a tool, +so they treat `required` like `allowed`. + +As in Foundation Models, the mode applies to every request in a response, +including each request that continues the response after tool calls. +With `required`, the model keeps calling tools +until a tool throws an error, +a `ToolExecutionDelegate` stops the response, +dynamic instructions remove the tools, +or a round of tool calls past `maximumToolRounds` +ends the response with `LanguageModelSession.ToolRoundLimitExceeded`. + +> [!NOTE] +> Anthropic rejects `{"type": "any"}` +> with manual extended thinking (`thinking: .enabled(budgetTokens:)`) +> and on models that don't support forced tool use. +> See [Forcing tool use](https://platform.claude.com/docs/en/agents-and-tools/tool-use/define-tools#forcing-tool-use). + A transcript keeps the tool-calling mode of each prompt. No AnyLanguageModel API accepts `ContextOptions` yet, diff --git a/Sources/AnyLanguageModel/GenerationOptions.swift b/Sources/AnyLanguageModel/GenerationOptions.swift index cfe15d38..6865fec4 100644 --- a/Sources/AnyLanguageModel/GenerationOptions.swift +++ b/Sources/AnyLanguageModel/GenerationOptions.swift @@ -132,10 +132,29 @@ public struct GenerationOptions: Sendable, Equatable { /// The tool calling requirements. /// - /// ``SystemLanguageModel``, ``PrivateCloudComputeLanguageModel``, and ``FoundationLanguageModel`` - /// pass it to Foundation Models - /// when built with Swift 6.4 or later and run on OS 27 or later, except on tvOS. - /// Other models ignore it. + /// Leave it `nil` to use the model's default. + /// Models handle it as follows: + /// + /// - ``SystemLanguageModel``, ``PrivateCloudComputeLanguageModel``, and ``FoundationLanguageModel`` + /// pass it to Foundation Models + /// when built with Swift 6.4 or later and run on OS 27 or later, except on tvOS. + /// Otherwise, they ignore it. + /// - ``OpenAILanguageModel``, ``OpenResponsesLanguageModel``, ``AnthropicLanguageModel``, + /// and ``GeminiLanguageModel`` send it as the API's tool-choice setting + /// with each request that has tools. + /// A tool-choice custom option or `extraBody` overrides it. + /// - ``OllamaLanguageModel``, `MLXLanguageModel`, `LlamaLanguageModel`, and `CoreMLLanguageModel` + /// have no tool-choice setting. + /// For ``ToolCallingMode/disallowed``, they offer the model no tools. + /// They can't make the model call a tool, + /// so they offer the tools as usual for ``ToolCallingMode/required``. + /// + /// As in Foundation Models, the mode applies to every request in a response, + /// including each request that continues the response after tool calls. + /// With ``ToolCallingMode/required``, the model calls tools until a tool throws, + /// a ``ToolExecutionDelegate`` stops the response, + /// dynamic instructions remove the tools, + /// or a round of tool calls past ``maximumToolRounds`` ends the response. /// /// - Note: This API is exclusive to AnyLanguageModel on OS 26. /// It follows the Foundation Models 27 `GenerationOptions.toolCallingMode` API, diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 88fe86aa..812d0809 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -82,6 +82,7 @@ public struct AnthropicLanguageModel: LanguageModel { /// How the model should use the provided tools. /// /// Use this to control whether the model can use tools and which tools it prefers. + /// When set, it takes precedence over ``GenerationOptions/toolCallingMode``. public var toolChoice: ToolChoice? /// Configuration for extended thinking. diff --git a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift index f18a0e55..d5fa65f0 100644 --- a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift @@ -35,6 +35,8 @@ public struct OpenResponsesLanguageModel: LanguageModel { /// reasoning/text options. Use ``extraBody`` for parameters not yet modeled. public struct CustomGenerationOptions: AnyLanguageModel.CustomGenerationOptions, Codable, Sendable { /// Controls which tool the model should use, if any. + /// + /// When set, it takes precedence over ``GenerationOptions/toolCallingMode``. public var toolChoice: ToolChoice? /// The list of tools that are permitted for this request. From 5aa61926dadda7c16cd87416d52bcc04272ee67e Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 19 Oct 2026 05:28:54 -0700 Subject: [PATCH 4/5] Fix the tool-calling mode overrides in the docs --- Sources/AnyLanguageModel/GenerationOptions.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/AnyLanguageModel/GenerationOptions.swift b/Sources/AnyLanguageModel/GenerationOptions.swift index 6865fec4..729b475a 100644 --- a/Sources/AnyLanguageModel/GenerationOptions.swift +++ b/Sources/AnyLanguageModel/GenerationOptions.swift @@ -142,7 +142,10 @@ public struct GenerationOptions: Sendable, Equatable { /// - ``OpenAILanguageModel``, ``OpenResponsesLanguageModel``, ``AnthropicLanguageModel``, /// and ``GeminiLanguageModel`` send it as the API's tool-choice setting /// with each request that has tools. - /// A tool-choice custom option or `extraBody` overrides it. + /// The `toolChoice` custom option + /// of ``AnthropicLanguageModel`` and ``OpenResponsesLanguageModel`` overrides it, + /// as does a `tool_choice` key in `extraBody` + /// for ``OpenAILanguageModel``, ``OpenResponsesLanguageModel``, and ``AnthropicLanguageModel``. /// - ``OllamaLanguageModel``, `MLXLanguageModel`, `LlamaLanguageModel`, and `CoreMLLanguageModel` /// have no tool-choice setting. /// For ``ToolCallingMode/disallowed``, they offer the model no tools. From 3f752486795507d385a5ff5cadddee159b890e2a Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 19 Oct 2026 05:28:54 -0700 Subject: [PATCH 5/5] Test tool-calling mode in Ollama streaming responses --- .../ToolCallingModeTests.swift | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift index b29cfe91..6e89a0b0 100644 --- a/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift +++ b/Tests/AnyLanguageModelTests/ToolCallingModeTests.swift @@ -214,6 +214,65 @@ struct ToolsOfferedTests { } } + @Test(arguments: modes) + func ollamaStreamingRequestOffersToolsUnlessDisallowed(mode: GenerationOptions.ToolCallingMode?) async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.ollama.response()) + let options = GenerationOptions(toolCallingMode: mode) + let response = try await Provider.ollama.makeSession() + .streamResponse(to: "Weather?", options: options) + .collect() + #expect(response.content == "Hello") + + let body = try #require(requestBodies().first) + #expect(body["stream"] == .bool(true)) + #expect((body["tools"] != nil) == (mode != .disallowed)) + #expect(body["tool_choice"] == nil) + } + + @Test func ollamaStreamingDisallowedModeRunsNoTool() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.ollama.response(tool: true)) + ToolCallingModeURLProtocol.enqueue(Provider.ollama.response()) + let called = Locked(false) + let session = LanguageModelSession( + model: Provider.ollama.makeModel(), + tools: [OneShotWeatherTool(called: called)] + ) + let options = GenerationOptions(toolCallingMode: .disallowed) + let response = try await session.streamResponse(to: "Weather?", options: options).collect() + #expect(response.content == "Hello") + // A tool call resolves against the offered tools, which are none. + #expect(!called.withLock { $0 }) + + let bodies = try requestBodies() + #expect(bodies.count == 2) + for body in bodies { + #expect(body["tools"] == nil) + } + } + + @Test func ollamaStreamingFollowUpRequestKeepsTools() async throws { + ToolCallingModeURLProtocol.reset() + ToolCallingModeURLProtocol.enqueue(Provider.ollama.response(tool: true)) + ToolCallingModeURLProtocol.enqueue(Provider.ollama.response()) + let called = Locked(false) + let session = LanguageModelSession( + model: Provider.ollama.makeModel(), + tools: [OneShotWeatherTool(called: called)] + ) + let options = GenerationOptions(toolCallingMode: .required) + let response = try await session.streamResponse(to: "Weather?", options: options).collect() + #expect(response.content == "Hello") + #expect(called.withLock { $0 }) + + let bodies = try requestBodies() + #expect(bodies.count == 2) + for body in bodies { + #expect(body["tools"] != nil) + } + } + @Test func requiredModeEndsAtToolRoundLimit() async throws { ToolCallingModeURLProtocol.reset() ToolCallingModeURLProtocol.enqueue(Provider.chat.response(tool: true))