Skip to content

fix: conditionally declare tools when OpenClaw is configured (fixes #12) #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
juliosuas wants to merge 1 commit into from
Closed

fix: conditionally declare tools when OpenClaw is configured (fixes #12) #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ enum GeminiConfig {
For messages, confirm recipient and content before delegating unless clearly urgent.
"""

static let noToolsSystemInstruction = """
You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural.

You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only.

If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features.
"""

// User-configurable values (Settings screen overrides, falling back to Secrets.swift)
static var apiKey: String { SettingsManager.shared.geminiAPIKey }
static var openClawHost: String { SettingsManager.shared.openClawHost }
Expand Down
83 changes: 45 additions & 38 deletions samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,46 +178,53 @@ class GeminiLiveService: ObservableObject {
}

private func sendSetupMessage() {
let setup: [String: Any] = [
"setup": [
"model": GeminiConfig.model,
"generationConfig": [
"responseModalities": ["AUDIO"],
"thinkingConfig": [
"thinkingBudget": 0
]
],
"systemInstruction": [
"parts": [
["text": GeminiConfig.systemInstruction]
]
],
"tools": [
[
"functionDeclarations": ToolDeclarations.allDeclarations()
]
],
"realtimeInputConfig": [
"automaticActivityDetection": [
"disabled": false,
"startOfSpeechSensitivity": "START_SENSITIVITY_HIGH",
"endOfSpeechSensitivity": "END_SENSITIVITY_LOW",
"silenceDurationMs": 500,
"prefixPaddingMs": 40
],
"activityHandling": "START_OF_ACTIVITY_INTERRUPTS",
"turnCoverage": "TURN_INCLUDES_ALL_INPUT"
],
"contextWindowCompression": [
"slidingWindow": [
"targetTokens": 80000
]
var setupContent: [String: Any] = [
"model": GeminiConfig.model,
"generationConfig": [
"responseModalities": ["AUDIO"],
"thinkingConfig": [
"thinkingBudget": 0
]
],
"systemInstruction": [
"parts": [
["text": GeminiConfig.isOpenClawConfigured
? GeminiConfig.systemInstruction
: GeminiConfig.noToolsSystemInstruction]
]
],
"realtimeInputConfig": [
"automaticActivityDetection": [
"disabled": false,
"startOfSpeechSensitivity": "START_SENSITIVITY_HIGH",
"endOfSpeechSensitivity": "END_SENSITIVITY_LOW",
"silenceDurationMs": 500,
"prefixPaddingMs": 40
],
"inputAudioTranscription": [:] as [String: Any],
"outputAudioTranscription": [:] as [String: Any]
]
"activityHandling": "START_OF_ACTIVITY_INTERRUPTS",
"turnCoverage": "TURN_INCLUDES_ALL_INPUT"
],
"contextWindowCompression": [
"slidingWindow": [
"targetTokens": 80000
]
],
"inputAudioTranscription": [:] as [String: Any],
"outputAudioTranscription": [:] as [String: Any]
]
sendJSON(setup)

// Only declare tools when OpenClaw is configured — otherwise Gemini
// will attempt tool calls that have no backend, getting stuck in "executing"
if GeminiConfig.isOpenClawConfigured {
setupContent["tools"] = [
["functionDeclarations": ToolDeclarations.allDeclarations()]
]
NSLog("[Gemini] Setup with tools (OpenClaw configured)")
} else {
NSLog("[Gemini] Setup without tools (OpenClaw not configured)")
}

sendJSON(["setup": setupContent])
}

private func sendJSON(_ json: [String: Any]) {
Expand Down
12 changes: 12 additions & 0 deletions samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,18 @@ class ToolCallRouter {
NSLog("[ToolCall] Received: %@ (id: %@) args: %@",
callName, callId, String(describing: call.args))

// Fast-fail if OpenClaw is not configured
if !GeminiConfig.isOpenClawConfigured {
NSLog("[ToolCall] OpenClaw not configured, rejecting tool call %@", callId)
let errorResult: ToolResult = .failure(
"OpenClaw is not configured. Tool calls are unavailable. " +
"Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more."
)
let response = buildToolResponse(callId: callId, name: callName, result: errorResult)
sendResponse(response)
return
}

// Circuit breaker: stop sending tool calls after repeated failures
if consecutiveFailures >= maxConsecutiveFailures {
NSLog("[ToolCall] Circuit breaker open (%d consecutive failures), rejecting %@",
Expand Down
6 changes: 6 additions & 0 deletions ...rc/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,10 @@ object GeminiConfig {
get() = openClawGatewayToken != "YOUR_OPENCLAW_GATEWAY_TOKEN"
&& openClawGatewayToken.isNotEmpty()
&& openClawHost != "http://YOUR_MAC_HOSTNAME.local"

const val NO_TOOLS_SYSTEM_INSTRUCTION = """You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural.

You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only.

If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features."""
}
20 changes: 16 additions & 4 deletions ...in/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ class GeminiLiveService {
}

private fun sendSetupMessage() {
val isOpenClawConfigured = GeminiConfig.isOpenClawConfigured

val setup = JSONObject().apply {
put("setup", JSONObject().apply {
put("model", GeminiConfig.MODEL)
Expand All @@ -222,12 +224,22 @@ class GeminiLiveService {
})
put("systemInstruction", JSONObject().apply {
put("parts", JSONArray().put(JSONObject().apply {
put("text", GeminiConfig.systemInstruction)
put("text", if (isOpenClawConfigured)
GeminiConfig.systemInstruction
else
GeminiConfig.NO_TOOLS_SYSTEM_INSTRUCTION)
}))
})
put("tools", JSONArray().put(JSONObject().apply {
put("functionDeclarations", ToolDeclarations.allDeclarationsJSON())
}))
// Only declare tools when OpenClaw is configured — otherwise Gemini
// will attempt tool calls that have no backend, getting stuck in "executing"
if (isOpenClawConfigured) {
put("tools", JSONArray().put(JSONObject().apply {
put("functionDeclarations", ToolDeclarations.allDeclarationsJSON())
}))
Log.d(TAG, "Setup with tools (OpenClaw configured)")
} else {
Log.d(TAG, "Setup without tools (OpenClaw not configured)")
}
put("realtimeInputConfig", JSONObject().apply {
put("automaticActivityDetection", JSONObject().apply {
put("disabled", false)
Expand Down
12 changes: 12 additions & 0 deletions ...ain/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw

import android.util.Log
import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiConfig
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
Expand Down Expand Up @@ -28,6 +29,17 @@ class ToolCallRouter(

Log.d(TAG, "Received: $callName (id: $callId) args: ${call.args}")

// Fast-fail if OpenClaw is not configured
if (!GeminiConfig.isOpenClawConfigured) {
Log.w(TAG, "OpenClaw not configured, rejecting tool call $callId")
val errorResult = ToolResult.Failure(
"OpenClaw is not configured. Tool calls are unavailable. " +
"Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more."
)
sendResponse(buildToolResponse(callId, callName, errorResult))
return
}

// Circuit breaker: stop sending tool calls after repeated failures
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
Log.d(TAG, "Circuit breaker open ($consecutiveFailures consecutive failures), rejecting $callId")
Expand Down