Skip to content

Fix Clone Loop AskUserQuestion threshold gating #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Turtle-Hwan merged 1 commit into from
May 22, 2026
+88 −58
Merged

Fix Clone Loop AskUserQuestion threshold gating #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 44 additions & 42 deletions hooks/ask-user-question-hook.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ function numericConfidence(value, fallback = 0) {
}

function chooseHighestConfidenceOption(prediction, options) {
const labels = options.map((option) => String(option?.label || '').trim()).filter(Boolean)
const rankedCandidates = rankedPredictionCandidates(prediction)
const mapped = []

Expand All @@ -76,13 +75,7 @@ function chooseHighestConfidenceOption(prediction, options) {
}

if (mapped.length) return mapped[0]

return {
answer: labels[0] || null,
confidence: rankedCandidates[0]?.confidence ?? 0,
text: rankedCandidates[0]?.text || '',
fallback: true,
}
return null
}

function formatOptions(options) {
Expand Down Expand Up @@ -203,7 +196,7 @@ function allowAnswer({ toolInput, answers, confidence, threshold }) {
hookSpecificOutput: {
hookEventName: 'PreToolUse',
permissionDecision: 'allow',
permissionDecisionReason: `Clone answered AskUserQuestion with a free-form predicted response. Confidence ${confidence}; threshold ${threshold} is advisory for questions.`,
permissionDecisionReason: `Clone answered AskUserQuestion with a predicted response. Confidence ${confidence}; threshold ${threshold}.`,
updatedInput: {
...toolInput,
questions: toolInput.questions,
Expand All @@ -220,14 +213,17 @@ function allowAnswer({ toolInput, answers, confidence, threshold }) {
)
}

async function safeReject({ predictionId, mcpSessionId }) {
if (!predictionId) return
try {
await submitFeedback({ predictionId, status: 'rejected', mcpSessionId })
appendHistory({ event: 'feedback-sent', source: 'ask-user-question', prediction_id: predictionId, status: 'rejected' })
} catch (error) {
appendHistory({ event: 'feedback-sent', source: 'ask-user-question', prediction_id: predictionId, status: 'rejected', error: error?.message || String(error) })
}
function deferQuestion({ decision, question, threshold, prediction, confidence, error }) {
appendHistory({
event: 'ask-user-question',
decision,
question,
threshold: Number(threshold),
confidence: confidence == null ? null : Number(confidence),
prediction_id: prediction?.id || null,
status: prediction?.status || null,
error: error ? error?.message || String(error) : undefined,
})
}

async function main() {
Expand Down Expand Up @@ -293,25 +289,22 @@ async function main() {
mcpSessionId: mcpSessionIdInitial,
})
} catch (error) {
const fallbackAnswer = String(options[0]?.label || '').trim()
if (fallbackAnswer) {
answers[questionText] = fallbackAnswer
confidenceValues.push(0)
appendHistory({
event: 'ask-user-question',
decision: 'auto-answer-fallback-mcp-error',
question: questionText,
answer: fallbackAnswer,
error: error?.message || String(error),
})
continue
}

appendHistory({
event: 'ask-user-question',
deferQuestion({
decision: 'defer-mcp-error',
question: questionText,
error: error?.message || String(error),
threshold: cloneThreshold,
error,
})
return
}

if (prediction?.status !== 'auto') {
deferQuestion({
decision: 'defer-non-auto',
question: questionText,
threshold: cloneThreshold,
prediction,
confidence: prediction.confidence,
})
return
}
Expand All @@ -326,16 +319,25 @@ async function main() {
}
: chooseHighestConfidenceOption(prediction, options)

if (!selection.answer) {
appendHistory({
event: 'ask-user-question',
decision: 'defer-no-options',
if (!selection?.answer) {
deferQuestion({
decision: 'defer-unmapped',
question: questionText,
threshold: cloneThreshold,
prediction,
confidence: prediction?.confidence,
})
return
}

if (!(Number.isFinite(Number(selection.confidence)) && Number(selection.confidence) >= Number(cloneThreshold))) {
deferQuestion({
decision: 'defer-low-confidence',
question: questionText,
threshold: Number(cloneThreshold),
prediction_id: prediction.id || null,
status: prediction.status || null,
threshold: cloneThreshold,
prediction,
confidence: selection.confidence,
})
await safeReject({ predictionId: prediction.id, mcpSessionId: mcpSessionIdInitial })
return
}

Expand Down
60 changes: 44 additions & 16 deletions tests/ask-user-question-hook.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ describe('AskUserQuestion PreToolUse hook', () => {
const output = JSON.parse(result.stdout)
assert.equal(output.hookSpecificOutput.hookEventName, 'PreToolUse')
assert.equal(output.hookSpecificOutput.permissionDecision, 'allow')
assert.match(output.hookSpecificOutput.permissionDecisionReason, /free-form predicted response/)
assert.match(output.hookSpecificOutput.permissionDecisionReason, /predicted response/)
assert.deepEqual(output.hookSpecificOutput.updatedInput, {
questions: toolInput.questions,
answers: {
Expand Down Expand Up @@ -266,13 +266,13 @@ describe('AskUserQuestion PreToolUse hook', () => {
)
})

it('answers with the free-form predicted response even when confidence is low or status is not auto', async () => {
it('does not answer when confidence is below threshold or status is not auto', async () => {
writeState(workdir)

await withMcpServer(
{
id: 'question-prediction-2',
status: 'escalated',
id: 'question-prediction-2-low',
status: 'auto',
threshold: 0.6,
predicted_response: 'I would run the focused tests before opening a PR.',
confidence: 0.42,
Expand All @@ -283,8 +283,32 @@ describe('AskUserQuestion PreToolUse hook', () => {
})

assert.equal(result.status, 0)
const output = JSON.parse(result.stdout)
assert.equal(output.hookSpecificOutput.updatedInput.answers['Continue?'], 'I would run the focused tests before opening a PR.')
assert.equal(result.stdout, '')
const history = readFileSync(join(workdir, '.claude', 'clone-loop.history.local.jsonl'), 'utf8')
assert.match(history, /"decision":"defer-low-confidence"/)
assert.ok(readFileSync(join(workdir, '.claude', 'clone-loop.local.md'), 'utf8'))
},
)

writeState(workdir)

await withMcpServer(
{
id: 'question-prediction-2-escalated',
status: 'escalated',
threshold: 0.6,
predicted_response: 'Open a PR.',
confidence: 0.91,
},
async (endpoint) => {
const result = await runHook(workdir, endpoint, {
questions: [{ question: 'Continue?', options: [{ label: 'Run tests' }, { label: 'Open PR' }] }],
})

assert.equal(result.status, 0)
assert.equal(result.stdout, '')
const history = readFileSync(join(workdir, '.claude', 'clone-loop.history.local.jsonl'), 'utf8')
assert.match(history, /"decision":"defer-non-auto"/)
assert.ok(readFileSync(join(workdir, '.claude', 'clone-loop.local.md'), 'utf8'))
},
)
Expand Down Expand Up @@ -322,18 +346,18 @@ describe('AskUserQuestion PreToolUse hook', () => {
)
})

it('falls back to the first option when neither free-form nor mapped candidate is available', async () => {
it('defers when neither free-form nor mapped candidate is available', async () => {
writeState(workdir)

await withMcpServer(
{
id: 'question-prediction-4',
status: 'escalated',
status: 'auto',
threshold: 0.6,
confidence: 0.21,
confidence: 0.91,
candidates: [
{ predicted_response: 'Use the default option', confidence: 0.21 },
{ predicted_response: 'Whatever is safest', confidence: 0.18 },
{ predicted_response: 'Use the default option', confidence: 0.91 },
{ predicted_response: 'Whatever is safest', confidence: 0.88 },
],
},
async (endpoint) => {
Expand All @@ -347,13 +371,15 @@ describe('AskUserQuestion PreToolUse hook', () => {
})

assert.equal(result.status, 0)
const output = JSON.parse(result.stdout)
assert.equal(output.hookSpecificOutput.updatedInput.answers['Which option?'], 'Default fast path')
assert.equal(result.stdout, '')
const history = readFileSync(join(workdir, '.claude', 'clone-loop.history.local.jsonl'), 'utf8')
assert.match(history, /"decision":"defer-unmapped"/)
assert.ok(readFileSync(join(workdir, '.claude', 'clone-loop.local.md'), 'utf8'))
},
)
})

it('falls back to the first option instead of asking the user when Clone MCP fails', async () => {
it('defers instead of choosing the first option when Clone MCP fails', async () => {
writeState(workdir)

await withFailingMcpServer(async (endpoint) => {
Expand All @@ -367,8 +393,10 @@ describe('AskUserQuestion PreToolUse hook', () => {
})

assert.equal(result.status, 0)
const output = JSON.parse(result.stdout)
assert.equal(output.hookSpecificOutput.updatedInput.answers['Which option?'], 'Run focused tests')
assert.equal(result.stdout, '')
const history = readFileSync(join(workdir, '.claude', 'clone-loop.history.local.jsonl'), 'utf8')
assert.match(history, /"decision":"defer-mcp-error"/)
assert.ok(readFileSync(join(workdir, '.claude', 'clone-loop.local.md'), 'utf8'))
})
})

Expand Down