retry-breaker

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, ToolCallResult } from 'claude-code'
3
4import type { Streak } from '../types'
5
6// Warns when the main loop or a subagent keeps retrying the same failing call
7// (a busy COM port, a Modbus timeout, a flaky test): after N identical
8// failures in a row from one loop, one toast and a row above the prompt.
9// "In a row" looks past what a retry loop does between attempts (a sleep, a
10// check): the streak ends when that same call succeeds, or after OK_RESET
11// successful calls of anything else in a row; a different call failing starts
12// a new one. Observe only: every result passes through unchanged, nothing is denied.
13// $.state, $.store, $.clock and $.agent.list only: no network, processes or model calls.
14
15const streaks = atom({ plugin: 'retry-breaker', key: 'streaks' } as const, {})
16
17const DEFAULT_THRESHOLD = 3
18const OK_RESET = 5
19const MAX_LOOPS = 50
20const IDLE_MS = 30 * 60_000
21const MAX_ROWS = 3
22const KEY_MAX = 500
23const SHORT_MAX = 60
24const ERROR_MAX = 80
25const META = new Set(['tool', 'agentId', 'requestMeta', 'tool_use_id'])
26
27const clip = (text: string, max: number) => {
28  const t = text.replace(/\s+/g, ' ').trim()
29  return t.length > max ? t.slice(0, max - 1) + '…' : t
30}
31
32const stable = (v: unknown): string => {
33  if (Array.isArray(v)) return `[${v.map(stable).join(',')}]`
34  if (v && typeof v === 'object')
35    return `{${Object.keys(v as object)
36      .sort()
37      .map(k => `${JSON.stringify(k)}:${stable((v as Record<string, unknown>)[k])}`)
38      .join(',')}}`
39  return JSON.stringify(v) ?? 'null'
40}
41
42const isShell = (tool: string) => tool === 'Bash' || tool === 'PowerShell'
43
44/** The call's identity within its loop and how it is shown. */
45export function callOf(e: Record<string, unknown>): { key: string; short: string } {
46  const tool = String(e.tool)
47  if (isShell(tool)) {
48    const command = String(e.command ?? '').replace(/\s+/g, ' ').trim()
49    return { key: `${tool}\u0000${command.slice(0, KEY_MAX)}`, short: clip(command, SHORT_MAX) }
50  }
51  const input = Object.fromEntries(Object.entries(e).filter(([k]) => !META.has(k)))
52  const json = stable(input).slice(0, KEY_MAX)
53  return { key: `${tool}\u0000${json}`, short: clip(`${tool} ${json}`, SHORT_MAX) }
54}
55
56/** The error's first non-empty line, or undefined when the call succeeded. */
57export function failureOf(ran: ToolCallResult): string | undefined {
58  let text: string
59  if (ran.deny !== undefined) text = ran.deny
60  else if (ran.isError) text = ran.text ?? (typeof ran.result === 'string' ? ran.result : 'error')
61  else return undefined
62  const line = text.split(/\r?\n/).find(l => l.trim() !== '') ?? 'error'
63  return clip(line, ERROR_MAX)
64}
65
66/** One failure counted: the loop's streak grown or restarted, idle ones dropped, at most MAX_LOOPS kept. */
67export function counted(
68  all: Record<string, Streak>,
69  loop: string,
70  call: { key: string; short: string; tool: string },
71  error: string,
72  now: number,
73  threshold: number,
74): { all: Record<string, Streak>; isCrossed: boolean } {
75  const prev = all[loop]
76  const same = prev && prev.key === call.key
77  const count = same ? prev.count + 1 : 1
78  const isCrossed = count >= threshold && !(same && prev.isShown)
79  const streak: Streak = {
80    key: call.key,
81    tool: call.tool,
82    short: call.short,
83    count,
84    lastError: error,
85    lastAt: now,
86    ...(same && prev.label ? { label: prev.label } : {}),
87    isShown: count >= threshold || Boolean(same && prev.isShown),
88    isDismissed: Boolean(same && prev.isDismissed),
89  }
90  const kept = Object.entries({ ...all, [loop]: streak })
91    .filter(([, s]) => now - s.lastAt <= IDLE_MS)
92    .sort((a, b) => b[1].lastAt - a[1].lastAt)
93    .slice(0, MAX_LOOPS)
94  return { all: Object.fromEntries(kept), isCrossed }
95}
96
97/** One success counted: the same call ends its loop's streak, as do OK_RESET successes of other calls in a row. */
98export function succeeded(all: Record<string, Streak>, loop: string, key: string): Record<string, Streak> {
99  const s = all[loop]
100  if (!s) return all
101  const okSince = (s.okSince ?? 0) + 1
102  if (s.key !== key && okSince < OK_RESET) return { ...all, [loop]: { ...s, okSince } }
103  const { [loop]: _gone, ...rest } = all
104  return rest
105}
106
107async function labelOf($: EngineInterface, agentId: string | undefined): Promise<string> {
108  if (!agentId) return 'Claude'
109  try {
110    const a = (await $.agent.list()).find(x => x.id === agentId)
111    const name = a ? clip(a.description || a.type || '', 40) : ''
112    if (name) return name
113  } catch {
114    // no roster here: fall back to the id
115  }
116  return `agent ${agentId.slice(0, 8)}`
117}
118
119const thresholdOf = async ($: EngineInterface) => {
120  const n = Number(await $.store.get('threshold'))
121  return Number.isInteger(n) && n >= 2 ? n : DEFAULT_THRESHOLD
122}
123const isOff = async ($: EngineInterface) => (await $.store.get('isOff')) === true
124
125export const register: Register = on => {
126  on('session.start', async ($, e, next) => {
127    await $.command.register({
128      name: 'retry-breaker',
129      description: 'Repeated-failure warnings: status, threshold N (default 3), off, on (retry-breaker)',
130      argumentHint: '[threshold N | off | on]',
131      immediate: true,
132    })
133    return next(e)
134  })
135
136  // Observer: runs the call first, then counts; whatever happens here, the result goes back unchanged.
137  on('tool.call', async ($, e, next) => {
138    const ran = await next(e)
139    if (await isOff($)) return ran
140    const loop = e.agentId ?? 'main'
141    const error = failureOf(ran)
142    if (error === undefined) {
143      if ((await read($, streaks))[loop]) {
144        const key = callOf(e as Record<string, unknown>).key
145        await update($, streaks, all => succeeded(all, loop, key))
146      }
147      return ran
148    }
149    const call = { ...callOf(e as Record<string, unknown>), tool: String(e.tool) }
150    const now = await $.clock.now()
151    const threshold = await thresholdOf($)
152    let isCrossed = false
153    await update($, streaks, all => {
154      const r = counted(all, loop, call, error, now, threshold)
155      isCrossed = r.isCrossed
156      return r.all
157    })
158    if (isCrossed) {
159      const label = await labelOf($, e.agentId)
160      await update($, streaks, all => (all[loop]?.key === call.key ? { ...all, [loop]: { ...all[loop], label } } : all))
161      const count = (await read($, streaks))[loop]?.count ?? threshold
162      $.ui.toast(`retry-breaker: ${label} failed the same ${call.tool} ${count}× in a row: ${call.short}`)
163    }
164    return ran
165  }).catch(($, e, next) => next(e))
166
167  on('command.run', { command: 'retry-breaker' }, async ($, e) => {
168    const words = String(e.args ?? '').trim().split(/\s+/).filter(Boolean)
169    const verb = (words[0] ?? 'status').toLowerCase()
170    if (verb === 'off') {
171      await $.store.set('isOff', true)
172      await update($, streaks, () => ({}))
173      return { text: 'retry-breaker is off: repeated failures are no longer tracked. /retry-breaker on to resume.' }
174    }
175    if (verb === 'on') {
176      await $.store.set('isOff', false)
177      return { text: `retry-breaker is on: warns after ${await thresholdOf($)} identical failures in a row.` }
178    }
179    if (verb === 'threshold') {
180      const n = Number(words[1])
181      if (!Number.isInteger(n) || n < 2 || n > 50) return { text: 'Usage: /retry-breaker threshold N, N a whole number from 2 to 50.' }
182      await $.store.set('threshold', n)
183      return { text: `retry-breaker: warns after ${n} identical failures in a row.` }
184    }
185    if (verb !== 'status') return { text: 'Usage: /retry-breaker [status | threshold N | off | on]' }
186    const off = await isOff($)
187    const threshold = await thresholdOf($)
188    const list = Object.entries(await read($, streaks)).sort((a, b) => b[1].lastAt - a[1].lastAt)
189    const head = `retry-breaker: ${off ? 'off' : 'on'}, threshold ${threshold}`
190    if (list.length === 0) return { text: `${head}. No failing streaks.` }
191    const who = (loop: string, s: Streak) => s.label ?? (loop === 'main' ? 'Claude' : `agent ${loop.slice(0, 8)}`)
192    const rows = list.map(([loop, s]) => `  ${who(loop, s)}: ${s.short} failed ${s.count}× (last error: ${s.lastError})${s.isDismissed ? ' [dismissed]' : ''}`)
193    return { text: [`${head}. Active streaks:`, ...rows].join('\n') }
194  })
195
196  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
197    const below = await next(e)
198    if (e.props.hasSurvey) return below
199    const all = await read($, streaks)
200    const shown = Object.entries(all)
201      .filter(([, s]) => s.isShown && !s.isDismissed)
202      .sort((a, b) => b[1].lastAt - a[1].lastAt)
203    if (shown.length === 0) return below
204    const { Box, Button, Text } = $.ui.resolve(e)
205    const dismiss = (loop: string, key: string) => () =>
206      void update($, streaks, cur => (cur[loop]?.key === key ? { ...cur, [loop]: { ...cur[loop], isDismissed: true } } : cur))
207    const extra = shown.length - MAX_ROWS
208    const mine = (
209      <Box flexDirection="column" width={e.props.bodyColumns}>
210        {shown.slice(0, MAX_ROWS).map(([loop, s]) => (
211          <Box key={`row-${loop}`} flexDirection="row" columnGap={1}>
212            <Box flexShrink={1}>
213              <Text wrap="truncate-end">
214                <Text color="#f59e0b" bold>
215                  [LOOP]{' '}
216                </Text>
217                {s.label ?? 'Claude'}: {s.short} failed {s.count}× (last error: {s.lastError})
218              </Text>
219            </Box>
220            <Button key={`dismiss-${loop}`} label="Dismiss" onPress={dismiss(loop, s.key)} />
221          </Box>
222        ))}
223        {extra > 0 ? <Text dimColor>+{extra} more (/retry-breaker)</Text> : null}
224      </Box>
225    )
226    return (
227      <Box flexDirection="column">
228        {mine}
229        {below}
230      </Box>
231    )
232  })
233}
234

1/** One loop's current run of identical failing calls. */
2export type Streak = {
3  /** tool + normalized input; a different key from the same loop starts a new streak */
4  key: string
5  tool: string
6  /** the call as shown: the shell command, or tool + input, clipped */
7  short: string
8  /** consecutive identical failures so far */
9  count: number
10  /** first line of the last error, clipped */
11  lastError: string
12  /** clock time of the last failure */
13  lastAt: number
14  /** successful calls of other commands since the last failure; OK_RESET of them end the streak */
15  okSince?: number
16  /** who: 'Claude' for the main loop, the agent's description/type, or 'agent <id>' */
17  label?: string
18  /** reached the threshold: toasted once and drawn in the band */
19  isShown: boolean
20  /** the person dismissed its row; stays hidden until the streak resets */
21  isDismissed: boolean
22}
23
24declare module 'claude-code' {
25  interface PluginState {
26    'retry-breaker': {
27      /** loop ('main' or the agent id) -> its current streak */
28      streaks: Record<string, Streak>
29    }
30  }
31}
32